Compare commits

...
Author SHA1 Message Date
Ljy-0827 a6de73aea3 feat(vom): add typed visual refs and target identity validation 2026-09-10 12:21:20 +08:00
Ljy-0827 fecb38b44c refactor(vom): preserve complete canvas candidates with exact deduplication 2026-09-10 12:21:19 +08:00
Ljy-0827 52a8e65b34 fix(vom): fix border radius incorrectly reject canvas 2026-09-10 12:21:19 +08:00
Ljy-0827 eb6cc0419b feat(vom): discover canvas candidates and resolve visible regions 2026-09-10 12:21:19 +08:00
Zhang GH ba6a14de7c Merge pull request #127 from NianJiuZst/codex/scroll-to-element-primitive
feat(interaction): add scroll-to-element primitive
2026-09-10 11:04:06 +08:00
drakezhang fd0e9de61b docs: document scroll-to primitive contract 2026-09-10 10:15:57 +08:00
drakezhang 77e3f5e1f3 feat(dsh-plugin): expose scroll-to action 2026-09-10 00:05:52 +08:00
drakezhang 61c97c9cb5 fix(interaction): honor scroll cancellation and visible bounds 2026-09-10 00:05:48 +08:00
drakezhang 29355e041d merge: integrate main into scroll-to PR 2026-09-09 23:47:27 +08:00
Zhang GH cff1ea62da Merge pull request #126 from NianJiuZst/codex/focus-blur-primitive
feat(interaction): add explicit focus and blur primitives
2026-09-09 23:23:50 +08:00
drakezhang 20e8770006 feat(dsh-plugin): expose focus and blur actions 2026-09-09 22:47:47 +08:00
drakezhang ac9e7948bf fix(interaction): verify focus changes and honor cancellation 2026-09-09 22:47:42 +08:00
drakezhang aadc1376ff merge: integrate main into focus and blur PR 2026-09-09 22:33:48 +08:00
Zhang GH 0fe36bc0b1 Merge pull request #206 from Tencent/fix/oopif-scrollbar-geometry
fix(geometry): preserve OOPIF scale with occupied scrollbars
2026-09-09 17:45:21 +08:00
drakezhang a62a28d8cd fix(geometry): preserve OOPIF scale with occupied scrollbars 2026-09-09 16:58:25 +08:00
Hannah Liu 896c7944d8 Merge pull request #203 from Tencent/fix/capture-document-consistency
fix(observation): validate DOM identity across capture phases | 校验跨采集阶段的 DOM 身份一致性
2026-09-09 16:26:24 +08:00
Hannah Liu 7e5f3bae26 Merge pull request #200 from Tencent/fix/observation-document-consistency
refactor(observation):  unify cross-frame capture coordination and data ownership | 统一跨 frame 采集协调与数据归属
2026-09-09 16:25:52 +08:00
Hannah Liu af4c2fa229 Merge pull request #198 from Tencent/refactor/document-capture-facts
refactor(observation): decouple DOM processing from capture orchestration | 解耦 DOM 处理与采集流程
2026-09-09 16:24:42 +08:00
Hannah Liu ee10da6fe9 Merge pull request #195 from Tencent/fix/frame-geometry-contracts
fix(geometry): unify cross-frame coordinates and screenshot conversion | 统一跨 frame 坐标与截图转换
2026-09-09 16:24:06 +08:00
drakezhang 4834d8b5fa test(geometry): identify the regression browser attachment 2026-09-09 16:11:39 +08:00
Ljy-0827 b4006d5b85 perf(observation): validate document identity against snapshot roots 2026-09-09 16:09:09 +08:00
Ljy-0827 2d1e5c6d9d fix(observation): validate document consistency with bounded identity checks 2026-09-09 16:09:09 +08:00
drakezhang 282e6f5623 fix(observation): carry snapshot units through capture coordination 2026-09-09 16:05:42 +08:00
Ljy-0827 bd68c293b6 perf(observation): collect frame AX trees with bounded concurrency 2026-09-09 15:58:56 +08:00
Ljy-0827 35a20582ad refactor(observation): separate capture coordination from document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 4e3365450a refactor(observation): integrate capture orchestration and document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 df5ed1a495 fix(observation): validate document consistency during capture 2026-09-09 15:58:40 +08:00
drakezhang eefbe03258 fix(observation): preserve snapshot unit normalization after document extraction 2026-09-09 15:58:28 +08:00
Ljy-0827 1d275f047c perf(observation): reduce document processing scheduling overhead 2026-09-09 15:54:41 +08:00
Ljy-0827 388b3ae2d5 refactor(observation): isolate document processing from capture orchestration 2026-09-09 15:54:41 +08:00
Ljy-0827 36d0d91419 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-09 15:54:41 +08:00
Ljy-0827 bcb9beb4f8 refactor(observation): defer visual facts until visual discovery 2026-09-09 15:54:41 +08:00
Ljy-0827 34ccb96fd6 fix(observation): await capture task cleanup before propagating cancellation 2026-09-09 15:54:41 +08:00
Ljy-0827 8dfa097e55 refactor(observation): organize capture around document facts 2026-09-09 15:54:41 +08:00
drakezhang 383e3128eb fix(geometry): normalize snapshot layout bounds and scroll together 2026-09-09 15:53:35 +08:00
Ljy-0827 2d2636030d perf(geometry): batch iframe owner size measurements 2026-09-09 13:20:21 +08:00
Ljy-0827 a28b9d55e7 fix(vom): surface incomplete iframe geometry in observations 2026-09-09 13:20:21 +08:00
Ljy-0827 c39874591f perf(observation): measure sibling iframe geometry with bounded concurrency 2026-09-09 13:20:21 +08:00
Ljy-0827 1485addfd5 fix(geometry): unify frame projection and screenshot coordinates 2026-09-09 13:20:21 +08:00
Zhang GH 90b0ff301b Merge pull request #204 from Tencent/release/0.2.1
Release/0.2.1
2026-09-09 11:35:05 +08:00
drakezhang e74f354e90 fix(release): support changelogs without unreleased section 2026-09-08 20:38:17 +08:00
drakezhang 05c036518a chore(release): 0.2.1 2026-09-08 20:35:14 +08:00
Zhang GH 020fb3212e Merge pull request #201 from Tencent/fix/windows-install-and-cancellation
fix(windows): harden installation and parent cancellation
2026-09-08 20:15:12 +08:00
Zhang GH 0822c53c65 fix(windows): preserve transfer cleanup after cancellation 2026-09-08 20:04:24 +08:00
Zhang GH 4450fbe28c Merge pull request #196 from Tencent/codex/dsh-plugin-docs
docs(dsh-plugin): clarify installation, configuration, and releases
2026-09-08 19:43:24 +08:00
drakezhang a3bb81bfab docs(dsh-plugin): clarify patch editing and reload behavior 2026-09-08 19:34:25 +08:00
Zhang GH caea98cd52 fix(windows): harden installation and parent cancellation 2026-09-08 19:21:54 +08:00
drakezhang 187b088013 docs(dsh-plugin): restore configuration guidance and simplify releases 2026-09-08 19:03:40 +08:00
drakezhang 5046b5ab44 docs(dsh-plugin): clarify setup and verify npm publishing 2026-09-08 18:44:20 +08:00
Zhang GH e8137b42cc Merge pull request #194 from Tencent/fix/dsh-runtime-peer-dependency
fix(dsh-plugin): remove obsolete client runtime host dependency
2026-09-08 17:34:33 +08:00
Zhang GH cb5dc148d5 fix(dsh-plugin): remove obsolete client runtime host dependency
Keep dsh-client-runtime as a development-only type dependency. Remove its peer metadata, client injection entry, and runtime external allowance so newer hosts do not preflight a package they no longer ship.

Fixes #193
2026-09-08 16:58:20 +08:00
Zhang GH 3c5f838c56 Merge pull request #192 from Tencent/fix/windows-update-reliability
fix(update): make Windows self-updates reliable
2026-09-07 20:37:32 +08:00
iuyo5678 9337bd557b test(update): use blocking connections in Windows release server 2026-09-07 20:27:04 +08:00
iuyo5678 162a31ae92 fix(update): make Windows self-updates reliable 2026-09-07 20:04:15 +08:00
Zhang GH 2f84234a31 Merge pull request #191 from Tencent/fix/windows-ipc-process-liveness
fix(windows): preserve pipe continuity and verify process liveness
2026-09-07 19:06:38 +08:00
iuyo5678 72c10aab75 fix(windows): preserve pipe continuity and verify process liveness 2026-09-07 18:25:32 +08:00
Zhang GH 1eeb55af02 Merge pull request #190 from Tencent/fix/dsh-observation-sync-and-capture
fix(dsh): synchronize observation state and manage thumbnail capture
2026-09-07 17:46:31 +08:00
drakezhang 4992a54864 docs(dsh): clarify observation subscription and snapshot contracts 2026-09-07 17:38:35 +08:00
drakezhang 8b195eb0df test(extension): return tab id from navigation recovery mock 2026-09-07 17:26:01 +08:00
drakezhang eae4db1729 Merge main for navigation recovery test compatibility 2026-09-07 17:25:22 +08:00
drakezhang e961395be9 fix(dsh): synchronize observation state and manage thumbnail capture 2026-09-07 16:24:04 +08:00
Zhang GH a6d1d02acc Merge pull request #179 from Tencent/fix/cdp-extension-frame-recovery
fix: recover navigation from cross-extension CDP restrictions
2026-09-07 16:05:46 +08:00
Zhang GH 5ebe609c40 Merge pull request #189 from Tencent/fix/verify-fill-before-success
fix(extension): verify fill results before reporting success
2026-09-07 15:59:21 +08:00
drakezhang 321494d93f test(fill): cover caret positioning and clarify append behavior 2026-09-07 15:56:13 +08:00
drakezhang f2184e74c5 fix(fill): support background input and automatic append 2026-09-07 15:28:21 +08:00
drakezhang eeb633d68e fix(extension): verify fill results before reporting success 2026-09-07 13:25:54 +08:00
Zhang GH 6d181105f2 Merge pull request #188 from Tencent/fix/match-form-state-by-element
fix(extension): match form state by element identity
2026-09-07 12:57:42 +08:00
drakezhang ebad0ed1b5 fix(extension): match form state by element identity 2026-09-07 12:21:01 +08:00
Zhang GH 498f8fb4a7 Merge pull request #187 from Tencent/fix/release-automation-state-on-tab-return
Fix/release automation state on tab return
2026-09-07 12:12:41 +08:00
drakezhang cef59137e4 fix(extension): handle tab return cleanup edge cases 2026-09-07 11:44:11 +08:00
drakezhang f70978bfcf fix(extension): release automation state when returning borrowed tabs 2026-09-07 11:30:01 +08:00
Zhang GH 1498cd425c Merge pull request #186 from Tencent/fix/closed-borrowed-tabs-block-session-stop
fix(extension): prevent closed borrowed tabs from blocking session stop
2026-09-07 11:19:12 +08:00
drakezhang e831d65db3 fix(extension): prevent closed borrowed tabs from blocking session stop 2026-09-07 11:12:44 +08:00
Zhang GH 47ac947f18 Merge pull request #174 from iAstro/fix/extension-locale-normalization
fix(i18n): match browser locale by language subtag instead of exact code
2026-09-04 17:29:31 +08:00
Zhang GH 69d64c4eb1 Merge pull request #176 from Tencent/fix/windows-update-detach
fix(bsk): fix Windows self-update failing to replace binary
2026-09-04 17:11:31 +08:00
Zhang GH 99c997d6d5 Merge pull request #66 from iAstro/fix/issue-57-user-tab-freedom
fix(extension): keep user-opened tabs free & fix agent-tab leak on session stop (issue #57)
2026-09-04 17:10:15 +08:00
iAstro 93853ebc65 refactor(i18n): drive locale normalisation from shipped resource keys
Per review feedback on #174: the hardcoded if/else in
normalizeLanguageCode required touching the function for every new
language, so a bare `ko` would miss the `ko-KR` resource #161 is about
to add and Korean users would land on English despite the fix.

Replace it with createLanguageNormalizer(resourceKeys), a factory that
derives its mapping from the resource keys i18next is configured with:

  1. exact resource key              -> unchanged
  2. one resource for the language   -> that key (ko -> ko-KR)
  3. several (zh-CN / zh-TW)         -> script subtag, then region
  4. nothing shipped                 -> unchanged, fallbackLng applies

getLanguageDetectionOptions now takes the resource keys and i18n.ts
passes Object.keys(resources). Chinese disambiguation stays as the only
special case - one language, two scripts; every other mapping derives
from the index automatically.

Test expectations for the zh-TW family change from "zh-TW" to "zh-CN":
only zh-CN ships today and the factory maps to shipped keys only, so
Traditional resolves directly to Simplified instead of leaking into the
fallback chain. The rendered UI is identical either way, and the new
extensibility cases lock in the automatic disambiguation once a zh-TW
bundle registers - plus the auto-resolution of a newly registered
language like ko-KR.

28 -> 31 tests; extension suite 832/832; tsc, biome and ext:build clean.
2026-09-04 15:42:40 +08:00
drakezhang 248bb90a2d test(extension): return home tab ids from mocks 2026-09-04 14:37:04 +08:00
drakezhang 6e6c178434 Merge remote-tracking branch 'origin/main' into fix/issue-57-explicit-tab-claims 2026-09-04 14:31:48 +08:00
drakezhang 386ef91c24 fix(extension): preserve existing tab scope semantics 2026-09-04 13:14:26 +08:00
drakezhang 0dc11b30ea fix(extension): claim agent tabs explicitly 2026-09-04 11:17:05 +08:00
iAstro 759eebd4b6 fix(i18n): match browser locale by language subtag instead of exact code
The extension ships en-US and zh-CN resources, but only en-US was ever
selected: every other browser language fell through to fallbackLng
("zh-CN"), so users on en, en-GB, en-CN, en-AU, ja-JP, fr-FR saw Chinese.

Match on the BCP 47 language subtag and normalise it to a resource key:

  en*                                -> en-US
  zh-Hans* / zh-CN / zh-SG / zh      -> zh-CN
  zh-Hant* / zh-TW / zh-HK / zh-MO   -> zh-TW
  anything else                      -> unchanged (fallbackLng default: en-US)

Simplified vs Traditional is decided by the script subtag (Hans/Hant) when
present, then by the region subtag, since Chrome reports zh-CN / zh-TW
rather than script subtags. The zh-TW branch is written now so adding a
Traditional translation later needs no code change.

chrome.i18n.getUILanguage() is now preferred over navigator.language, which
is unreliable on the chrome-extension:// origin the popup runs on.

Adds a locale-resolution regression matrix (28 cases).

Closes #168
2026-09-04 10:03:43 +08:00
drakezhang fd3ce202a1 fix: recover navigation from cross-extension CDP restrictions
Classify Chrome extension-frame access failures and provide an actionable CLI hint. Recover navigate and reload only when their initial CDP setup is denied, using browser navigation events and verifying that CDP access returns.

Preserve normal navigation, Agent Window checks, transfer error details, and network-idle semantics. Never replay an already-dispatched navigation.
2026-09-03 18:21:21 +08:00
drakezhangandCursor 618bd6a0df fix(bsk): fix Windows self-update failing to replace binary
Two issues prevented the detached update helper from completing:

1. Only CREATE_NO_WINDOW was set — the child cmd.exe process was still
   attached to the parent console session and got killed when the
   terminal closed. Add CREATE_NEW_PROCESS_GROUP to fully detach it.

2. `timeout /t 1 /nobreak` requires a console and fails silently in a
   no-window context, causing the retry loop to spin with no delay.
   Replace with `ping -n 2 127.0.0.1` which works in any environment.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-03 12:24:53 +08:00
Zhang GH c455a09fc3 Merge pull request #163 from dayi/fix/recording-http-random-uuid
fix(extension): support UUID generation on insecure origins
2026-09-03 10:46:12 +08:00
Dean Wang b6be356e40 test(extension): cover UUID fallback during recording startup 2026-09-03 07:40:57 +08:00
Zhang GH 945bf1523d Merge pull request #173 from Tencent/release/0.2.0
Release/0.2.0
2026-09-02 19:00:21 +08:00
drakezhangandCursor a928a9fb76 test: update popup version display assertions for CLI/Ext labels
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 18:32:33 +08:00
drakezhangandCursor 0b2d7e95cc style: format release script for biome
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 18:26:35 +08:00
drakezhangandCursor 556bea1645 chore(release): 0.2.0
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 18:17:56 +08:00
drakezhangandCursor 0302d5592b chore: add release script, changelog, and version display labels
- scripts/release.mjs: unified version bump + tag + push automation
- CHANGELOG.md: document all changes from 0.1.x through 0.2.0
- popup: show "CLI {ver} / Ext {ver}" instead of bare version numbers

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 18:05:32 +08:00
Hannah Liu eb34ec0dbf Merge pull request #124 from Tencent/feat/expand-upload-download
feat(bsk): support drop to upload
2026-09-02 17:56:55 +08:00
drakezhangandCursor 43067b6bfe style: remove extra blank lines in file-transfer test
Fix biome formatter error on CI.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 17:52:57 +08:00
drakezhangandCursor 6cc0d38489 docs(skill): compress CLI SKILL.md with file transfer guidance
Restore the lean style from PR #167 while integrating essential file
transfer documentation for upload/download. Reduces the skill from
362 lines / 26k chars to 180 lines / 9.2k chars.

A/B eval (24 browser runs, Composer 2.5 Fast, core suite) confirms
no pass-rate regression: candidate 12/12 vs baseline 10/12 (baseline
failures were a network blip and a behavioral variance, not skill
quality).

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 17:43:58 +08:00
drakezhangandCursor 0b71963344 merge: resolve conflicts with updated feat/upload-download base
Keep feat/expand-upload-download (drop mode superset) over the base
branch version for all 12 conflicting files. The drop-to-upload branch
already includes all upload/download functionality plus the new --mode
drop mechanism, overlay suppression, OOPIF coordinate handling, and
expanded error reasons.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 17:03:35 +08:00
Hannah Liu 59e2c3224e Merge pull request #123 from Tencent/feat/upload-download
feat(bsk): add secure, session-scoped file upload and download | 支持安全、会话级的文件上传与下载
2026-09-02 16:47:19 +08:00
drakezhangandCursor d70df753f6 merge: resolve SKILL.md conflicts with main
Keep the expanded command-reference tables from feat/upload-download
(including the new File Transfer section) over main's compact two-line
format.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 16:42:15 +08:00
Zhang GH 66ae555e40 Merge pull request #172 from Tencent/fix/pip-close-and-obs-mediatype
fix(dsh-plugin): PiP window close button & observation thumbnail media type detection
2026-09-02 16:25:33 +08:00
drakezhangandCursor fee7c0983c fix: sort import order to satisfy biome organizeImports
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 16:21:26 +08:00
drakezhangandCursor 51473c01ff fix(dsh-plugin): add PiP close button and sniff observation thumbnail media type
The Document PiP window had no close/return entry once popped out —
users could not dismiss it from inside the window. Add an onClosePip
callback and render a close button in the PiP header; clicking it
calls pipWindow.close() and the content returns to its carrier.

The observation thumbnail publisher hardcoded mediaType as "image/png"
regardless of the actual capture bytes. Reuse the existing
sniffImageMediaType helper (exported from image.ts) so the HTTP
thumbnail endpoint declares the correct Content-Type when Chromium
returns JPEG captures.

Closes #153 (Bug 2 fully; Bug 1 was fixed in #135, this covers the
remaining observation-thumbnail path).

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 16:14:24 +08:00
Zhang GH 49744233f7 Merge pull request #171 from Tencent/fix/borrow-confirmation-ux
fix(borrow): improve confirmation UX — proactive focus and longer tim…
2026-09-02 15:23:12 +08:00
drakezhangandCursor 95571f8b70 fix(borrow): improve confirmation UX — proactive focus and longer timeout
The borrow confirmation overlay was easily missed because the Agent
Window stole focus and the 5-second auto-deny was too aggressive for
users who had context-switched away.

- Focus the user window proactively when a borrow request is issued so
  the overlay is immediately visible even when the Agent Window was
  last focused.
- Raise CONFIRMATION_TIMEOUT_MS from 5 s to 60 s, giving users enough
  time to notice and respond — the OS notification with
  requireInteraction: true now has a matching window of usefulness.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 15:11:40 +08:00
Zhang GH 8344e311fa Merge pull request #169 from Tencent/perf/vom-hover-probe-opt-in
Perf/vom hover probe opt in
2026-09-02 14:27:29 +08:00
drakezhangandCursor 63338cd98e Merge branch 'main' into perf/vom-hover-probe-opt-in
main rewrote SKILL.md into a leaner core (#167) while this branch documented
the opt-in --probe-hover flag against the old structure. Keep main's rewritten
prose and re-apply the probe-hover guidance on top of it: the [has-submenu] /
[expanded] markers, when reaching for --probe-hover is warranted, and its slot
in the page-reading escalation ladder ahead of get-html/screenshot.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 12:18:06 +08:00
Zhang GH bd193ceb5d Merge pull request #167 from Tencent/refactor/lean-browser-skill
Refactor/lean browser skill
2026-09-02 12:09:42 +08:00
Zhang GH 5ed828736f Merge pull request #165 from Tencent/perf/vom-algorithmic-optimizations
perf(vom): optimize rendering algorithms
2026-09-02 12:09:22 +08:00
drakezhang ee620acf64 docs(skill): give probe-hover a decidable trigger condition
The previous wording asked the agent to notice when a page "looks like"
it hides content behind hover. That is not something an agent reading a
text tree can judge: hover-only content is invisible by definition, and
the marker that used to point at it came from the probing that is now off
by default.

Replaces it with a condition the agent can actually evaluate — an
expected control is missing and no [has-submenu] marker points at a
trigger — and places it on the escalation ladder ahead of get-html and
screenshot, which is where that dead end is currently reached.

Also documents [has-submenu]/[expanded] as the primary hover signal. It
comes from aria-haspopup in the accessibility tree, costs nothing, and is
unaffected by probing being off, so it covers menus that are marked up
correctly.
2026-09-02 10:59:18 +08:00
drakezhang 05a36075b8 perf(vom): make active hover probing opt-in on observe
Every tool.observe moved the real cursor across the page to look for
hover-only menus and tooltips. Measured against real Chrome, that cost a
median of 2.5s and up to 4.7s per observation — 73% to 99% of total
observe wall clock — and returned 3 hits across 12 pages. A positive
control confirms the probes work; the yield on real pages simply does not
justify putting them on the default path.

observe now probes only when asked, via `probe_hover` (`--probe-hover`).
Agents that know which element hides a menu should keep using
`bsk hover <ref>`, which is both cheaper and precise. A held hover latch
still suppresses probing, since probing would move the cursor off the
element the caller is deliberately holding.

Correctness fixes that apply whether or not probing is requested:

- Hover probing ran between the DOM snapshot and the accessibility tree
  fetch, so a hover that opened a menu left one half of an observation
  describing the page before the change and the other half after it. Both
  chains now run after all capture is complete.
- The overlay bypass toggled once per chain and swallowed restore
  failures, which could leave the agent overlay hidden for the rest of
  the tab's life. It is now reference counted across both chains, and a
  failed restore is reported instead of discarded.
- Observations report whether they hovered the page and whether that
  revealed content the static tree lacks, so callers can judge how far
  the live page may have drifted from the returned snapshot.

Probe budgets were checked at the top of the loop, so a candidate could
start with 1ms remaining and still run a full settle window, overshooting
by up to 634ms. Candidates are now admitted only if they fit. Raising
MAX_HOVER_PROBE_MS to 2600 is part of that fix rather than a relaxation:
2000 was never the real ceiling, and enforcing it literally would have
silently dropped a candidate the positive control depends on.

Tooltip misses are cached too, with a shorter TTL than hits, so repeat
observations of the same page no longer re-probe every unlabelled control
from scratch.
2026-09-02 10:59:18 +08:00
Ljy-0827 1bd13d5c3c fix(bsk): fix upload download racing 2026-09-02 10:56:02 +08:00
drakezhangandCursor 59d6533eff docs(skill): trim agent instructions to a leaner core
The CLI skill had grown to 20.8 KB of command tables and flag catalogs that
`bsk --help` already answers at runtime, so every agent paid for them on every
task. Rewrite both skills around the durable rules an agent cannot rediscover
-- session lifecycle, the observe/act/observe loop, the Agent Window boundary,
and the stop-on-success contract -- and delegate flag details to the CLI's own
help output. The CLI skill drops to 6.9 KB (-67%) and the DSH skill to 5.1 KB.

Keep the parts that help discovery has no answer for: the literal
`[hover first: ...]` observation marker, the `request-help` outcome values, the
complete command-name inventory, and the required flags agents guess wrong
(`fill --value`, `select --value`, `screenshot --out`, `emulate --device`).
Also bind `--no-focus` explicitly to `bsk session start`, since a bare mention
led agents to pass it to `bsk navigate`.

Validated with 36 agent runs of the `core` eval suite against a real browser,
comparing the old and new instructions with everything else held constant.
Pass rate did not regress (the lean version was better on `form-controls`) and
token cost per task came out even. The inventory and flag signatures cut
`--help` round-trips from 1.7 to 0.7 per task and removed the invented
`bsk back` / `bsk forward` calls the first draft produced.

Tighten the DSH size assertion to a range so the budget is bounded from both
sides and accidental truncation fails the test.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-01 18:39:19 +08:00
drakezhangandCursor 536d3a9844 chore: ignore local pnpm store and git worktrees
Both directories are local scratch space that kept showing up as untracked
noise in `git status`.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-01 18:39:06 +08:00
iAstro a77353bce8 fix(extension): classify the home tab by id, not a racy boot flag (issue #57)
SessionManager.classifyNewTab's `initializing` branch relied on a
windowInitializing flag that start() cleared *before* the home tab's
chrome.tabs.onCreated event was dispatched (the event is asynchronous), so
the branch never ran in production and the home tab's classification was
subject to a timing race. Match the home tab by its tab id instead — the id
is already returned by ensureActiveTab and is reliable regardless of event
ordering.

Also refresh the overlay-bridge docstring: overlay.ready now makes the
background *push* the per-tab overlay state (and the Agent Window also
pushes on tab create / control-mode change), rather than a request/reply.
2026-09-01 15:45:36 +08:00
iAstro 91621b2545 fix(extension): release the pending agent-tab slot when tab creation fails
tool.tab_create increments pendingAgentTabCount *before* chrome.tabs.create
so the onCreated listener can tell an agent tab from a user-opened tab. When
the create failed, no onCreated event arrived and the counter leaked, so the
next tab the user opened was misclassified as agent-created — the exact
confusion the counter exists to prevent, a variant of issue #57.

handleTabCreate now releases the slot on failure via
SessionManager.releaseAgentTabPending. An abort (cancelled) is excluded: the
tab did open (and was cleaned up), so its onCreated still consumes the slot.

Added a regression test asserting that after a failed tab_create, a
user-opened tab is still classified as 'user' (not 'agent').
2026-09-01 15:36:44 +08:00
drakezhang 641d934337 perf(vom): optimize rendering algorithms 2026-09-01 14:55:36 +08:00
Ljy-0827 a4fed136e3 fix(extension): keep user-opened tabs free & fix agent-tab leak on session stop (issue #57)
- session_stop: exclude tabs still tracked in agentCreatedTabs from the
  release decision so a tab that failed to close is no longer mistaken for
  a user tab (which previously kept the window open and leaked the tab).
- Distinguish user-created tabs from agent-created tabs in the Agent Window
  via a pending-tab counter consumed by chrome.tabs.onCreated, so tabs the
  user opens themselves (new-tab button / Cmd+T / open in new tab) are never
  taken over by the BrowserSkill control mask.
- background overlay: decide per-tab state BEFORE showing controls
  (overlayStateForTab + pushOverlayStateForTab) so a user tab receives hidden
  on its first overlay.ready ping instead of flashing control then hiding.
- AgentWindowApi.ensureActiveTab now resolves with the activated/created tab
  id instead of void, so SessionManager can record homeTabId. This fixes the
  tsc failure flagged in review (the interface previously returned
  Promise<void> while callers treated the result as number | null).
- Wire the production tabManagement.tabs / tabsQuery deps into
  tool.session_stop, both in the dispatcher and in the disconnect cleanup.
  They were never injected, which left the agent-tab cleanup and the
  window-release path as dead code outside tests (behaviour identical to
  main). Added a dispatcher-level integration test that fails without the
  wiring: a surviving user tab must release the window, not close it.
2026-09-01 11:24:56 +08:00
Dean Wang 3c77ef7b01 fix(extension): support UUID generation on insecure origins 2026-09-01 10:30:40 +08:00
Ljy-0827 3b21f2f0a7 fix(bsk): download bug fix 2026-08-31 17:58:33 +08:00
Ljy-0827 405433ab65 fix(bsk): fix ci 2026-08-31 17:58:33 +08:00
Ljy-0827 58c7f38a05 feat(bsk): harden browser file transfer transactions 2026-08-31 17:58:33 +08:00
Ljy-0827 e0c88295c7 fix(bsk): fix ci 2026-08-31 17:58:33 +08:00
Ljy-0827 dc36e38cb9 fix(bsk): upload function safety 2026-08-31 17:58:33 +08:00
Ljy-0827 d5de9b0154 feat(bsk): fix upload function that cannot bypass layout 2026-08-31 17:58:33 +08:00
Ljy-0827 995061514c feat(bsk): fix different type of upload 2026-08-31 17:58:33 +08:00
Ljy-0827 406bb21b7e feat(bsk): support upload and download 2026-08-31 17:58:33 +08:00
Zhang GH f06a514bd2 Merge pull request #149 from Tencent/fix/protocol-upgrade-reminder
fix:将协议版本差异改为升级提醒
2026-08-31 12:25:29 +08:00
Zhang GH 1e53ee63ae Merge pull request #155 from Tencent/ci/edge-addons-publishing
ci(extension): publish updates to Microsoft Edge Add-ons
2026-08-31 11:24:22 +08:00
Zhang GH 120dc441bb Merge pull request #158 from Tencent/codex/fix-dsh-screenshot-test-name
test(dsh-plugin): use renamed screenshot tool
2026-08-29 16:08:01 +08:00
drakezhang 710b1219f4 test(dsh-plugin): use renamed screenshot tool 2026-08-29 16:05:06 +08:00
Zhang GH 177ecd3ebf Merge pull request #157 from Tencent/refactor/simplify-dsh-commands
Refactor/simplify dsh commands
2026-08-29 15:55:16 +08:00
Zhang GH 0e6b9c4542 Merge pull request #135 from cyhano/fix/screenshot-media-type
fix: use detected media type for screenshots instead of hard-coded image/png
2026-08-29 15:52:43 +08:00
drakezhang 506adbe708 Merge remote-tracking branch 'origin/main' into refactor/simplify-dsh-commands 2026-08-29 15:16:31 +08:00
Zhang GH 984d017b0f Merge pull request #156 from Tencent/fix/unify-dsh-plugin-package-id
fix(dsh-plugin): use the published package name as the Cordis id
2026-08-29 15:12:02 +08:00
drakezhang 5df93143c5 refactor(dsh-plugin): simplify browser commands 2026-08-29 14:38:48 +08:00
Zhang GH 7163f2f9b0 Merge pull request #151 from xiyanjun/docs/evaluate-json-ok
docs(skill): clarify bsk evaluate exit-code vs --json .ok semantics
close #133
2026-08-28 22:06:04 +08:00
drakezhang e9f0584af3 ci(extension): publish updates to Microsoft Edge Add-ons 2026-08-28 21:39:53 +08:00
drakezhangandCursor 2997f495db fix(dsh-plugin): use the published package name as the Cordis id
dsh 0.1.0-rc.8 imports and registers the client bundle by export const name, so a leftover dsh-plugin-browserskill id fails to load the 0.1.1 web UI.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-28 21:28:06 +08:00
xiyanjun f96bd72c3c docs(skill): clarify bsk evaluate exit-code vs --json .ok semantics
The exit code reflects RPC success only, not JS success. Document that programmatic callers must use --json and check the .ok field to detect JS errors. Addresses the discoverability gap raised in issue #133.
2026-08-28 18:02:38 +08:00
Zhang GH 329c68054b Merge pull request #150 from Tencent/codex/browser-eval-harness
test: 增加可扩展的浏览器能力回归测试框架
2026-08-28 16:41:00 +08:00
drakezhang 5cd43553ed docs(eval): clarify agent-neutral scope 2026-08-28 15:36:38 +08:00
drakezhang 9ad33801c7 test: add extensible browser evaluation harness 2026-08-28 12:16:39 +08:00
Zhang GH 7aaa5a2d01 Merge pull request #147 from Tencent/fix/dsh-plugin-session-lifecycle
fix(dsh-plugin): stabilize skill loading and browser sessions
2026-08-28 12:11:08 +08:00
drakezhang 210b21b637 style(dsh-plugin): format observation 2026-08-28 12:06:47 +08:00
Hannah Liu 48f64ec0df Merge pull request #122 from Tencent/feat/vom-functional-refactor
feat(vom): refactor semantic structure generation for grouped observations | 重构语义结构生成,支持分组式 VOM 观察
2026-08-28 11:24:34 +08:00
“haonan”andCursor c48a53269e style: rustfmt doctor protocol-version test asserts
cargo fmt --check failed on two long assert lines in PR CI.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-27 22:50:01 +08:00
“haonan”andCursor 3bbd6ab19f fix: treat protocol version differences as upgrade reminders
Agents were stopping on popup and status copy that looked like a hard incompatibility. Keep the connection usable, reframe exit 5 as an unknown method, and ask users to upgrade instead of aborting.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-27 22:22:30 +08:00
drakezhang 1a70d2200e fix(dsh-plugin): stabilize skill loading and browser sessions
Register bundled browser instructions at agent scope, prioritize foreground commands over thumbnail capture, and unify cancellation and session cleanup.
2026-08-27 19:59:20 +08:00
Zhang GH 554861d6b1 Merge pull request #131 from Tencent/feat/dsh-browser-tools-parity
feat(dsh-plugin): add Browser tool parity and dedicated Harness skill
2026-08-27 16:29:30 +08:00
Ljy-0827 0d7f976335 fix(vom): fix repeat names 2026-08-26 15:28:27 +08:00
Ljy-0827 d1d3eb6a94 fix(vom): bug fix 2026-08-26 15:28:27 +08:00
Ljy-0827 38fd7d3f4c fix(vom): safety policy and rebase bug fix 2026-08-26 15:28:27 +08:00
Ljy-0827 b5062ac19b feat(vom): fix name lost 2026-08-26 15:28:27 +08:00
Ljy-0827 41cb576256 feat(vom): function refactor 2026-08-26 15:28:27 +08:00
Hannah Liu f060a7c074 Merge pull request #142 from Tencent/chore/release-ext-0.1.7
chore(release): extension 0.1.7
2026-08-26 15:19:33 +08:00
Ljy-0827 60b62d5420 chore(release): extension 0.1.7 2026-08-26 15:12:44 +08:00
Hannah Liu 70b851a3e8 Merge pull request #141 from Tencent/chore/release-bsk-0.1.11
chore(release): bsk 0.1.11
2026-08-26 15:06:30 +08:00
Ljy-0827 da3534d066 chore(release): bsk 0.1.11 2026-08-26 14:59:09 +08:00
BB-fat a004291848 feat(dsh-plugin): better-sidebar 集成、会话关停与归档清理、悬浮窗修复 (#130)
* better-sidebar 集成:observation 视图迁入 better-sidebar 载体,按会话作用域展示 bsk 会话
* 会话关停与归档清理:支持会话 dispose 与归档自动清理
* 悬浮窗修复:悬浮 overlay 拖拽/缩放/PiP 弹出等问题修复
* 修复 Document PiP 空白窗口:载体卸载(关 tab、切会话、插件 HMR)时主动 close() PiP 窗口,避免残留空白窗口
2026-08-25 10:33:46 +08:00
Ljy-0827 b1b1ae1d48 feat(bsk): update skill to support drop 2026-08-24 16:55:16 +08:00
Ljy-0827 8822bba9a5 feat(bsk): support drop to upload 2026-08-24 16:55:16 +08:00
Ljy-0827 c74c119a82 fix(bsk): download bug fix 2026-08-24 16:26:04 +08:00
cyhanoandDeepSeek Harness 60fbf30e24 fix: use detected media type for screenshots instead of hard-coded image/png
browser_screenshot declared every capture as image/png regardless of the
actual bytes. On Chromium/Edge builds where captureVisibleTab returns JPEG
bytes, the stored attachment reference recorded mediaType "image/png" against
JPEG bytes, so dsh-attachment-local's read-time metadata verification fails
with ATTACHMENT_CORRUPT ("Stored attachment metadata does not match its
reference"), surfacing as code UNKNOWN and triggering endless LLM retries.

- image.ts: sniff the capture's magic bytes (PNG/JPEG/WebP/GIF) and declare
  that real media type to attachments.saveImage, instead of always "image/png".
- tools.ts: assemble the tool result from ref.mediaType (the store's verified
  type) rather than re-declaring "image/png" as const, and relax the output
  schema's const to the accepted media-type enum.
- tests: add a regression case asserting JPEG capture bytes surface as
  image/jpeg.

Co-Authored-By: DeepSeek Harness <noreply@deepseek.com>
2026-08-24 15:31:34 +08:00
Ljy-0827 b801bc1f62 fix(bsk): fix ci 2026-08-24 15:02:52 +08:00
Ljy-0827 bf59753aa3 feat(bsk): harden browser file transfer transactions 2026-08-24 15:02:52 +08:00
Ljy-0827 a86cbf724f fix(bsk): fix ci 2026-08-24 15:00:22 +08:00
Ljy-0827 76d447caa0 fix(bsk): upload function safety 2026-08-24 15:00:22 +08:00
Ljy-0827 14e5bbe4fc feat(bsk): fix upload function that cannot bypass layout 2026-08-24 15:00:22 +08:00
Ljy-0827 bef722107f feat(bsk): fix different type of upload 2026-08-24 14:59:53 +08:00
Ljy-0827 6800f8fdfe feat(bsk): support upload and download 2026-08-24 14:59:11 +08:00
Hannah Liu 4b6cdde168 Merge pull request #115 from Tencent/fix/recorder-iframe
fix(recording): support user action recording in iframes and OOPIFs
2026-08-24 13:18:05 +08:00
Ljy-0827 4aade4b94f fix(recorder): bug fix, oopif load too late 2026-08-24 10:42:46 +08:00
Ljy-0827 616af1099f fix(record): saftey policy and rebase bug fix 2026-08-24 10:42:46 +08:00
Ljy-0827 3656f0dd3c fix(recorder): ci fix 2026-08-24 10:42:46 +08:00
Ljy-0827 9742d83213 fix(recorder): bug fix 2026-08-24 10:42:46 +08:00
Ljy-0827 615e866608 fix(recorder): add iframe support 2026-08-24 10:42:46 +08:00
Ljy-0827 c08fed5de3 fix(record): saftey policy and rebase bug fix 2026-08-24 10:42:45 +08:00
“haonan” 887d1ed0b9 fix(record): drop 'hover' in tracev2 for compatible 2026-08-24 10:42:45 +08:00
Ljy-0827 1a3d3b7de8 fix(record): bug fix 2026-08-24 10:42:45 +08:00
“haonan”andCursor 72834980d2 fix(record): record switch_tab and keep Trace v3 states per tab
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-24 10:42:45 +08:00
“haonan” 38b0debba9 fix(test): share one BSK_HOME lock across lib tests 2026-08-24 10:40:37 +08:00
“haonan” 4677149e5e fix(record): recover failed Trace exports and hint --url when the default start page fails
Reject legacy JSON --output before recording starts, persist the completed Trace before writing the bundle so `bsk record stop --output <dir>` can retry after an export failure, and tell users to pass --url when https://example.com/ does not load.
2026-08-23 21:26:40 +08:00
Ljy-0827 8ea5477e11 fix(record): safety problem and bug fix 2026-08-23 21:26:40 +08:00
“haonan” 17af1af65a feat(record): export Trace v3 bundles with settled page states 2026-08-23 21:26:40 +08:00
Ljy-0827 6300eda0aa fix(record): rebase and dto policy 2026-08-23 21:26:40 +08:00
Ljy-0827 87cdcfe5ca fix(recording): using unified geometry 2026-08-23 21:26:40 +08:00
“haonan”andCursor 21476c3884 feat(record): capture Trace v3 states after pages settle
Wait for DOM quiet after actions, match targets to VOM refs, and reduce
drafts into deduped state-linked Trace v3 while keeping the v2 reducer.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:40 +08:00
“haonan” c2a225377b fix(protocol): align trace schemas with v2/v3 classify rules
RecordedTrace validation now follows classify_value — numeric version
selects v3, otherwise v2 — so mixed pages/states and v2 envelopes with
a version field fail Schema checks before Recall import. Keep standalone
trace_step as a v2|v3 union for tools that validate a single step.
2026-08-23 21:26:39 +08:00
“haonan” e64f4cac9f fix(protocol): keep generic trace schema compatible with v2 and v3
Split standalone v2/v3 schemas and drop TraceV2 deny_unknown_fields so legacy traces with extra keys still parse; mix and version checks stay in RecordedTrace.
2026-08-23 21:26:39 +08:00
Ljy-0827 368507b07d fix(protocol): compatible with dto 2026-08-23 21:26:39 +08:00
Ljy-0827 b750eed5d4 fix(protocol): integrate with vom 2026-08-23 21:26:39 +08:00
“haonan” 3d159c0254 test(protocol): align handshake fixtures with 1.0 floor and clarify v2/v3 naming 2026-08-23 21:26:39 +08:00
“haonan”andCursor 524516cb56 fix(protocol): align Trace v2/v3 classification and hover steps
Require Trace v3 version const 3, reject mixed pages/states payloads, and
align TypeScript selection optionality with the generated schema. Trace v2
on main already records hover steps, so the frozen v2 schema must accept
them instead of dropping them during reduction.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:39 +08:00
“haonan”andCursor f98198949a feat(protocol): negotiate Trace v3 with v2 compatibility
Define Trace v3 wire types and generated schemas while keeping protocol
1.1 peers compatible with 1.0. Default record start still produces Trace
v2, and only an explicit trace_version=3 request enables the new shape.
Refs #90

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:39 +08:00
Zhang GH d05279f991 Merge pull request #98 from NianJiuZst/codex/fix-daemon-idle-close-race
fix(daemon): make idle disconnect updates atomic
2026-08-23 21:25:24 +08:00
NianJiuZst 7a10be21cb feat(interaction): add scroll-to-element primitive 2026-08-21 20:56:24 +08:00
NianJiuZst 0055e90f91 feat(interaction): add explicit focus and blur primitives 2026-08-21 19:14:40 +08:00
NianJiuZst 8352c3c67a fix(daemon): refresh stale idle-task comment
After folding the IPC connection count and activity timestamp into a
single mutex-protected IpcActivityState, the idle task no longer
references the connection count directly; should_exit_for_idle already
returns false while a connection is open or within the idle window.
Update the comment to describe the post-refactor control flow so it
matches the code.
2026-08-20 13:53:23 +08:00
NianJiuZst 4eedd98c1d fix(daemon): make idle disconnect updates atomic 2026-08-20 13:53:23 +08:00
355 changed files with 49509 additions and 6481 deletions
+29
View File
@@ -37,6 +37,35 @@ jobs:
- name: Run Rust tests
run: cargo test --workspace --locked
windows-daemon:
name: Windows daemon, installer and cancellation regression tests
runs-on: windows-latest
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- name: Run Windows named-pipe tests
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
- name: Run Windows process liveness tests
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
- name: Run Windows update tests
run: cargo test -p bsk --lib --locked cli::update::tests
- name: Run Windows update and cancellation process tests
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
- name: Run installer tests (Windows PowerShell 5.1)
shell: powershell
run: ./scripts/install-windows.test.ps1
- name: Run installer tests (PowerShell 7)
shell: pwsh
run: ./scripts/install-windows.test.ps1
frontend:
name: Frontend lint, typecheck, tests, build
runs-on: ubuntu-latest
+177
View File
@@ -325,3 +325,180 @@ jobs:
echo "Chrome Web Store submission created successfully:"
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
publish-edge-addons:
needs: [resolve, build-extension]
runs-on: ubuntu-latest
environment: edge-addons
permissions:
contents: read
steps:
- name: Download build artifact
uses: actions/download-artifact@v8
with:
name: extension-zip
path: dist
- name: Validate Microsoft Edge Add-ons configuration
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
run: |
set -euo pipefail
if [ -z "$API_KEY" ]; then
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
exit 1
fi
if [ -z "$CLIENT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
exit 1
fi
if [ -z "$PRODUCT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
exit 1
fi
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
exit 1
fi
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
if [ -z "$ARCHIVE" ]; then
echo "::error::No extension zip found in the downloaded artifact"
exit 1
fi
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
- name: Upload and publish Microsoft Edge Add-ons extension
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
operation_id_from_headers() {
local headers="$1"
local location
location="$(awk '
tolower($1) == "location:" {
sub(/\r$/, "")
sub(/^[^:]*:[[:space:]]*/, "")
print
exit
}
' "$headers")"
location="${location%%\?*}"
printf '%s' "${location##*/}"
}
poll_operation() {
local operation_name="$1"
local status_url="$2"
local status="InProgress"
local http_status
for attempt in $(seq 1 30); do
sleep 5
http_status="$(curl --silent --show-error \
--output "$STATUS_RESPONSE" \
--write-out '%{http_code}' \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
"$status_url")"
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
exit 1
fi
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
echo "$operation_name status check $attempt/30: $status"
case "$status" in
Succeeded)
return 0
;;
InProgress)
;;
Failed)
echo "::error::Microsoft Edge Add-ons $operation_name failed"
jq . "$STATUS_RESPONSE"
exit 1
;;
*)
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
jq . "$STATUS_RESPONSE"
exit 1
;;
esac
done
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
jq . "$STATUS_RESPONSE"
exit 1
}
HTTP_STATUS="$(curl --silent --show-error \
--output "$UPLOAD_RESPONSE" \
--dump-header "$UPLOAD_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/zip" \
--data-binary "@$ARCHIVE" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
exit 1
fi
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
if [ -z "$UPLOAD_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
cat "$UPLOAD_HEADERS"
exit 1
fi
poll_operation \
"package upload" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
HTTP_STATUS="$(curl --silent --show-error \
--output "$PUBLISH_RESPONSE" \
--dump-header "$PUBLISH_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/json" \
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
"$API_ROOT/products/$PRODUCT_ID/submissions")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
exit 1
fi
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
if [ -z "$PUBLISH_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
cat "$PUBLISH_HEADERS"
exit 1
fi
poll_operation \
"submission" \
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
echo "Microsoft Edge Add-ons submission created successfully:"
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
+8
View File
@@ -7,10 +7,18 @@ dist/
.output/
.wxt/
.web-extension-id
.pnpm-store/
# Git worktrees
.worktrees/
# Logs
*.log
# Local browser evaluation artifacts
evals/browser/results/
evals/browser/agents.local.json
# macOS
.DS_Store
+116
View File
@@ -0,0 +1,116 @@
# Changelog
All notable changes to BrowserSkill will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/).
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [Unreleased]
### Added
- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin,
with ancestor-clipped visible bounds, iframe support and cooperative cancellation
## [0.2.1] - 2026-09-09
### Changed
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
### Fixed
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
- CLI: Windows named-pipe connection continuity and stale daemon process detection
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
- Extension: match observed form state to the correct element after page changes
- Extension: UUID generation fallback for recording and overlays on insecure origins
- Browser locale matching for regional language variants
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
- DSH Plugin: removed the obsolete client runtime host dependency
## [0.2.0] - 2026-09-02
### Added
- File transfer: upload and download support across CLI, Extension, and DSH Plugin
- File transfer: drag-and-drop upload (`drop-to-upload`)
- VOM semantic graph, name enrichment, and hover perception modules
- VOM hover probing (opt-in via `observe` parameter)
- DSH Plugin: browser tool parity with CLI commands
- Edge Add-ons automated publishing in CI
- Protocol upgrade reminder when CLI / Extension protocol versions differ
- Browser evaluation harness (`evals/browser/`)
- Unified release script (`scripts/release.mjs`)
### Changed
- **Version scheme**: all three components now share the same semver
- VOM rendering algorithm optimizations
- Leaner SKILL.md agent instructions
- DSH Plugin: simplified browser commands
- Borrow confirmation UX — proactive focus and longer timeout
- PiP window now has a close button
### Fixed
- Screenshot media type detection (was hard-coded to `image/png`)
- DSH Plugin session lifecycle stability
- DSH Plugin Cordis package ID mismatch
- Observation thumbnail media type sniffing
- Upload/download race conditions and layout bypass issues
- VOM repeated name and safety policy issues
---
*Previous releases used independent version numbers per component.*
## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29
### Added
- DSH Plugin sidebar integration, session lifecycle, and archive cleanup
- Recorder iframe and OOPIF support
### Changed
- VOM functional refactor
### Fixed
- Recorder safety policy and bug fixes
## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08
### Added
- VOM observation recording and settled-state detection
- Record overlay timer
### Fixed
- Browser keepalive disconnect handling
## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29
### Added
- CLI auto-update mechanism
- Trace v3 protocol and recorder
### Fixed
- MV3 keepalive disconnect
## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22
### Added
- More browser interaction actions
### Fixed
- Windows named-pipe hash-only path issue
## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07
Initial public release pair.
## CLI 0.1.6 — 2026-06-30
### Fixed
- Minor CLI fixes
## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22
First tagged releases.
Generated
+3 -2
View File
@@ -134,9 +134,10 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.1.10"
version = "0.2.1"
dependencies = [
"anyhow",
"base64",
"bsk-protocol",
"clap",
"console",
@@ -168,7 +169,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.1.10"
version = "0.2.1"
dependencies = [
"schemars",
"semver",
+3 -2
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.1.10"
version = "0.2.1"
edition = "2024"
rust-version = "1.85"
license = "MIT"
@@ -36,13 +36,14 @@ flate2 = "1.1.9"
tar = "0.4.46"
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
sha2 = "0.11.0"
base64 = "0.22"
tempfile = "3"
nix = { version = "0.29", features = ["signal", "process"] }
rand = "0.8"
uuid = { version = "1", features = ["v4", "serde"] }
# Workspace crates (version required for `cargo publish` of dependents)
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
[profile.release]
strip = "symbols"
+17 -7
View File
@@ -147,20 +147,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
BrowserSkill ships a first-class dsh plugin on npm as
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
overlay of each Agent Window.
It gives the agent native `browser_*` tools and a live view of its browser sessions
in the Web UI. The plugin runs `bsk` on the agent's behalf.
Add it to a dsh profile, then start that profile:
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
to a dsh profile and start it (replace `web` with your profile name):
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
configuration, and the observation overlay.
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
Restart the profile after upgrading. See the
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
## How It Works
@@ -201,6 +207,9 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected
## For Developers
The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol
and plugin entry points, visible bounds and interruption behavior.
The repository is a Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI and local daemon
@@ -208,6 +217,7 @@ The repository is a Cargo + pnpm workspace:
- `apps/extension` — browser extension
- `packages/ui` and `packages/i18n` — shared extension UI support
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
## License
+12 -3
View File
@@ -125,16 +125,22 @@ bsk install-skill
## DeepSeek Harness 插件
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具,由插件代为调用 `bsk`,并在 Web UI 中实时展示浏览器会话。
把它装进某个 dsh profile,然后启动该 profile:
先安装 `bsk` CLI 并连接浏览器扩展,再将插件装进 dsh profile 并启动(将 `web` 替换为你的 profile 名称):
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
插件自带 `browser-skill` skill,所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新;升级此插件请运行:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
## 工作原理
@@ -171,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
## 面向开发者
[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口,以及可见区域、错误和中断语义。
本仓库是 Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI 与本地 daemon
@@ -178,6 +186,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
- `apps/extension` — 浏览器扩展
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
## 许可证
+6 -3
View File
@@ -1,6 +1,6 @@
# BrowserSkill — Privacy Policy
**Last updated:** May 25, 2026
**Last updated:** August 20, 2026
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
## 2. Single Purpose
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
## 3. Data the Extension Accesses
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
## 4. Data the Extension Does **Not** Collect
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
- Send any data to remote servers, the Extension's authors, or any third party.
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
- Use webcam, microphone, geolocation, or any device sensor.
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
- Track users across websites or across sessions.
@@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
@@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
## 8. User Control
@@ -0,0 +1,81 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>BrowserSkill file-transfer fixture</title>
<style>
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
</style>
</head>
<body>
<h1>Upload then download the same image</h1>
<p id="status">No image selected</p>
<div class="actions">
<button id="upload" type="button">Upload through file input</button>
<input id="file" type="file" accept="image/png" hidden />
<a id="download" aria-disabled="true">Download uploaded image</a>
</div>
<div id="drop" role="button" tabindex="0">Drop an image here</div>
<img id="preview" alt="Uploaded image preview" hidden />
<script>
const input = document.querySelector('#file');
const upload = document.querySelector('#upload');
const drop = document.querySelector('#drop');
const download = document.querySelector('#download');
const preview = document.querySelector('#preview');
const status = document.querySelector('#status');
let objectUrl;
window.__bskTransferFixture = { events: [], result: null };
function accept(file, mechanism) {
if (!file) return;
if (objectUrl) URL.revokeObjectURL(objectUrl);
objectUrl = URL.createObjectURL(file);
preview.src = objectUrl;
preview.hidden = false;
download.href = objectUrl;
download.download = file.name;
download.removeAttribute('aria-disabled');
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
window.__bskTransferFixture.result = {
mechanism,
name: file.name,
size: file.size,
type: file.type,
hasPathProperty: 'path' in file,
};
}
upload.addEventListener('click', () => input.click());
input.addEventListener('change', () => accept(input.files[0], 'input'));
for (const type of ['dragenter', 'dragover']) {
drop.addEventListener(type, event => {
const types = [...event.dataTransfer.types];
window.__bskTransferFixture.events.push({ type, types });
if (!types.includes('Files')) return;
event.preventDefault();
drop.classList.add('dragging');
});
}
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
drop.addEventListener('drop', event => {
window.__bskTransferFixture.events.push({
type: 'drop',
types: [...event.dataTransfer.types],
fileCount: event.dataTransfer.files.length,
});
event.preventDefault();
drop.classList.remove('dragging');
accept(event.dataTransfer.files[0], 'drop');
});
</script>
</body>
</html>
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
"version": "0.1.6",
"version": "0.2.1",
"private": true,
"type": "module",
"scripts": {
@@ -29,6 +29,24 @@ function fakeApi() {
}
describe("ChromiumCdp", () => {
it("changes attachment identity only across real debugger attachments", async () => {
const { api, onEvent, onDetach } = fakeApi();
const cdp = new ChromiumCdp(api);
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
const first = cdp.getAttachmentId(4);
expect(first).toBeTruthy();
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
expect(cdp.getAttachmentId(4)).toBe(first);
onDetach.fire({ tabId: 4 }, "target_closed");
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
expect(cdp.getAttachmentId(4)).not.toBe(first);
await cdp.detach(4);
expect(cdp.getAttachmentId(4)).toBeUndefined();
});
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -254,6 +272,69 @@ describe("ChromiumCdp", () => {
expect(cdp.isAttached(7)).toBe(false);
});
it.each([
false,
true,
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
const { api } = fakeApi();
const attached = new Set<number>();
let finishDetach!: () => void;
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
attached.add(tabId!);
});
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
attached.delete(tabId!);
});
vi.mocked(api.detach).mockImplementationOnce(
({ tabId }) =>
new Promise<void>((resolve, reject) => {
finishDetach = () => {
attached.delete(tabId!);
if (rejectDetach) reject(new Error("tab already detached"));
else resolve();
};
}),
);
const cdp = new ChromiumCdp(api);
cdp.trackSessionTab("aa11", 7);
await cdp.ensureAttached(7);
const returning = cdp.releaseSessionTab("aa11", 7);
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
let duplicateDetachFinished = false;
const duplicateDetach = cdp.detach(7).then(() => {
duplicateDetachFinished = true;
});
cdp.trackSessionTab("bb22", 7);
// Register rejection handlers immediately; the pre-fix driver rejects
// both commands while the browser still has the previous attachment.
const pending = Promise.allSettled([
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
cdp.send(7, "DOM.getDocument"),
]);
await cdp.send(8, "DOM.getDocument");
const finishedEarly = duplicateDetachFinished;
finishDetach();
await Promise.all([returning, duplicateDetach]);
expect(await pending).toEqual([
{ status: "fulfilled", value: { ok: true } },
{ status: "fulfilled", value: { ok: true } },
]);
expect(finishedEarly).toBe(false);
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
2,
);
expect(cdp.isAttached(7)).toBe(true);
expect(cdp.isAttached(8)).toBe(true);
await cdp.detachSession("bb22");
expect(api.detach).toHaveBeenCalledTimes(2);
expect(cdp.isAttached(7)).toBe(false);
expect(attached.has(7)).toBe(false);
expect(cdp.isAttached(8)).toBe(true);
});
it("detachAll() iterates every cached tab", async () => {
const { api } = fakeApi();
const cdp = new ChromiumCdp(api);
@@ -651,6 +732,155 @@ describe("ChromiumCdp", () => {
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
});
it("releases only the returning session's claim on a tab", async () => {
const { api } = fakeApi();
const cdp = new ChromiumCdp(api);
await cdp.ensureAttached(1);
await cdp.ensureAttached(2);
cdp.trackSessionTab("aa11", 1);
cdp.trackSessionTab("bb22", 1);
cdp.trackSessionTab("aa11", 2);
await cdp.releaseSessionTab("unknown", 1);
await cdp.releaseSessionTab("aa11", 1);
expect(api.detach).not.toHaveBeenCalled();
await cdp.releaseSessionTab("bb22", 1);
await cdp.releaseSessionTab("bb22", 1);
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
expect(cdp.isAttached(2)).toBe(true);
await cdp.detachSession("aa11");
expect(api.detach).toHaveBeenCalledTimes(2);
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
});
it.each([
false,
true,
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
const { api } = fakeApi();
let finishAttach!: () => void;
vi.mocked(api.attach).mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
finishAttach = resolve;
}),
);
const cdp = new ChromiumCdp(api);
cdp.trackSessionTab("aa11", 1);
const attaching = cdp.ensureAttached(1);
const releasing = cdp.releaseSessionTab("aa11", 1);
if (newOwner) cdp.trackSessionTab("bb22", 1);
finishAttach();
await Promise.all([attaching, releasing]);
expect(cdp.isAttached(1)).toBe(newOwner);
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
});
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
const { api, onEvent } = fakeApi();
let controlled = true;
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
cdp.trackSessionTab("aa11", 1);
cdp.trackSessionTab("bb22", 1);
await cdp.ensureAttached(1);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
type: "confirm",
message: "before return",
});
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
vi.mocked(api.sendCommand).mockClear();
controlled = false;
await cdp.releaseSessionTab("aa11", 1);
expect(cdp.isAttached(1)).toBe(true);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
type: "confirm",
message: "after return",
});
await Promise.resolve();
expect(api.sendCommand).not.toHaveBeenCalled();
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
controlled = true;
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
type: "alert",
message: "borrowed again",
});
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
{ tabId: 1 },
"Page.handleJavaScriptDialog",
{ accept: true },
);
});
it("ignores dialog events after detach, including a pending eligibility check", async () => {
const { api, onEvent } = fakeApi();
let resolveEligibility!: (value: boolean) => void;
const shouldAutoAcceptDialog = vi.fn(
() =>
new Promise<boolean>((resolve) => {
resolveEligibility = resolve;
}),
);
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
await cdp.ensureAttached(1);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
await cdp.detach(1);
resolveEligibility(true);
await Promise.resolve();
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
expect(api.sendCommand).not.toHaveBeenCalledWith(
expect.anything(),
"Page.handleJavaScriptDialog",
expect.anything(),
);
expect(cdp.dialogsSince(1, 0)).toEqual([]);
});
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
const { api, onEvent } = fakeApi();
const shouldAutoAcceptDialog = vi.fn(async () => {
throw new Error("tab is gone");
});
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
await cdp.ensureAttached(1);
vi.mocked(api.sendCommand).mockClear();
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
await Promise.resolve();
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
expect(api.sendCommand).not.toHaveBeenCalled();
expect(cdp.dialogsSince(1, 0)).toEqual([]);
});
it("does not restore dialog state when an acceptance completes after detach", async () => {
const { api, onEvent } = fakeApi();
const cdp = new ChromiumCdp(api);
await cdp.ensureAttached(1);
let finishAccept!: () => void;
vi.mocked(api.sendCommand).mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
finishAccept = resolve;
}),
);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
await cdp.detach(1);
finishAccept();
await Promise.resolve();
expect(cdp.dialogsSince(1, 0)).toEqual([]);
expect(cdp.dialogCursor(1)).toBe(0);
});
it("detachSession only detaches tabs no other session owns", async () => {
const { api } = fakeApi();
const cdp = new ChromiumCdp(api);
@@ -164,7 +164,9 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
export class ChromiumCdp {
private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>();
private readonly attachmentIds = new Map<number, string>();
private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>();
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
private readonly dialogSequences = new Map<number, number>();
@@ -182,7 +184,13 @@ export class ChromiumCdp {
private networkSubscription: { dispose(): void } | null = null;
private frameTargetSubscription: { dispose(): void } | null = null;
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
constructor(
api: CdpDebuggerApi = chromeDebuggerApi,
private readonly options: {
/** CDP observation alone does not authorize dismissing user dialogs. */
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
} = {},
) {
this.api = api;
this.bindAutoDetach();
this.bindDialogHandler();
@@ -191,8 +199,17 @@ export class ChromiumCdp {
this.bindFrameTargetHandler();
}
/** Identity of this actual debugger attachment, independent of frame topology. */
getAttachmentId(tabId: number): string | undefined {
return this.attachmentIds.get(tabId);
}
/** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching.
// New observers must wait before opening the next connection to that tab.
const detaching = this.detachInFlight.get(tabId);
if (detaching) await detaching;
if (this.attachedTabs.has(tabId)) return;
const existing = this.attachInFlight.get(tabId);
if (existing) {
@@ -206,6 +223,7 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId);
this.attachmentIds.set(tabId, crypto.randomUUID());
await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
});
@@ -425,20 +443,32 @@ export class ChromiumCdp {
/** Detach if attached; never throws. */
async detach(tabId: number): Promise<void> {
const existing = this.detachInFlight.get(tabId);
if (existing) {
await existing;
return;
}
this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId);
this.attachmentIds.delete(tabId);
this.clearDialogState(tabId);
this.clearConsoleState(tabId);
this.clearNetworkState(tabId);
this.clearFrameState(tabId);
try {
await this.api.detach({ tabId });
} catch (err) {
// Tab may already be gone — Chrome auto-detaches on close. Log
// at debug so production builds aren't noisy.
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
}
const detach = (async () => {
try {
await this.api.detach({ tabId });
} catch (err) {
// Tab may already be gone — Chrome auto-detaches on close. Log
// at debug so production builds aren't noisy.
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
}
})().finally(() => {
this.detachInFlight.delete(tabId);
});
this.detachInFlight.set(tabId, detach);
await detach;
}
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
@@ -453,6 +483,17 @@ export class ChromiumCdp {
this.tabOwners.set(tabId, owners);
}
/** Release one session's claim, preserving attachments still used by another. */
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
const owners = this.tabOwners.get(tabId);
if (!owners?.delete(sessionId) || owners.size > 0) return;
this.tabOwners.delete(tabId);
// An observation may still be attaching when the tab is returned. Wait
// for it so detach cannot miss the attachment or remove a new owner's claim.
await this.attachInFlight.get(tabId)?.catch(() => {});
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
}
/** Subscribe to all CDP events. Returned disposable removes the listener. */
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
dispose(): void;
@@ -469,6 +510,7 @@ export class ChromiumCdp {
this.attachInFlight.clear();
this.tabOwners.clear();
this.attachedTabs.clear();
this.attachmentIds.clear();
this.dialogBuffers.clear();
this.dialogSequences.clear();
this.consoleBuffers.clear();
@@ -639,13 +681,23 @@ export class ChromiumCdp {
}
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
const parsed = parseDialogOpeningParams(params);
try {
if (
this.options.shouldAutoAcceptDialog &&
!(await this.options.shouldAutoAcceptDialog(tabId))
) {
return;
}
// The tab may have been returned while its current scope was checked.
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
if (parsed.type === "prompt") {
handleParams.promptText = parsed.defaultPrompt ?? "";
}
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
this.dialogSequences.set(tabId, sequence);
this.appendDialog(tabId, {
@@ -805,6 +857,7 @@ export class ChromiumCdp {
const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
if (typeof source.tabId === "number") {
this.attachedTabs.delete(source.tabId);
this.attachmentIds.delete(source.tabId);
this.attachInFlight.delete(source.tabId);
this.tabOwners.delete(source.tabId);
this.clearDialogState(source.tabId);
@@ -835,15 +888,9 @@ export class ChromiumCdp {
/** Detach tabs only when no other live session has claimed them. */
async detachSession(sessionId: string): Promise<void> {
const tabsToDetach: number[] = [];
for (const [tabId, owners] of this.tabOwners) {
owners.delete(sessionId);
if (owners.size === 0) {
this.tabOwners.delete(tabId);
tabsToDetach.push(tabId);
}
}
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
await Promise.all(
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
);
}
}
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
id: "req-1",
isActiveTab: true,
tabTitle: "Example",
timeoutMs: 5000,
timeoutMs: 60_000,
onAllow,
onDeny,
},
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
);
expect(screen.getByText("允许借用标签页?")).toBeTruthy();
for (let i = 0; i < 5; i++) {
for (let i = 0; i < 60; i++) {
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
id: "req-fallback",
isActiveTab: true,
tabTitle: "Example",
timeoutMs: 5000,
timeoutMs: 60_000,
onAllow,
onDeny,
},
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
/>,
);
for (let i = 0; i < 5; i++) {
for (let i = 0; i < 60; i++) {
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
@@ -1,10 +1,12 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", {
runtime: {
sendMessage: vi.fn(() => Promise.resolve()),
sendMessage: vi.fn((message: { sequence?: number }) =>
Promise.resolve({ ok: true, sequence: message.sequence }),
),
},
});
@@ -46,58 +48,6 @@ function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
}
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
});
describe("record-capture semantic", () => {
let steps: RecordStepPayload[];
@@ -174,6 +124,19 @@ describe("record-capture semantic", () => {
]);
});
it("does not emit page navigation from a child Document capture", () => {
const originalUrl = location.href;
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
captureNavigation: false,
});
history.pushState({}, "", "#inside-frame");
expect(steps).toEqual([]);
capture.dispose();
history.replaceState({}, "", originalUrl);
});
it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = `
<div id="chrome">page chrome</div>
@@ -228,6 +191,7 @@ describe("record-capture semantic", () => {
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
});
expect(steps[1]).toMatchObject({
op: "click",
@@ -0,0 +1,124 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
import { RecordFrameAgent } from "../recording/frame-agent";
class PortListeners<T extends (...args: never[]) => unknown> {
readonly values = new Set<T>();
addListener = (listener: T) => this.values.add(listener);
removeListener = (listener: T) => this.values.delete(listener);
}
function portHarness() {
const onMessage = new PortListeners<(message: unknown) => void>();
const onDisconnect = new PortListeners<() => void>();
const outbound: RecordFramePortMessage[] = [];
const port = {
name: "bsk-record-frame",
onMessage,
onDisconnect,
postMessage(message: RecordFramePortMessage) {
outbound.push(message);
if (message.type === "ready") {
queueMicrotask(() => {
for (const listener of onMessage.values) {
listener({
type: "ready_ack",
requestId: message.requestId,
producerId: message.producerId,
});
}
});
}
},
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port;
return {
port,
outbound,
receive(message: RecordFramePortMessage) {
for (const listener of onMessage.values) listener(message);
},
};
}
describe("RecordFrameAgent", () => {
afterEach(() => {
vi.unstubAllGlobals();
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("starts when crypto.randomUUID is unavailable", async () => {
const harness = portHarness();
const getRandomValues = vi.fn((bytes: Uint8Array) => {
bytes.fill(0x11);
return bytes;
});
vi.stubGlobal("crypto", { getRandomValues });
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage: vi.fn(),
},
});
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(getRandomValues).toHaveBeenCalled();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
.fn<(message: { sequence: number }) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage,
},
});
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-1",
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-2",
ok: true,
}),
);
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
});
});
@@ -0,0 +1,36 @@
import { describe, expect, it, vi } from "vitest";
import type { RecordStepMessage } from "@/lib/record-bridge";
import { RecordStepDelivery } from "../record-step-delivery";
describe("RecordStepDelivery", () => {
it("retries from the first unacknowledged sequence before sending later steps", async () => {
const sent: RecordStepMessage[] = [];
const send = vi.fn(async (message: RecordStepMessage) => {
sent.push(message);
if (sent.length === 1) throw new Error("service worker unavailable");
return { ok: true, sequence: message.sequence };
});
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
await expect(delivery.flush()).resolves.toBe(true);
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
});
it("keeps unacknowledged steps pending for a later flush", async () => {
const send = vi
.fn<(message: RecordStepMessage) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
await expect(delivery.flush()).resolves.toBe(false);
await expect(delivery.flush()).resolves.toBe(true);
expect(send).toHaveBeenCalledTimes(3);
});
});
+76 -150
View File
@@ -1,9 +1,9 @@
import {
type CaptureTargetDescriptor,
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
@@ -12,22 +12,8 @@ import {
hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/recording/draft-policy";
import {
closestHoverSurfaceCandidate,
collectHoverSurfaceStates,
@@ -37,42 +23,21 @@ import {
isLikelyHoverSurfaceOwner,
} from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const knownRecordRequests = new Set<string>();
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
const payload = { type: RECORD_STEP, requestId, step };
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
() => true,
() => {
failedStepDeliveries.add(requestId);
return false;
},
);
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
sends.add(pending);
pendingStepSends.set(requestId, sends);
void pending.then(() => {
sends.delete(pending);
if (sends.size === 0) pendingStepSends.delete(requestId);
});
}
export interface RecordCaptureController {
dispose(): void;
}
interface FillSession {
element: FillableElement;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
baselineValue: string;
lastValue: string;
pendingCommit?: "enter" | "suggestion" | "blur";
}
interface HoverCandidate {
element: Element;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
recordedAt: number;
score: number;
eligible: boolean;
@@ -157,6 +122,24 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
return null;
}
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
const rect = el.getBoundingClientRect();
return {
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
tag: el.tagName.toLowerCase(),
};
}
function geometryForEventTarget(
target: EventTarget | null,
): RecordStepPayload["geometry"] | undefined {
if (!(target instanceof Element)) return undefined;
const clickable = target.closest(
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
);
return captureGeometry(clickable ?? target);
}
function fillableValue(el: FillableElement): string {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return el.value;
@@ -232,7 +215,10 @@ function collectHoverTriggerLabelText(root: Element): string {
return normalizeLabelText(text);
}
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
function compactHoverTargetName(
el: Element,
desc: CaptureTargetDescriptor,
): CaptureTargetDescriptor {
if (!desc.name) return desc;
const fullText = normalizeLabelText(el.textContent ?? "");
const compactName = collectHoverTriggerLabelText(el);
@@ -250,7 +236,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
return desc;
}
function isWeakHoverTarget(target: TargetDescriptor): boolean {
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
@@ -261,9 +247,9 @@ function looksLikeAvatarElement(el: Element): boolean {
function normalizeHoverTarget(
el: Element,
desc: TargetDescriptor,
desc: CaptureTargetDescriptor,
decision: HoverTriggerDecision,
): TargetDescriptor {
): CaptureTargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
@@ -277,7 +263,7 @@ function normalizeHoverTarget(
return desc;
}
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return {
@@ -373,6 +359,7 @@ function scheduleInputCompletionCommit(
export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
options: { captureNavigation?: boolean } = {},
): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
@@ -411,14 +398,17 @@ export function startRecordCapture(
emitStep({
op: "fill",
target: session.target,
geometry: captureGeometry(session.element),
value,
commit: session.pendingCommit ?? "blur",
...(isPassword ? { redacted: true } : {}),
});
};
const commitFillSession = () => {
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
if (!fillSession || composing) return;
const session = fillSession;
session.pendingCommit = commit;
fillSession = null;
emitFill(session);
committedValues.set(session.element, session.lastValue);
@@ -624,6 +614,7 @@ export function startRecordCapture(
emitStep({
op: "hover",
target: hover.target,
geometry: captureGeometry(hover.element),
});
emittedHoverElements.add(hover.element);
};
@@ -678,6 +669,7 @@ export function startRecordCapture(
emitStep({
op: "click",
target,
geometry: geometryForEventTarget(eventTarget(event)),
expects_navigation: true,
});
};
@@ -779,7 +771,7 @@ export function startRecordCapture(
scheduleInputCompletionCommit(
sessionElement,
syncFillSessionValue,
commitFillSession,
() => commitFillSession("suggestion"),
(el) => fillSession?.element === el,
);
return;
@@ -839,6 +831,7 @@ export function startRecordCapture(
emitStep({
op: "select",
target: desc,
geometry: captureGeometry(target),
values,
labels,
expects_navigation: true,
@@ -879,7 +872,7 @@ export function startRecordCapture(
};
}
if (fillable) {
commitFillSession();
commitFillSession(event.key === "Enter" ? "enter" : "blur");
}
const desc = describeEventTarget(target);
if (!desc && !event.key) return;
@@ -887,7 +880,7 @@ export function startRecordCapture(
emitStep({
op: "press",
key: event.key,
...(desc ? { target: desc } : {}),
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
...(modifiers.length ? { modifiers } : {}),
expects_navigation: event.key === "Enter",
});
@@ -903,23 +896,33 @@ export function startRecordCapture(
document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true);
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
urlObserver.observe(document, { subtree: true, childList: true });
const captureNavigation = options.captureNavigation ?? true;
const urlObserver = captureNavigation
? new MutationObserver(() => emitNavigateIfChanged())
: null;
urlObserver?.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged();
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
window.addEventListener("pagehide", commitFillSession);
// Wrap: passing commitFillSession directly would forward the DOM event as
// the `commit` argument and stamp it onto the recorded fill step.
const onPageHide = () => commitFillSession();
if (captureNavigation) {
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
}
window.addEventListener("pagehide", onPageHide);
const originalPushState = history.pushState;
const originalReplaceState = history.replaceState;
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
if (captureNavigation) {
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
}
return {
dispose() {
@@ -933,93 +936,16 @@ export function startRecordCapture(
document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true);
urlObserver.disconnect();
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
window.removeEventListener("pagehide", commitFillSession);
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
urlObserver?.disconnect();
if (captureNavigation) {
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
}
window.removeEventListener("pagehide", onPageHide);
if (captureNavigation) {
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
}
},
};
}
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
if (!knownRecordRequests.has(message.requestId)) {
knownRecordRequests.add(message.requestId);
failedStepDeliveries.delete(message.requestId);
}
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
sendRecordStep(message.requestId, step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
void Promise.all(pending).then((delivered) => {
finishStop();
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
if (succeeded) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
sendResponse(
succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
},
);
});
return true;
}
if (isRecordCancelMessage(message)) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -0,0 +1,66 @@
import { createRandomUuid } from "@/lib/random-uuid";
import {
isAcceptedRecordStepAck,
RECORD_STEP,
type RecordStepMessage,
type RecordStepPayload,
} from "@/lib/record-bridge";
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
export class RecordStepDelivery {
readonly #requestId: string;
readonly #producerId: string;
readonly #send: SendRecordStepMessage;
readonly #pending = new Map<number, RecordStepPayload>();
#nextSequence = 1;
#tail = Promise.resolve();
constructor(
requestId: string,
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
producerId: string = createRandomUuid(),
) {
this.#requestId = requestId;
this.#producerId = producerId;
this.#send = send;
}
enqueue(step: RecordStepPayload): void {
const sequence = this.#nextSequence;
this.#nextSequence += 1;
this.#pending.set(sequence, step);
void this.#schedule();
}
async flush(): Promise<boolean> {
await this.#tail;
await this.#schedule();
return this.#pending.size === 0;
}
#schedule(): Promise<void> {
const drain = () => this.#drain();
this.#tail = this.#tail.then(drain, drain);
return this.#tail;
}
async #drain(): Promise<void> {
for (const [sequence, step] of this.#pending) {
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: this.#requestId,
producerId: this.#producerId,
sequence,
step,
};
try {
const response = await this.#send(message);
if (!isAcceptedRecordStepAck(response, sequence)) return;
this.#pending.delete(sequence);
} catch {
return;
}
}
}
}
@@ -0,0 +1,210 @@
import { createRandomUuid } from "@/lib/random-uuid";
import {
markRecordingDocument,
type RecordingDocumentMarker,
waitForDocumentElement,
} from "@/lib/recording/document-marker";
import {
isRecordFrameStartMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_QUERY,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "@/lib/recording/frame-bridge";
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
import { RecordStepDelivery } from "../record-step-delivery";
interface ActiveFrameRecording {
requestId: string;
producerId: string;
port: chrome.runtime.Port;
delivery: RecordStepDelivery;
capture: RecordCaptureController;
marker: RecordingDocumentMarker;
}
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
export class RecordFrameAgent {
#active: ActiveFrameRecording | null = null;
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
#disposed = false;
async start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
if (this.#active?.requestId === message.requestId) return { ok: true };
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
if (this.#startPromise) {
await this.#startPromise;
return this.start(message);
}
if (this.#active) this.cancel(this.#active.requestId);
const starting = this.#start(message);
this.#startPromise = starting;
try {
return await starting;
} finally {
if (this.#startPromise === starting) this.#startPromise = null;
}
}
cancel(requestId: string): void {
const active = this.#active;
if (!active || active.requestId !== requestId) return;
active.capture.dispose();
active.marker.restore();
active.port.disconnect();
this.#active = null;
}
dispose(): void {
this.#disposed = true;
if (this.#active) this.cancel(this.#active.requestId);
}
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = createRandomUuid();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
try {
await registerPort(port, message.requestId, producerId);
} catch {
marker.restore();
port.disconnect();
return { ok: false, error: "failed to register recording document" };
}
if (this.#disposed) {
marker.restore();
port.disconnect();
return { ok: false, error: "recording document is disposed" };
}
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
const capture = startRecordCapture(
message.requestId,
(step) => {
marker.ensure();
delivery.enqueue(step);
},
{ captureNavigation: window.top === window },
);
const active: ActiveFrameRecording = {
requestId: message.requestId,
producerId,
port,
delivery,
capture,
marker,
};
this.#active = active;
port.onMessage.addListener((raw: unknown) => {
const command = raw as Partial<RecordFramePortMessage>;
if (command.type === "stop" && command.requestId === active.requestId) {
void this.#stop(active, command.commandId);
} else if (command.type === "cancel" && command.requestId === active.requestId) {
this.cancel(active.requestId);
}
});
port.onDisconnect.addListener(() => {
if (this.#active !== active) return;
active.capture.dispose();
active.marker.restore();
this.#active = null;
});
return { ok: true };
}
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
if (!commandId || this.#active !== active) return;
active.capture.dispose();
const succeeded = await active.delivery.flush();
try {
active.port.postMessage({
type: "stopped",
requestId: active.requestId,
commandId,
ok: succeeded,
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
} satisfies RecordFramePortMessage);
} catch {
return;
}
if (!succeeded || this.#active !== active) return;
active.marker.restore();
this.#active = null;
active.port.disconnect();
}
}
function registerPort(
port: chrome.runtime.Port,
requestId: string,
producerId: string,
): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
reject(new Error("recording document registration timed out"));
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
const cleanup = () => {
clearTimeout(timer);
port.onMessage.removeListener(onMessage);
port.onDisconnect.removeListener(onDisconnect);
};
const onMessage = (raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (
message.type !== "ready_ack" ||
message.requestId !== requestId ||
message.producerId !== producerId
) {
return;
}
cleanup();
resolve();
};
const onDisconnect = () => {
cleanup();
reject(new Error("recording document port disconnected"));
};
port.onMessage.addListener(onMessage);
port.onDisconnect.addListener(onDisconnect);
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
});
}
export function attachRecordFrameAgent(): () => void {
const agent = new RecordFrameAgent();
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
) => {
if (!isRecordFrameStartMessage(message)) return false;
void agent.start(message).then(sendResponse);
return true;
};
chrome.runtime.onMessage.addListener(onMessage);
void chrome.runtime
.sendMessage({ type: RECORD_FRAME_QUERY })
.then((response: RecordFrameQueryResponse | undefined) => {
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
return agent.start({
type: RECORD_FRAME_START,
requestId: response.requestId,
startedAtMs: response.startedAtMs,
});
})
.catch(() => {});
return () => {
chrome.runtime.onMessage.removeListener(onMessage);
agent.dispose();
};
}
+87 -29
View File
@@ -21,10 +21,11 @@ import {
type OverlayMode,
} from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
import { SessionManager } from "@/session-manager/manager";
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
import {
attachBorrowNotificationButtonHandler,
attachBorrowNotificationClickHandler,
@@ -38,7 +39,10 @@ import {
attachRecordFinishListener,
attachRecordQueryListener,
attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record";
import { chromeTabsApi } from "@/tools/shared";
import { chromeTabMutationApi } from "@/tools/tabs";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
import { WSTransport } from "@/transport/ws-transport";
@@ -47,7 +51,12 @@ export default defineBackground(() => {
const controller = new ConnectionController();
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
const sessions = new SessionManager();
const cdp = new ChromiumCdp();
const cdp = new ChromiumCdp(undefined, {
shouldAutoAcceptDialog: async (tabId) => {
const tab = await chrome.tabs.get(tabId);
return sessions.findByWindowId(tab.windowId) !== null;
},
});
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
let overlayGeneration = 0;
const controlModes = new Map<string, OverlayMode>();
@@ -78,6 +87,24 @@ export default defineBackground(() => {
};
}
/**
* Authoritative overlay state for a specific tab. Agent Window tabs are
* free by default; only tabs explicitly claimed through session startup,
* `tab_create`, or `tab_borrow` receive the control overlay.
*/
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
if (typeof tabId === "number" && typeof windowId === "number") {
const ctx = sessions.findByWindowId(windowId);
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
}
return {
type: OVERLAY_AGENT_STATE,
sessionId: null,
mode: "hidden",
generation: overlayGeneration,
};
}
async function pushOverlayStateToTab(
tabId: number,
state: OverlayAgentStateMessage,
@@ -89,13 +116,18 @@ export default defineBackground(() => {
}
}
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
const state = overlayStateForTab(tabId, windowId);
await pushOverlayStateToTab(tabId, state);
}
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
const state = overlayStateForWindow(windowId);
const tabs = await chrome.tabs.query({ windowId });
await Promise.all(
tabs.map((tab) =>
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
),
tabs.map((tab) => {
if (typeof tab.id !== "number") return Promise.resolve();
return pushOverlayStateForTab(tab.id, windowId);
}),
);
}
@@ -134,23 +166,66 @@ export default defineBackground(() => {
if (typeof tab.windowId !== "number") return;
pushOverlayStateForAgentWindow(tab.windowId);
});
// Newly observed tabs are free unless the agent's own creation path has
// already claimed their concrete id. This listener only renders that
// state; it never infers or mutates ownership from event ordering.
chrome.tabs.onCreated.addListener((tab) => {
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
if (!sessions.findByWindowId(tab.windowId)) return;
void pushOverlayStateForTab(tab.id, tab.windowId);
});
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
});
// Re-sync the storage.session flag on SW startup so a previous SW's
// stale `true` does not keep waking us on every page load until the
// first mutation (review M4/M5 round 3 m-R3-1).
void sessionsLive.refresh();
const cleanupAfterDisconnect = createDisconnectCleanup({
manager: sessions,
sessionStopDeps: { cdp },
// Same contract as the dispatcher's `tool.session_stop`: without these
// deps the agent-tab cleanup and the window-release path are dead code.
sessionStopDeps: {
cdp,
tabManagement: { tabs: chromeTabMutationApi },
tabsQuery: chromeTabsApi,
},
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
});
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
frameCoordinator: recordFrameCoordinator,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
} satisfies RecordRuntimeDeps;
recordFrameCoordinator.attach();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp,
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
onAgentTabClaimed: (tabId, windowId) => {
void pushOverlayStateForTab(tabId, windowId);
},
approveBorrow: (ctx) =>
requestBorrowConfirmation(ctx.tabId, {
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
@@ -170,27 +245,6 @@ export default defineBackground(() => {
}),
});
dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener();
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked,
@@ -305,7 +359,11 @@ export default defineBackground(() => {
}
if (msg.kind === OVERLAY_MSG_READY) {
sendResponse(overlayStateForWindow(sender.tab?.windowId));
// Resolve from explicit tab ownership before sending so free tabs never
// flash the control mask.
if (typeof sender.tab?.id === "number") {
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
}
return false;
}
+34 -45
View File
@@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
type CaptureSuppressAck,
type CaptureSuppressMessage,
@@ -45,11 +40,17 @@ import {
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_FINISH,
RECORD_QUERY,
type RecordCancelMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type {
BorrowCancelMessage,
@@ -69,7 +70,6 @@ export default defineContentScript({
if (window.top !== window) return;
const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
@@ -228,8 +228,6 @@ export default defineContentScript({
if (previousHelp) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderAll();
}
@@ -259,6 +257,9 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| CaptureSuppressMessage
| RecordStartMessage
| RecordStopMessage
| RecordCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
@@ -269,41 +270,32 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse);
}
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
{
activeRequestId: activeRecordRequestId,
capture: recordCapture,
setActiveRequestId: (id) => {
activeRecordRequestId = id;
},
setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId, startedAtMs) => {
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderAll();
},
if (isRecordStartMessage(message)) {
activeRecordRequestId = message.requestId;
overlays.setAgentRecordRequest({
id: message.requestId,
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId: message.requestId,
});
},
sendResponse as unknown as
| ((response: RecordStartAck | RecordStopAck) => void)
| undefined,
);
return needsAsync;
});
renderAll();
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
return false;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
if (activeRecordRequestId !== message.requestId) return false;
overlays.clearAgentRecordRequest(message.requestId);
activeRecordRequestId = null;
renderAll();
if (isRecordStopMessage(message)) {
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
}
return false;
}
if (
@@ -526,9 +518,6 @@ export default defineContentScript({
chrome.runtime.onMessage.removeListener(onMessage);
chrome.storage.onChanged.removeListener(onStorageChange);
window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
});
},
@@ -122,8 +122,8 @@ describe("App", () => {
expect(screen.queryByText(/^扩展 v/)).toBeNull();
expect(screen.queryByText(/^daemon v/)).toBeNull();
expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION);
expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion);
expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`);
expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`);
expect(screen.getByText("03c3e47f")).toBeTruthy();
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
@@ -200,6 +200,9 @@ describe("App", () => {
expect(copied).toContain("--browser 03c3e47f");
expect(copied).toContain('--purpose "发布 wiki 文档"');
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
expect(copied).toContain("./trace");
expect(copied).toContain("trace.json");
expect(copied).toContain("states/");
// Button label stays static; a transient toast confirms the copy.
expect(copyButton.textContent).toContain("复制录制指令");
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
@@ -244,6 +247,41 @@ describe("App", () => {
expect(copyButton.getAttribute("disabled")).not.toBeNull();
expect(screen.getByText("连接后可用")).toBeTruthy();
});
it("treats protocol drift as still connected and prompts an upgrade", async () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
state: "version_skew",
instanceId: "03c3e47f",
handshake: {
server: "bh",
version: mockDaemonVersion,
protocol_version: "1.0",
},
},
statusState: "version_skew",
setLabel,
setConnectionEnabled,
});
render(<App />);
expect(screen.getByText("已连接")).toBeTruthy();
expect(screen.getByText("可升级")).toBeTruthy();
expect(screen.queryByText("协议不一致")).toBeNull();
expect(screen.queryByText("Action needed")).toBeNull();
expect(screen.queryByText("兼容")).toBeNull();
const warning = screen.getByText(/协议版本不同,请及时升级/);
expect(warning.textContent).toContain("CLI 协议");
expect(warning.textContent).toContain("扩展协议");
openRecordView();
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
expect(copyButton.getAttribute("disabled")).toBeNull();
fireEvent.click(copyButton);
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
});
});
describe("control hints toggle", () => {
+6 -5
View File
@@ -79,6 +79,7 @@ export function App() {
}, [copiedTick]);
const isSkewed = statusState === "version_skew";
const connectionLive = statusState === "connected" || isSkewed;
const daemonVersion = snapshot.handshake?.version ?? "—";
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
const extensionVersion = snapshot.extensionVersion || "—";
@@ -90,7 +91,7 @@ export function App() {
setCopiedInstanceId(true);
};
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
const recordReady = connectionLive && Boolean(snapshot.instanceId);
const recordPurpose = purposeDraft.trim();
const recordStartUrl = startUrlDraft.trim();
const recordCommand = snapshot.instanceId
@@ -206,12 +207,12 @@ export function App() {
</div>
{isSkewed && (
<p
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
className="mt-2 text-xs leading-snug text-muted-foreground"
data-slot="popup-version-skew-warning"
>
{t("popup.versionSkewWarning", {
extensionProtocol: PROTOCOL_VERSION,
daemonProtocol,
cliProtocol: daemonProtocol,
})}
</p>
)}
@@ -266,9 +267,9 @@ export function App() {
data-slot="popup-meta"
>
<div className="flex shrink-0 items-center gap-1">
<span title={t("popup.extensionVersionHint")}>{extensionVersion}</span>
<span title={t("popup.daemonVersionHint")}>CLI {daemonVersion}</span>
<span aria-hidden>/</span>
<span title={t("popup.daemonVersionHint")}>{daemonVersion}</span>
<span title={t("popup.extensionVersionHint")}>Ext {extensionVersion}</span>
</div>
<div className="flex min-w-0 items-center justify-end gap-1">
<span className="shrink-0">{t("popup.instanceTitle")}</span>
@@ -0,0 +1,14 @@
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_start",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
const dispose = attachRecordFrameAgent();
ctx.onInvalidated(dispose);
},
});
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import {
CAPTURE_SUPPRESS,
type CaptureSuppressMessage,
withOverlaysHiddenForCapture,
withExtensionOverlayHidden,
} from "../capture-suppress-bridge";
function recordingSendToTab(events: string[]) {
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
});
}
describe("withOverlaysHiddenForCapture", () => {
describe("withExtensionOverlayHidden", () => {
it("sends begin → fn → end and returns fn's result", async () => {
const events: string[] = [];
const sendToTab = recordingSendToTab(events);
const result = await withOverlaysHiddenForCapture(
const result = await withExtensionOverlayHidden(
7,
async () => {
events.push("capture");
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
const sendToTab = recordingSendToTab(events);
await expect(
withOverlaysHiddenForCapture(
withExtensionOverlayHidden(
7,
async () => {
events.push("capture");
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
});
const fn = vi.fn(async () => "shot");
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
expect(result).toBe("shot");
expect(fn).toHaveBeenCalledTimes(1);
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
return { type: CAPTURE_SUPPRESS, ok: true };
});
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
expect(result).toBe("shot");
expect(sendToTab).toHaveBeenCalledTimes(2);
@@ -26,20 +26,20 @@ function handshake(
}
describe("computeConnectedState (protocol-based compat)", () => {
it("returns connected when protocol strings match", () => {
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
it("returns connected when daemon protocol equals extension protocol", () => {
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
kind: "connected",
});
});
it("returns version_skew when daemon protocol minor is newer", () => {
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
kind: "version_skew",
});
});
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
kind: "version_skew",
});
});
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
});
it("rejects when extension is below daemon min_compatible_protocol", () => {
const result = computeConnectedState(handshake("1.0", "1.5"));
const result = computeConnectedState(handshake("1.1", "1.5"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("min_compatible_protocol");
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState({
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.0",
protocol_version: "1.1",
min_compatible_peer: "0.1.0",
});
expect(result).toEqual({ kind: "connected" });
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
});
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("daemon min_compatible_protocol");
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
const second = transport.send.mock.calls[1]?.[0] as { id: string };
expect(second.id).not.toBe(first.id);
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
await Promise.resolve();
expect(controller.snapshot().state).not.toBe("connected");
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
});
});
@@ -0,0 +1,101 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForDocumentSettled } from "../recording/document-settle";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForDocumentSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const controller = new AbortController();
const settled = waitForDocumentSettled(
cdp,
{ target: { tabId: 7 } },
{
signal: controller.signal,
},
);
controller.abort();
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
it("probes a same-process iframe through its isolated execution context", async () => {
vi.useFakeTimers();
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
const settled = waitForDocumentSettled(cdp, {
target: { tabId: 7 },
frameId: "child-frame",
});
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledWith(
7,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child-frame" }),
);
expect(send).toHaveBeenCalledWith(
7,
"Runtime.evaluate",
expect.objectContaining({ contextId: 91 }),
);
});
it("routes an OOPIF probe through its CDP target session", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const target = { tabId: 7, sessionId: "oopif-session" };
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 27 }),
);
});
});
@@ -0,0 +1,30 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { createRandomUuid } from "../random-uuid";
describe("createRandomUuid", () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it("uses crypto.randomUUID when it is available", () => {
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
expect(randomUUID).toHaveBeenCalledOnce();
});
it("falls back to getRandomValues and sets UUID v4 bits", () => {
const bytes = new Uint8Array(16).fill(0xff);
const getRandomValues = vi.fn((target: Uint8Array) => {
target.set(bytes);
return target;
});
vi.stubGlobal("crypto", { getRandomValues });
const uuid = createRandomUuid();
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
expect(getRandomValues).toHaveBeenCalledOnce();
});
});
@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import {
isAcceptedRecordStepAck,
isRecordStepMessage,
RECORD_STEP,
type RecordStepMessage,
} from "../record-bridge";
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: "rec-1",
producerId: "document-1",
sequence: 1,
step: { op: "click", target: { tag: "button", name: "Submit" } },
};
describe("record step delivery protocol", () => {
it("requires a producer and a positive integer sequence", () => {
expect(isRecordStepMessage(message)).toBe(true);
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
});
it("accepts only the acknowledgement for the sent sequence", () => {
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
false,
);
});
});
@@ -0,0 +1,213 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
class ListenerSet<T extends (...args: never[]) => unknown> {
readonly listeners = new Set<T>();
addListener = (listener: T) => this.listeners.add(listener);
removeListener = (listener: T) => this.listeners.delete(listener);
}
function fakePort(sender: chrome.runtime.MessageSender) {
const messages: unknown[] = [];
const onMessage = new ListenerSet<(message: unknown) => void>();
const onDisconnect = new ListenerSet<() => void>();
return {
port: {
name: RECORD_FRAME_PORT,
sender,
onMessage,
onDisconnect,
postMessage: vi.fn((message: unknown) => messages.push(message)),
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port,
messages,
receive(message: unknown) {
for (const listener of onMessage.listeners) listener(message);
},
};
}
describe("RecordFrameCoordinator", () => {
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
const onMessage = new ListenerSet<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => boolean
>();
beforeEach(() => {
onConnect.listeners.clear();
onMessage.listeners.clear();
vi.stubGlobal("chrome", {
runtime: { onConnect, onMessage },
});
});
it("answers child-document queries for the initial tab before it is armed", () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
const query = (tabId: number) => {
let response: unknown;
for (const listener of onMessage.listeners) {
listener(
{ type: RECORD_FRAME_QUERY },
{
tab: { id: tabId },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender,
(value) => {
response = value;
},
);
}
return response;
};
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
expect(query(4)).toEqual({ active: false });
});
it("starts a child Document that appears after the initial frame snapshot", async () => {
let notifyFrameNavigation:
| ((frame: {
tabId: number;
frameId: number;
documentId?: string;
lifecycle: "committed" | "completed";
}) => void)
| undefined;
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument,
subscribeFrameNavigation(listener) {
notifyFrameNavigation = listener;
return () => {};
},
});
coordinator.attach();
const onDocumentReady = vi.fn();
coordinator.begin("rec-1", 10, 3, onDocumentReady);
await coordinator.armTab("rec-1", 3);
sendToDocument.mockClear();
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(sendToDocument).toHaveBeenCalledWith(
3,
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
{ documentId: "late-child" },
);
});
expect(onDocumentReady).not.toHaveBeenCalled();
const child = fakePort({
tab: { id: 3 },
frameId: 7,
documentId: "late-child",
} as chrome.runtime.MessageSender);
for (const listener of onConnect.listeners) listener(child.port);
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(onDocumentReady).toHaveBeenCalledWith({
tabId: 3,
documentId: "late-child",
browserFrameId: 7,
producerId: "producer-1",
});
});
});
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [
{ frameId: 0, documentId: "top-document" },
{ frameId: 7, documentId: "child-document" },
],
sendToDocument,
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
const sender = {
tab: { id: 3 },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
tabId: 3,
documentId: "child-document",
browserFrameId: 7,
producerId: "producer-1",
});
const stopping = coordinator.stop("rec-1");
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
const stop = frame.messages.find(
(message): message is { type: "stop"; commandId: string } =>
typeof message === "object" &&
message !== null &&
(message as { type?: string }).type === "stop",
);
expect(stop).toBeDefined();
frame.receive({
type: "stopped",
requestId: "rec-1",
commandId: stop!.commandId,
ok: true,
});
await expect(stopping).resolves.toBe(true);
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
});
it("rejects a producer used from another Document", async () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await coordinator.armTab("rec-1", 3);
const sender = {
tab: { id: 3 },
frameId: 0,
documentId: "top-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(
coordinator.sourceFor("rec-1", "producer-1", {
...sender,
documentId: "other-document",
}),
).toBeNull();
});
});
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { markRecordingDocument } from "../recording/document-marker";
import { ObservationNodeIndex } from "../recording/observation-capture";
describe("recording document marker", () => {
it("rejects a non-random Document identity", () => {
expect(() => markRecordingDocument("user@example.com")).toThrow(
"recording Document identity must be a random UUID",
);
});
it("restores the Document attribute exactly", () => {
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
const producerId = "123e4567-e89b-42d3-a456-426614174000";
const marker = markRecordingDocument(producerId);
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
recordingDocumentMarkerValue(producerId),
);
marker.restore();
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
});
it("indexes the CDP scope from the safe frame identity", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("producer-1")).toEqual({
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
});
});
it("fails closed when a recording identity maps to multiple Documents", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "left",
target: { tabId: 3, sessionId: "left-session" },
recordingDocumentId: "duplicate",
},
{
frameId: "right",
target: { tabId: 3, sessionId: "right-session" },
recordingDocumentId: "duplicate",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("duplicate")).toBeUndefined();
});
});
@@ -0,0 +1,146 @@
import { describe, expect, it } from "vitest";
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import type { RecordingDraftStep } from "../recording/types";
const URL = "https://example.com/login";
function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({
url: URL,
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
});
session.cursor.lastSettled = {
stateId: state.id,
rootFrameId: "root",
index: new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "input",
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
}),
url: URL,
};
return session;
}
function finalizedFillTrace(value: string, redactValues: boolean) {
const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = {
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Password" },
value,
targetHint: {
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
},
};
session.bindDraft(draft, 1);
draft.postStateId = draft.preStateId;
return buildTraceV3({
registry: session.registry,
drafts: [draft],
annotations: session.annotations,
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
redactValues,
});
}
function finalizedFillBody(value: string, redactValues: boolean): string {
return finalizedFillTrace(value, redactValues).states[0]!.body;
}
describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private";
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
expect(dumped).toContain("step 1: fill");
expect(dumped).not.toContain(secret);
});
it("keeps ordinary fill details", () => {
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
});
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({
url: "https://example.com/a\nb",
title: "hello\nworld",
vomText: "@vom 1",
});
const trace = buildTraceV3({
registry,
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
});
});
describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry();
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
});
it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry();
registry.register({ url: URL, vomText: "same" });
const state = registry.register({
url: URL,
title: "Login",
vomText: "same",
truncated: true,
});
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
});
});
describe("draft binding", () => {
it("keeps capture semantics when no observation exists", () => {
const session = new RecordingObservationSession();
const draft: RecordingDraftStep = {
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
};
session.bindDraft(draft, 1);
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
expect(draft.preStateId).toBeUndefined();
});
it("does not bind an unmatched new action to a stale observation", () => {
const session = sessionWithInput();
const draft: RecordingDraftStep = {
op: "click",
captureTarget: { tag: "button", role: "button", name: "Confirm" },
targetHint: {
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
},
};
session.bindDraft(draft, 2, true);
expect(draft.preStateId).toBeUndefined();
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
role: "button",
name: "Confirm",
unmatched: true,
});
});
});
@@ -0,0 +1,289 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
const captureRecordingObservation = vi.hoisted(() => vi.fn());
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
vi.mock("../recording/observation-capture", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
return { ...actual, captureRecordingObservation };
});
vi.mock("../recording/document-settle", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/document-settle")>();
return { ...actual, waitForDocumentSettled };
});
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
import type { RecordingDraftStep } from "../recording/types";
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
};
}
function runtime(): RecordingObservationRuntime {
return new RecordingObservationRuntime({
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
});
}
describe("RecordingObservationRuntime", () => {
afterEach(() => {
captureRecordingObservation.mockReset();
waitForDocumentSettled.mockClear();
});
it("shares one initial capture and includes it in flush", async () => {
let release!: (value: ReturnType<typeof observation>) => void;
captureRecordingObservation.mockReturnValueOnce(
new Promise((resolve) => {
release = resolve;
}),
);
const recording = runtime();
const first = recording.captureInitial(7);
const second = recording.captureInitial(7);
const flushed = recording.flush();
let flushCompleted = false;
void flushed.then(() => {
flushCompleted = true;
});
await Promise.resolve();
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
expect(flushCompleted).toBe(false);
release(observation());
await Promise.all([first, second, flushed]);
expect(flushCompleted).toBe(true);
});
it("cancels an in-flight initial capture", async () => {
captureRecordingObservation.mockImplementationOnce(
(input: { signal?: AbortSignal }) =>
new Promise((_, reject) => {
input.signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
}),
);
const recording = runtime();
const pending = recording.captureInitial(7);
recording.cancel();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
});
it("retries a failed initial capture when the recording settles at stop", async () => {
captureRecordingObservation
.mockRejectedValueOnce(new Error("document swapped"))
.mockResolvedValueOnce(observation());
const recording = runtime();
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
await recording.settleTrailing(7, []);
const trace = recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
});
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
expect(trace.states).toHaveLength(1);
});
it("keeps runtime matching data out of the serialized trace", async () => {
const captured = observation();
captured.index = new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "button",
rect: { x: 10, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
});
captureRecordingObservation.mockResolvedValueOnce(captured);
const recording = runtime();
await recording.captureInitial(7);
const dumped = JSON.stringify(
recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
}),
);
expect(dumped).toContain("RootWebArea");
expect(dumped).not.toContain("matchNodes");
expect(dumped).not.toContain("localRect");
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
await recording.captureInitial(4);
await recording.captureInitial(5);
expect(transition).toEqual({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
it("binds an iframe draft to its marked CDP Document scope", async () => {
captureRecordingObservation.mockResolvedValueOnce({
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
}),
});
const recording = runtime();
const drafts: RecordingDraftStep[] = [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "Save" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.captureInitial(7);
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.targetHint).toMatchObject({
frameId: "child",
geometrySpace: "local",
});
expect(draft.matchedTarget?.ref).toBe("e1");
});
it("refreshes a late iframe before matching its first action", async () => {
const iframeObservation = {
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [
{
ref: "e1",
backendNodeId: 42,
frameId: "child",
role: "button",
name: "表格视图",
line: 1,
},
],
}),
};
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockResolvedValueOnce(iframeObservation)
.mockResolvedValueOnce(iframeObservation);
const recording = runtime();
await recording.captureInitial(7);
await recording.refreshDocument(7, "producer-1");
const drafts: RecordingDraftStep[] = [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "表格视图" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
expect(waitForDocumentSettled).toHaveBeenCalledWith(
expect.anything(),
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
{ signal: expect.any(AbortSignal) },
);
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
});
it("does not poison final flush when a frame readiness refresh fails", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockRejectedValueOnce(new Error("child Document replaced"));
const recording = runtime();
await recording.captureInitial(7);
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
"child Document replaced",
);
await expect(recording.flush()).resolves.toBeUndefined();
});
});
@@ -1,80 +0,0 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
expects_navigation: true,
});
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
});
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
const buffer = {
steps: [
{
op: "click" as const,
target: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
});
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
const buffer = {
steps: [
{
op: "select" as const,
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigated_to: "https://example.com/list?cat=tech",
},
]);
});
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/a",
pendingNavigation: false,
};
observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(buffer.steps[0]).toMatchObject({
op: "navigate",
url: "https://example.com/b",
});
});
});
@@ -0,0 +1,136 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { SettleController } from "../recording/settle-controller";
import type { RecordingDraftStep } from "../recording/types";
const OBSERVATION: RegisteredObservation = {
stateId: "s-next",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/next",
};
describe("SettleController", () => {
afterEach(() => vi.useRealTimers());
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
let firstSignal: AbortSignal | undefined;
vi.spyOn(session, "capture")
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
firstSignal = signal;
return new Promise<RegisteredObservation>((_resolve, reject) => {
signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
});
})
.mockResolvedValueOnce(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.schedule(drafts, 0);
await vi.advanceTimersByTimeAsync(180);
drafts.push({
op: "click",
captureTarget: { tag: "button", name: "Second" },
preStateId: "s-next",
});
controller.schedule(drafts, 1);
expect(firstSignal?.aborted).toBe(true);
expect(drafts[0]?.postStateId).toBe("s-next");
await vi.advanceTimersByTimeAsync(500);
await controller.flush();
expect(drafts[1]?.postStateId).toBe("s-next");
});
it("does not consume a newer redirect while reading the prior landing URL", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const tabsApi: ChromeTabsApi = {
get: vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
session.cursor.lastSettled = OBSERVATION;
const drafts: RecordingDraftStep[] = [];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
await vi.advanceTimersByTimeAsync(180);
expect(tabsApi.get).toHaveBeenCalledTimes(1);
controller.scheduleRedirect(drafts, OBSERVATION.url);
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(500);
await controller.flushRedirects();
expect(drafts).toEqual([]);
expect(controller.hasPending).toBe(false);
});
it("settles an action in its own OOPIF document scope", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
const target = { tabId: 7, sessionId: "oopif-session" };
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await controller.flush();
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 39 }),
);
expect(drafts[0]?.postStateId).toBe("s-next");
});
});
@@ -0,0 +1,164 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
expects_navigation: true,
});
expect(buffer.steps).toEqual([
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.navigation.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(
buffer,
{
op: "hover",
target: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
{
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
);
expect(buffer.steps).toEqual([
{
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
targetHint: {
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
},
]);
});
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
const buffer = {
steps: [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
navigation: {
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
});
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
const buffer = {
steps: [
{
op: "select" as const,
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
navigation: {
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
{
op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigatedTo: "https://example.com/list?cat=tech",
},
]);
});
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
};
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({
op: "navigate",
url: "https://example.com/b",
});
});
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://passport.example/login",
pendingNavigation: false,
},
};
const hop1 = observeRecordedNavigation(
buffer,
"https://passport.example/callback",
false,
"link",
["server_redirect"],
);
expect(hop1).toEqual({
kind: "coalesce_redirect",
url: "https://passport.example/callback",
});
expect(buffer.steps).toEqual([]);
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
"client_redirect",
]);
expect(hop2).toEqual({
kind: "coalesce_redirect",
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
});
});
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
tabs.commit(6);
expect(tabs.currentTabId).toBe(6);
});
it("commits the freshly observed URL when returning to an existing tab", () => {
const tabs = new RecordingTabCoordinator(4);
tabs.navigation(5, "https://example.com/old");
tabs.commit(5, "https://example.com/new");
expect(tabs.currentTabId).toBe(5);
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
});
});
@@ -0,0 +1,159 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { matchObservationTarget } from "../recording/target-matcher";
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
return {
backendNodeId,
frameId,
tag: "button",
rect: { x, y: 20, w: 100, h: 30 },
localRect: { x, y: 20, w: 100, h: 30 },
};
}
function observation(
matchNodes: CaptureVomMatchNode[],
refs: RenderedRef[],
): RegisteredObservation {
return {
stateId: "s1",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
url: "https://example.com",
};
}
describe("matchObservationTarget", () => {
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
const geometry = node(42, "child");
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
const ref: RenderedRef = {
ref: "e1",
backendNodeId: 42,
frameId: "child",
line: 1,
};
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
expect(candidate).toEqual({ frameId: "child", geometry, ref });
});
it("matches a unique node using canonical top-level viewport geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("uses frame id with backend node id so sibling frames cannot collide", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "left"), node(42, "right")],
[
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
],
),
hint: {
frameId: "right",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e2");
});
it("matches iframe capture geometry in the child viewport coordinate space", () => {
const child = node(42, "child", 410);
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
const target = matchObservationTarget({
observation: observation(
[child],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: {
frameId: "child",
geometrySpace: "local",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e1");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "child")],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("fails closed when the source Document has no VOM frame mapping", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: {
frameId: null,
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
fallback: { tag: "button", role: "button", name: "发布" },
});
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, line: 1 },
{ ref: "e2", backendNodeId: 43, line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target.unmatched).toBe(true);
});
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
const target = matchObservationTarget({
observation: observation(
[node(41, "root"), node(42, "child")],
[
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
],
),
hint: { frameId: "child" },
fallback: { tag: "button", role: "button", name: "保存" },
});
expect(target.ref).toBe("e2");
});
it("uses semantics to disambiguate equal geometry in one frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", role: "button", name: "取消" },
});
expect(target.ref).toBe("e2");
});
});
@@ -1,6 +1,15 @@
import { describe, expect, it } from "vitest";
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
import type { RecordingDraftStep } from "@/lib/recording/types";
import { shouldRecordPress } from "../recording/draft-policy";
import { buildTraceV2 } from "../recording/trace-reducer-v2";
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
return buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
...(startUrl ? { startUrl } : {}),
});
}
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
@@ -18,36 +27,47 @@ describe("shouldRecordPress", () => {
});
describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => {
const drafts: DraftTraceStep[] = [
const drafts: RecordingDraftStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
page_url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigatedTo: "https://example.com/results/42",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/p/99",
pageUrl: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/p/99",
pageUrl: "https://example.com/p/99",
},
];
@@ -109,9 +129,9 @@ describe("reduceTraceSteps", () => {
[
{
op: "fill",
target: { tag: "input", role: "textbox", name: "Search query" },
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
value: "",
page_url: "https://example.com/search",
pageUrl: "https://example.com/search",
},
],
"https://example.com/search",
@@ -125,16 +145,16 @@ describe("reduceTraceSteps", () => {
]);
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
navigatedTo: "https://example.com/list?cat=tech",
pageUrl: "https://example.com/list",
},
],
"https://example.com/list",
@@ -151,36 +171,32 @@ describe("reduceTraceSteps", () => {
});
});
it("keeps hover steps before menu clicks", () => {
it("drops hover steps, which only exist in trace v3", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { tag: "span", role: "button", name: "Account" },
page_url: "https://example.com/app",
captureTarget: { tag: "span", role: "button", name: "Account" },
pageUrl: "https://example.com/app",
},
{
op: "click",
target: { tag: "a", role: "link", name: "Profile" },
page_url: "https://example.com/app",
captureTarget: { tag: "a", role: "link", name: "Profile" },
pageUrl: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
expect(steps.map((s) => s.op)).toEqual(["click"]);
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
"https://example.com/start",
),
buildTraceV2({
steps: [{ op: "navigate", url: "https://example.com/other" }],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/start",
}).entry.start_url,
).toBe("https://example.com/start");
});
});
@@ -0,0 +1,132 @@
import { describe, expect, it } from "vitest";
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => {
it("removes form literals when value redaction is requested", () => {
const reduced = reduceTraceStepsV3(
[
{
op: "fill",
value: "private@example.com",
preStateId: "s1",
postStateId: "s1",
},
{
op: "select",
values: ["private-account-id"],
labels: ["Private account"],
preStateId: "s1",
postStateId: "s1",
},
],
{ redactValues: true },
);
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
expect(reduced.steps[1]).toMatchObject({ op: "select" });
expect(reduced.steps[1]).not.toHaveProperty("selection");
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("keeps hover steps before menu clicks", () => {
const drafts: RecordingDraftStep[] = [
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
preStateId: "s1",
postStateId: "s1",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
preStateId: "s1",
postStateId: "s1",
},
];
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
});
it("does not export a tab transition without both observation endpoints", () => {
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
expect(
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
).toEqual([]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
{
op: "navigate",
url: "https://example.com/final",
transitionQualifiers: ["server_redirect"],
preStateId: "s2",
postStateId: "s3",
},
];
const before = structuredClone(drafts);
const reduced = reduceTraceStepsV3(drafts);
expect(drafts).toEqual(before);
expect(reduced.steps).toEqual([
expect.objectContaining({
id: 1,
op: "navigate",
state: "s1",
to: "https://example.com/final",
result: { state: "s3" },
}),
]);
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
});
it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
const trace = buildTraceV3({
registry,
drafts: [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "Save" },
preStateId: state.id,
postStateId: state.id,
},
],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.version).toBe(TRACE_VERSION_V3);
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
expect(trace.steps[0]).toMatchObject({
op: "click",
target: { role: "button", name: "Save", unmatched: true },
});
});
});
@@ -1,9 +1,9 @@
/**
* Wire protocol and background-side helper for hiding the extension's
* in-page overlay while a screenshot is captured, so captured frames only
* contain the page itself.
* Wire protocol and background-side helper for temporarily excluding the
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
* coordinate-driven native input uses it so Chrome hit-tests the page.
*
* The background wraps every capture in `withOverlaysHiddenForCapture`,
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
* which sends `begin` (the content script hides the overlay host and only
* acks once the compositor has produced an overlay-free frame) and always
* follows up with `end`, even when the capture throws. Tabs without the
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
* (or any other `begin` failure) simply skips suppression — there is no
* overlay to hide in that tab.
*/
export async function withOverlaysHiddenForCapture<T>(
export async function withExtensionOverlayHidden<T>(
tabId: number,
fn: () => Promise<T>,
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
+6 -5
View File
@@ -1,12 +1,13 @@
/**
* Build a semantic TargetDescriptor for an interacted element.
* Build a semantic capture descriptor for an interacted element.
*
* Trace steps are an LLM *textbook*: each click must say what to look for
* on screen (usually a short visible name). Tag-only noise like
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
*/
export interface TargetDescriptor {
/** Content-script capture descriptor before VOM geometric matching. */
export interface CaptureTargetDescriptor {
role?: string;
name?: string;
tag: string;
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
* (visible name), or at least a form `name_attr` for checkbox/radio.
* Recording “点击div” with no name fails this bar.
*/
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
const name = target.name?.trim();
if (name && isActionableLabel(name)) return true;
if (
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
return undefined;
}
export function describeTarget(el: Element): TargetDescriptor {
export function describeTarget(el: Element): CaptureTargetDescriptor {
const tag = el.tagName.toLowerCase();
const role = inferRole(el);
const name = accessibleName(el);
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
};
}
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
if (!(target instanceof Element)) return null;
const clickable = resolveClickableElement(target);
if (!clickable) return null;
+4 -2
View File
@@ -3,8 +3,10 @@
* between the content-script control overlay and the background SW.
*
* Content script → background:
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
* overlay state for the sender's window.
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
* (see overlayStateForTab) and pushes the authoritative overlay state for
* that tab. It also proactively pushes on tab create and control-mode
* change, so a tab usually receives state before it ever sends ready.
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
* daemon (via a `session.user_interrupt` WS event) to cancel
* every inflight + queued tool call for that session with
+26
View File
@@ -0,0 +1,26 @@
/**
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
*
* Crypto.randomUUID() is restricted to secure contexts, while
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
* origins. BrowserSkill records many intranet pages served over HTTP, so the
* fallback is required for recording to work on those pages.
*/
export function createRandomUuid(): string {
const cryptoApi = globalThis.crypto;
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
const bytes = new Uint8Array(16);
cryptoApi.getRandomValues(bytes);
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
return [
hex.slice(0, 8),
hex.slice(8, 12),
hex.slice(12, 16),
hex.slice(16, 20),
hex.slice(20),
].join("-");
}
+39 -3
View File
@@ -3,7 +3,7 @@
* service worker and a tab's content script.
*/
import type { TargetDescriptor } from "./describe-target";
import type { CaptureTargetDescriptor } from "./describe-target";
export const RECORD_START = "bsk-record-start";
export const RECORD_STEP = "bsk-record-step";
@@ -16,6 +16,10 @@ export interface RecordStartAck {
ok: true;
}
export type RecordStepAck =
| { ok: true; sequence: number }
| { ok: false; expectedSequence: number; error: string };
export type RecordStopAck = { ok: true } | { ok: false; error: string };
export interface RecordQueryMessage {
@@ -42,7 +46,7 @@ export interface RecordStartMessage {
export interface RecordStepPayload {
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
target?: TargetDescriptor;
target?: CaptureTargetDescriptor;
value?: string;
key?: string;
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
@@ -50,17 +54,30 @@ export interface RecordStepPayload {
labels?: string[];
url?: string;
redacted?: boolean;
commit?: "enter" | "suggestion" | "blur";
/** Page URL when the step was captured. */
page_url?: string;
/** Event-target bounds in the source frame's viewport coordinate space. */
geometry?: {
rect: { x: number; y: number; w: number; h: number };
tag: string;
};
/** Capture-only hint; never persisted unless converted to navigated_to. */
expects_navigation?: boolean;
/** Whether an observed URL change was synchronously caused by the action. */
navigation_caused_by_action?: boolean;
/** Raw webNavigation transition metadata for cause mapping. */
transitionType?: string;
transitionQualifiers?: string[];
}
export interface RecordStepMessage {
type: typeof RECORD_STEP;
requestId: string;
/** Stable for the lifetime of one content-script document. */
producerId: string;
/** Monotonic within one content-script producer and recording request. */
sequence: number;
step: RecordStepPayload;
}
@@ -112,8 +129,27 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
if (
m.type !== RECORD_STEP ||
typeof m.requestId !== "string" ||
typeof m.producerId !== "string" ||
m.producerId.length === 0 ||
typeof m.sequence !== "number" ||
!Number.isSafeInteger(m.sequence) ||
m.sequence < 1
) {
return false;
}
const step = m.step;
if (typeof step !== "object" || step === null) return false;
return typeof (step as RecordStepPayload).op === "string";
}
export function isAcceptedRecordStepAck(
value: unknown,
sequence: number,
): value is Extract<RecordStepAck, { ok: true }> {
if (typeof value !== "object" || value === null) return false;
const ack = value as Partial<RecordStepAck>;
return ack.ok === true && ack.sequence === sequence;
}
@@ -1,136 +0,0 @@
import type { DraftTraceStep } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
const pageUrl = payload.page_url;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "hover":
return payload.target
? {
op: "hover",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
op: "fill",
target: payload.target,
value: payload.value ?? "",
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target ? { target: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
target: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select") {
buffer.steps[i] = { ...step, navigated_to: url };
return true;
}
break;
}
return false;
}
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
): void {
if (!url || url === buffer.currentUrl) return;
buffer.currentUrl = url;
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
if (!annotateLastStepNavigation(buffer, url)) {
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
}
return;
}
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
}
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
): void {
if (payload.op === "navigate") {
if (payload.url) {
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
}
return;
}
const step = toDraftStep({
...payload,
page_url: payload.page_url ?? buffer.currentUrl,
});
if (!step) return;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
}
@@ -0,0 +1,39 @@
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
export interface RecordingDocumentMarker {
restore(): void;
ensure(): void;
}
export function waitForDocumentElement(): Promise<HTMLElement> {
if (document.documentElement) return Promise.resolve(document.documentElement);
return new Promise((resolve) => {
const observer = new MutationObserver(() => {
if (!document.documentElement) return;
observer.disconnect();
resolve(document.documentElement);
});
observer.observe(document, { childList: true });
});
}
export function markRecordingDocument(
producerId: string,
root: HTMLElement = document.documentElement,
): RecordingDocumentMarker {
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
const markerValue = recordingDocumentMarkerValue(producerId);
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
ensure();
return {
ensure,
restore() {
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
},
};
}
@@ -0,0 +1,134 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
const SETTLE_MIN_MS = 150;
const SETTLE_QUIET_MS = 250;
const SETTLE_MAX_MS = 2_000;
const SETTLE_POLL_MS = 60;
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export interface DocumentSettleScope {
target: CdpTarget;
/** Omit only for the target's root document. */
frameId?: string;
}
type SettleOutcome = "quiet" | "timeout" | "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
interface ProbeContext {
executionContextId?: number;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
async function executionContextForScope(
cdp: CdpRunner,
scope: DocumentSettleScope,
): Promise<number | undefined> {
if (!scope.frameId) return undefined;
const result = await sendToCdpTarget<{ executionContextId?: number }>(
cdp,
scope.target,
"Page.createIsolatedWorld",
{
frameId: scope.frameId,
worldName: SETTLE_WORLD_NAME,
grantUniveralAccess: false,
},
);
return result.executionContextId;
}
async function readQuietProbe(
cdp: CdpRunner,
scope: DocumentSettleScope,
context: ProbeContext,
): Promise<QuietProbe | null> {
try {
if (scope.frameId && context.executionContextId === undefined) {
context.executionContextId = await executionContextForScope(cdp, scope);
if (context.executionContextId === undefined) return null;
}
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
cdp,
scope.target,
"Runtime.evaluate",
{
expression: QUIET_PROBE,
returnByValue: true,
...(context.executionContextId !== undefined
? { contextId: context.executionContextId }
: {}),
},
);
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
context.executionContextId = undefined;
return null;
}
}
export async function waitForDocumentSettled(
cdp: CdpRunner,
scope: DocumentSettleScope,
options: { signal?: AbortSignal } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
const context: ProbeContext = {};
for (;;) {
if (options.signal?.aborted) return "cancelled";
await sleep(SETTLE_POLL_MS, options.signal);
if (options.signal?.aborted) return "cancelled";
const probe = await readQuietProbe(cdp, scope, context);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe || probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
@@ -0,0 +1,34 @@
import type { RecordingDraftStep } from "./types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
}
export function resolveDraftStartUrl(
drafts: RecordingDraftStep[],
explicitStartUrl?: string,
fallbackUrl?: string,
): string {
if (explicitStartUrl) return explicitStartUrl;
const navigation = drafts.find(
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
);
if (navigation) return navigation.url;
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
}
@@ -0,0 +1,65 @@
export const RECORD_FRAME_PORT = "bsk-record-frame";
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
export const RECORD_FRAME_START = "bsk-record-frame-start";
export interface RecordFrameQueryMessage {
type: typeof RECORD_FRAME_QUERY;
}
export interface RecordFrameQueryResponse {
active: boolean;
requestId?: string;
startedAtMs?: number;
}
export interface RecordFrameStartMessage {
type: typeof RECORD_FRAME_START;
requestId: string;
startedAtMs: number;
}
export type RecordFramePortMessage =
| {
type: "ready";
requestId: string;
producerId: string;
}
| {
type: "ready_ack";
requestId: string;
producerId: string;
}
| {
type: "stop";
requestId: string;
commandId: string;
}
| {
type: "cancel";
requestId: string;
}
| {
type: "stopped";
requestId: string;
commandId: string;
ok: boolean;
error?: string;
};
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
return (
typeof value === "object" &&
value !== null &&
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
);
}
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
if (typeof value !== "object" || value === null) return false;
const message = value as Partial<RecordFrameStartMessage>;
return (
message.type === RECORD_FRAME_START &&
typeof message.requestId === "string" &&
typeof message.startedAtMs === "number"
);
}
@@ -0,0 +1,400 @@
import { createRandomUuid } from "../random-uuid";
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "./frame-bridge";
export interface RecordingCaptureScope {
tabId: number;
documentId: string;
browserFrameId: number;
producerId: string;
}
interface ArmedRecording {
requestId: string;
startedAtMs: number;
tabIds: Set<number>;
agents: Map<string, FrameAgent>;
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
finishing: boolean;
}
interface FrameAgent extends RecordingCaptureScope {
requestId: string;
port: chrome.runtime.Port;
stopWaiters: Map<string, (ok: boolean) => void>;
}
interface BrowserFrame {
frameId: number;
documentId?: string;
}
interface BrowserFrameNavigation extends BrowserFrame {
tabId: number;
lifecycle: "committed" | "completed";
}
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
export interface RecordFrameCoordinatorDeps {
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
sendToDocument(
tabId: number,
message: RecordFrameStartMessage,
target: { documentId?: string; frameId?: number },
): Promise<unknown>;
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
}
function documentKey(tabId: number, documentId: string): string {
return `${tabId}:${documentId}`;
}
function senderAddress(
sender: chrome.runtime.MessageSender,
): { tabId: number; documentId: string; browserFrameId: number } | null {
const tabId = sender.tab?.id;
const documentId = sender.documentId;
const browserFrameId = sender.frameId;
if (
typeof tabId !== "number" ||
typeof documentId !== "string" ||
typeof browserFrameId !== "number"
) {
return null;
}
return { tabId, documentId, browserFrameId };
}
function defaultDeps(): RecordFrameCoordinatorDeps {
return {
async getAllFrames(tabId) {
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
},
sendToDocument(tabId, message, target) {
return chrome.tabs.sendMessage(tabId, message, target);
},
subscribeFrameNavigation(listener) {
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
listener({ ...details, lifecycle: "committed" });
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
listener({ ...details, lifecycle: "completed" });
chrome.webNavigation.onCommitted.addListener(onCommitted);
chrome.webNavigation.onCompleted.addListener(onCompleted);
return () => {
chrome.webNavigation.onCommitted.removeListener(onCommitted);
chrome.webNavigation.onCompleted.removeListener(onCompleted);
};
},
};
}
export class RecordFrameCoordinator {
readonly #deps: RecordFrameCoordinatorDeps;
readonly #recordings = new Map<string, ArmedRecording>();
#attached = false;
#detachFrameNavigation: (() => void) | null = null;
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
this.#deps = deps;
}
attach(): () => void {
if (this.#attached) return () => {};
this.#attached = true;
chrome.runtime.onConnect.addListener(this.#onConnect);
chrome.runtime.onMessage.addListener(this.#onMessage);
this.#detachFrameNavigation =
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
return () => {
if (!this.#attached) return;
this.#attached = false;
chrome.runtime.onConnect.removeListener(this.#onConnect);
chrome.runtime.onMessage.removeListener(this.#onMessage);
this.#detachFrameNavigation?.();
this.#detachFrameNavigation = null;
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
this.#recordings.clear();
};
}
begin(
requestId: string,
startedAtMs: number,
initialTabId: number,
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
): void {
const previous = this.#recordings.get(requestId);
if (previous) this.#cancelAgents(previous);
this.#recordings.set(requestId, {
requestId,
startedAtMs,
// Register the initial tab before navigation begins. A newly-created
// child document can query at document_start before armTab's frame
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
tabIds: new Set([initialTabId]),
agents: new Map(),
...(onDocumentReady ? { onDocumentReady } : {}),
finishing: false,
});
}
async armTab(requestId: string, tabId: number): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording || recording.finishing) return false;
recording.tabIds.add(tabId);
let frames: BrowserFrame[];
try {
frames = await this.#deps.getAllFrames(tabId);
} catch {
frames = [{ frameId: 0 }];
}
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
const results = await Promise.all(
frames.map(async (frame) => {
const started = await this.#startFrame(recording, tabId, frame);
return { frameId: frame.frameId, started };
}),
);
return results.some((result) => result.frameId === 0 && result.started);
}
sourceFor(
requestId: string,
producerId: string,
sender: chrome.runtime.MessageSender,
): RecordingCaptureScope | null {
const address = senderAddress(sender);
const recording = this.#recordings.get(requestId);
if (!address || !recording) return null;
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
if (!agent || agent.producerId !== producerId) return null;
return {
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
};
}
async stop(requestId: string): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = createRandomUuid();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
const succeeded = results.every(Boolean);
if (succeeded) this.#recordings.delete(requestId);
else recording.finishing = false;
return succeeded;
}
cancel(requestId: string): void {
const recording = this.#recordings.get(requestId);
if (!recording) return;
this.#cancelAgents(recording);
this.#recordings.delete(requestId);
}
readonly #onMessage = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordFrameQueryResponse) => void,
): boolean => {
if (!isRecordFrameQueryMessage(message)) return false;
const tabId = sender.tab?.id;
const recording =
typeof tabId === "number"
? [...this.#recordings.values()].find(
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
)
: undefined;
sendResponse(
recording
? {
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
}
: { active: false },
);
return false;
};
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
if (frame.frameId === 0) return;
for (const recording of this.#recordings.values()) {
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
// onCommitted can race document_start, while onCompleted runs after the
// content script is available. Starting twice is safe and closes both
// the early-query and late-SPA-frame gaps.
void (async () => {
const started = await this.#startFrame(recording, frame.tabId, frame);
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
const agent = frame.documentId
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
: [...recording.agents.values()].find(
(candidate) =>
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
);
if (!agent) return;
await recording.onDocumentReady?.({
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
});
})();
}
};
readonly #onConnect = (port: chrome.runtime.Port): void => {
if (port.name !== RECORD_FRAME_PORT) return;
const address = senderAddress(port.sender ?? {});
if (!address) {
port.disconnect();
return;
}
let registered: FrameAgent | null = null;
port.onMessage.addListener((raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (message.type === "ready") {
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
port.disconnect();
return;
}
const requestId = message.requestId;
const producerId = message.producerId;
if (registered) {
if (registered.requestId === requestId && registered.producerId === producerId) {
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
} else {
port.disconnect();
}
return;
}
const recording = this.#recordings.get(requestId);
if (
!recording ||
recording.finishing ||
!recording.tabIds.has(address.tabId) ||
producerId.length === 0
) {
port.disconnect();
return;
}
const agent: FrameAgent = {
...address,
requestId,
producerId,
port,
stopWaiters: new Map(),
};
registered = agent;
const key = documentKey(address.tabId, address.documentId);
const previous = recording.agents.get(key);
if (previous && previous !== agent) previous.port.disconnect();
recording.agents.set(key, agent);
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
return;
}
if (
message.type === "stopped" &&
registered &&
message.requestId === registered.requestId &&
typeof message.commandId === "string"
) {
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
registered.stopWaiters.delete(message.commandId);
}
});
port.onDisconnect.addListener(() => {
if (!registered) return;
for (const resolve of registered.stopWaiters.values()) resolve(true);
const recording = this.#recordings.get(registered.requestId);
const key = documentKey(registered.tabId, registered.documentId);
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
});
};
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => {
agent.stopWaiters.delete(commandId);
resolve(false);
}, RECORD_FRAME_STOP_TIMEOUT_MS);
agent.stopWaiters.set(commandId, (ok) => {
clearTimeout(timer);
resolve(ok);
});
try {
agent.port.postMessage({
type: "stop",
requestId,
commandId,
} satisfies RecordFramePortMessage);
} catch {
clearTimeout(timer);
agent.stopWaiters.delete(commandId);
resolve(true);
}
});
}
async #startFrame(
recording: ArmedRecording,
tabId: number,
frame: BrowserFrame,
): Promise<boolean> {
const message: RecordFrameStartMessage = {
type: RECORD_FRAME_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
const response = await this.#deps.sendToDocument(tabId, message, {
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
});
return isStarted(response);
} catch {
return false;
}
}
#cancelAgents(recording: ArmedRecording): void {
for (const agent of recording.agents.values()) {
try {
agent.port.postMessage({
type: "cancel",
requestId: recording.requestId,
} satisfies RecordFramePortMessage);
} catch {
// A destroyed Document has no remaining capture state to cancel.
}
}
}
}
function isStarted(value: unknown): boolean {
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
}
export const recordFrameCoordinator = new RecordFrameCoordinator();
@@ -0,0 +1,5 @@
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
}
@@ -0,0 +1,140 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type CaptureVomMatchNode,
type CaptureVomObservationResult,
captureVomObservation,
} from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
export interface IndexedObservationNode {
frameId: string;
geometry: CaptureVomMatchNode;
ref?: RenderedRef;
}
export interface CapturedRecordingObservation {
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
title?: string;
vomText: string;
truncated: boolean;
}
export interface RegisteredObservation {
stateId: string;
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
}
export interface RecordingDocumentScope {
frameId: string;
target: CdpTarget;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
function frameTagKey(frameId: string, tag: string): string {
return `${frameId}:${tag.toLowerCase()}`;
}
export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
constructor(
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
Partial<Pick<CaptureVomObservationResult, "frames">>,
) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
this.#refById.set(ref.ref, ref);
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const frame of input.frames ?? []) {
const documentId = frame.recordingDocumentId;
if (!documentId) continue;
this.#scopeByProducer.set(
documentId,
this.#scopeByProducer.has(documentId)
? null
: { frameId: frame.frameId, target: frame.target },
);
}
for (const geometry of input.matchNodes) {
const { frameId } = geometry;
const entry = {
frameId,
geometry,
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
};
const key = frameTagKey(frameId, geometry.tag);
const bucket = this.#nodesByFrameTag.get(key) ?? [];
bucket.push(entry);
this.#nodesByFrameTag.set(key, bucket);
}
}
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
}
ref(refId: string): RenderedRef | undefined {
return this.#refById.get(refId);
}
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
documentScope(producerId: string): RecordingDocumentScope | undefined {
return this.#scopeByProducer.get(producerId) ?? undefined;
}
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureRecordingObservation(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
maxTokens: number;
redactValues: boolean;
signal?: AbortSignal;
}): Promise<CapturedRecordingObservation> {
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
maxTokens: input.maxTokens,
redactValues: input.redactValues,
conditionalSurfaceProbe: false,
signal: input.signal,
});
return {
rootFrameId: captured.rootFrameId,
index: new ObservationNodeIndex(captured),
url,
title,
vomText: captured.text,
truncated: captured.truncated,
};
}
@@ -0,0 +1,128 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
import { RecordingStateRegistry } from "./state-registry";
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
const MIN_CAPTURE_INTERVAL_MS = 200;
export interface TabObservationCursor {
lastSettled: RegisteredObservation | null;
lastCaptureAt: number;
}
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return (
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
}
export class RecordingObservationSession {
readonly registry: RecordingStateRegistry;
readonly cursor: TabObservationCursor;
readonly annotations: StepAnnotation[];
readonly #maxTokens: number;
readonly #redactValues: boolean;
constructor(
options: {
registry?: RecordingStateRegistry;
cursor?: TabObservationCursor;
annotations?: StepAnnotation[];
maxTokens?: number;
redactValues?: boolean;
} = {},
) {
this.registry = options.registry ?? new RecordingStateRegistry();
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
this.annotations = options.annotations ?? [];
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
this.#redactValues = options.redactValues ?? false;
}
async capture(
cdp: CdpRunner,
tabsApi: ChromeTabsApi,
tabId: number,
signal?: AbortSignal,
): Promise<RegisteredObservation> {
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
if (waitMs > 0) await abortableDelay(waitMs, signal);
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
const captured = await captureRecordingObservation({
cdp,
tabsApi,
tabId,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
signal,
});
const state = this.registry.register({
url: captured.url,
title: captured.title,
vomText: captured.vomText,
truncated: captured.truncated,
});
const observation: RegisteredObservation = {
stateId: state.id,
rootFrameId: captured.rootFrameId,
index: captured.index,
url: captured.url,
};
this.cursor.lastSettled = observation;
this.cursor.lastCaptureAt = Date.now();
return observation;
}
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
const observation = this.cursor.lastSettled;
if (isTargeted(draft)) {
draft.matchedTarget = observation
? matchObservationTarget({
observation,
hint: draft.targetHint,
fallback: draft.captureTarget,
})
: unmatchedTarget(draft.captureTarget);
}
if (!observation) return;
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
if (previousActionPending && unmatched) return;
draft.preStateId = observation.stateId;
this.registry.markStep(observation.stateId, draftId);
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
const ref = observation.index.ref(draft.matchedTarget.ref);
if (!ref) return;
this.annotations.push({
draftId,
op: draft.op,
line: ref.line,
stateId: observation.stateId,
...(draft.op === "fill" && !this.#redactValues
? { detail: JSON.stringify(draft.value) }
: {}),
});
}
}
@@ -0,0 +1,277 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
import { buildTraceV3 } from "./trace-builder-v3";
import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
pendingCapture: PendingCapture | null;
frameRefresh: Promise<void>;
frameRefreshAbort: AbortController | null;
}
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
export class RecordingObservationRuntime {
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #registry = new RecordingStateRegistry();
readonly #annotations: StepAnnotation[] = [];
readonly #contexts = new Map<number, TabRecordingContext>();
readonly #maxTokens?: number;
readonly #redactValues: boolean;
constructor(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
maxTokens?: number;
redactValues?: boolean;
}) {
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#maxTokens = input.maxTokens;
this.#redactValues = input.redactValues ?? false;
}
#context(tabId: number): TabRecordingContext {
const existing = this.#contexts.get(tabId);
if (existing) return existing;
const session = new RecordingObservationSession({
registry: this.#registry,
annotations: this.#annotations,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
});
const context: TabRecordingContext = {
session,
settle: new SettleController({
session,
cdp: this.#cdp,
tabsApi: this.#tabsApi,
tabId,
}),
pendingCapture: null,
frameRefresh: Promise.resolve(),
frameRefreshAbort: null,
};
this.#contexts.set(tabId, context);
return context;
}
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
/** Refresh the safe observation after a late child Document becomes usable. */
async refreshDocument(tabId: number, producerId: string): Promise<void> {
const context = this.#context(tabId);
context.frameRefreshAbort?.abort();
const abort = new AbortController();
context.frameRefreshAbort = abort;
const previous = context.frameRefresh;
const refresh = (async () => {
await previous;
if (abort.signal.aborted) return;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (abort.signal.aborted) return;
// The first capture discovers the safe producer -> CDP Document scope.
await this.#capture(tabId);
if (abort.signal.aborted) return;
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
if (scope) {
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
}
if (abort.signal.aborted) return;
// Capture again after the child SPA is quiet so the next action can be
// matched against refs that actually exist in its pre-action state.
await this.#capture(tabId);
})();
const tracked = refresh.catch(() => {});
context.frameRefresh = tracked;
try {
await refresh;
} finally {
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
}
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
producerId?: string,
): Promise<void> {
const draft = drafts[draftIndex];
if (!draft) return;
const context = this.#context(tabId);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// Post-action settle can still provide a usable state.
}
}
const scope = producerId
? context.session.cursor.lastSettled?.index.documentScope(producerId)
: undefined;
if (
producerId &&
(draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select")
) {
draft.targetHint = {
...(draft.targetHint ?? {}),
frameId: scope?.frameId ?? null,
geometrySpace: "local",
};
}
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
context.settle.schedule(drafts, draftIndex, scope);
}
scheduleSettle(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
scope?: DocumentSettleScope,
): void {
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
}
clearRedirect(tabId: number): void {
this.#contexts.get(tabId)?.settle.clearRedirect();
}
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async #flushContext(context: TabRecordingContext): Promise<void> {
await context.frameRefresh;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId: number): Promise<void> {
await this.#contexts.get(tabId)?.settle.flushRedirects();
}
async flush(): Promise<void> {
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// The trace may remain observation-free when CDP is unavailable.
}
}
await context.settle.settleTrailing(drafts);
inferMissingPostStates(drafts);
}
cancel(): void {
for (const context of this.#contexts.values()) {
context.frameRefreshAbort?.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
buildTrace(input: {
drafts: RecordingDraftStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
drafts: input.drafts,
annotations: this.#annotations,
startedAt: input.startedAt,
purpose: input.purpose,
startUrl: input.startUrl,
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
@@ -0,0 +1,233 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import { RecordingObservationSession } from "./observation-session";
import type { RecordingDraftStep } from "./types";
interface PendingSettle {
abort: AbortController;
scope: DocumentSettleScope;
}
interface PendingRedirect {
url: string;
abort: AbortController;
}
const CAPTURE_RETRY_DELAY_MS = 250;
function delay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
for (let index = 0; index < drafts.length - 1; index += 1) {
const draft = drafts[index];
const next = drafts[index + 1];
if (draft && next && !draft.postStateId && next.preStateId) {
draft.postStateId = next.preStateId;
}
}
}
export class SettleController {
readonly #session: RecordingObservationSession;
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #tabId: number;
readonly #rootScope: DocumentSettleScope;
readonly #pending = new Map<number, PendingSettle>();
#queue = Promise.resolve();
#pendingRedirect: PendingRedirect | null = null;
#redirectQueue = Promise.resolve();
constructor(input: {
session: RecordingObservationSession;
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
}) {
this.#session = input.session;
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#tabId = input.tabId;
this.#rootScope = { target: { tabId: input.tabId } };
}
get hasPending(): boolean {
return this.#pending.size > 0 || this.#pendingRedirect !== null;
}
async #captureWithRetry(signal?: AbortSignal) {
try {
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
} catch (error) {
if (signal?.aborted) throw error;
await delay(CAPTURE_RETRY_DELAY_MS, signal);
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
}
}
schedule(
drafts: RecordingDraftStep[],
draftIndex: number,
scope: DocumentSettleScope = this.#rootScope,
): void {
if (scope.target.tabId !== this.#tabId) {
throw new Error(
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
);
}
const landing = drafts[draftIndex]?.preStateId;
for (const [index, pending] of this.#pending) {
if (index >= draftIndex) continue;
pending.abort.abort();
this.#pending.delete(index);
if (landing && drafts[index] && !drafts[index].postStateId) {
drafts[index].postStateId = landing;
}
}
this.#pending.get(draftIndex)?.abort.abort();
const pending = { abort: new AbortController(), scope };
this.#pending.set(draftIndex, pending);
const task = async () => {
if (pending.abort.signal.aborted) return;
try {
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled") return;
const observation = await this.#captureWithRetry(pending.abort.signal);
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
drafts[draftIndex].postStateId = observation.stateId;
}
} catch (error) {
if (!isAbortError(error)) {
console.warn(
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
error,
);
}
} finally {
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
}
};
this.#queue = this.#queue.then(task, task).catch(() => {});
}
cancel(): void {
for (const pending of this.#pending.values()) pending.abort.abort();
this.#pending.clear();
this.clearRedirect();
}
async flush(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#queue;
await current;
if (current === this.#queue) return;
}
console.warn("[bsk record] settle queue kept growing; using completed observations");
}
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
const trailing: RecordingDraftStep[] = [];
for (let index = drafts.length - 1; index >= 0; index -= 1) {
const draft = drafts[index];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
try {
const observation = await this.#captureWithRetry();
for (const draft of trailing) draft.postStateId = observation.stateId;
} catch (error) {
console.warn("[bsk record] final observation at stop failed", error);
}
}
clearRedirect(): void {
this.#pendingRedirect?.abort.abort();
this.#pendingRedirect = null;
}
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
this.#pendingRedirect?.abort.abort();
const pending: PendingRedirect = { url, abort: new AbortController() };
this.#pendingRedirect = pending;
const task = () => this.#settleRedirect(drafts, pending);
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
}
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
let finalUrl = pending.url;
try {
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
} catch {
// The navigation event URL remains the best available destination.
}
if (this.#pendingRedirect !== pending) return;
this.#pendingRedirect = null;
if (
!finalUrl ||
finalUrl === "about:blank" ||
this.#session.cursor.lastSettled?.url === finalUrl
) {
return;
}
const last = drafts[drafts.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
await this.flush();
return;
}
const draft: RecordingDraftStep = {
op: "navigate",
url: finalUrl,
pageUrl: finalUrl,
cause: "browser",
transitionQualifiers: ["server_redirect"],
preStateId: this.#session.cursor.lastSettled?.stateId,
};
drafts.push(draft);
const draftIndex = drafts.length - 1;
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
this.schedule(drafts, draftIndex);
await this.flush();
}
async flushRedirects(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#redirectQueue;
await current;
if (current === this.#redirectQueue) return;
}
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
}
}
@@ -0,0 +1,57 @@
export interface RecordedStateEntry {
id: string;
url: string;
title?: string;
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
vomText: string;
truncated: boolean;
stepsHere: number[];
}
function stateIdentity(url: string, body: string): string {
return `${url}\0${body}`;
}
export class RecordingStateRegistry {
readonly #entriesById = new Map<string, RecordedStateEntry>();
readonly #idByIdentity = new Map<string, string>();
#nextId = 1;
register(input: {
url: string;
title?: string;
vomText: string;
truncated?: boolean;
}): RecordedStateEntry {
const identity = stateIdentity(input.url, input.vomText);
const existingId = this.#idByIdentity.get(identity);
if (existingId) {
const existing = this.#entriesById.get(existingId)!;
if (!existing.title && input.title) existing.title = input.title;
if (input.truncated) existing.truncated = true;
return existing;
}
const entry: RecordedStateEntry = {
id: `s${this.#nextId}`,
url: input.url,
...(input.title ? { title: input.title } : {}),
vomText: input.vomText,
truncated: input.truncated ?? false,
stepsHere: [],
};
this.#nextId += 1;
this.#entriesById.set(entry.id, entry);
this.#idByIdentity.set(identity, entry.id);
return entry;
}
values(): RecordedStateEntry[] {
return [...this.#entriesById.values()];
}
markStep(stateId: string, draftId: number): void {
const entry = this.#entriesById.get(stateId);
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
}
}
@@ -0,0 +1,155 @@
import type { RecordStepPayload } from "../record-bridge";
import { hasRedirectQualifier } from "./navigation-policy";
import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer {
steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): RecordingDraftStep | null {
const pageUrl = payload.page_url;
const common = {
...(pageUrl ? { pageUrl } : {}),
...(targetHint ? { targetHint } : {}),
};
switch (payload.op) {
case "click":
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
case "hover":
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
case "fill":
return payload.target
? {
op: "fill",
captureTarget: payload.target,
value: payload.value ?? "",
...(payload.commit ? { commit: payload.commit } : {}),
...(payload.redacted ? { redacted: true } : {}),
...common,
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target ? { captureTarget: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...common,
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
captureTarget: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...common,
}
: null;
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
const step = buffer.steps[index];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
step.navigatedTo = url;
return index;
}
break;
}
return -1;
}
export type NavigationObserveResult =
| { kind: "noop" }
| { kind: "annotated"; index: number }
| { kind: "appended"; index: number }
| { kind: "coalesce_redirect"; url: string };
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent =
navigation.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
} else {
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
}
if (hasRedirectQualifier(transitionQualifiers)) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
pageUrl: url,
transitionType,
transitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): number | null {
if (payload.op === "navigate") {
if (!payload.url) return null;
const result = observeRecordedNavigation(
buffer,
payload.url,
payload.navigation_caused_by_action,
payload.transitionType,
payload.transitionQualifiers,
);
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep(
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
targetHint,
);
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
return buffer.steps.length - 1;
}
@@ -0,0 +1,61 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number, currentUrl?: string): void {
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
this.#currentTabId = tabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -0,0 +1,86 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { TargetDescriptorV3 } from "@/transport/types";
import type { CaptureTargetDescriptor } from "../describe-target";
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
import type { TargetGeometry, TargetMatchHint } from "./types";
const MATCH_TOLERANCE_PX = 2;
function close(a: number, b: number): boolean {
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
}
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(
candidate: IndexedObservationNode,
geometry: TargetGeometry,
geometrySpace: "top" | "local",
): boolean {
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
return rect != null && rectMatches(geometry.rect, rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
return {
...(fallback?.role ? { role: fallback.role } : {}),
...(fallback?.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
function normalized(value: string | undefined): string {
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
}
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
const role = normalized(fallback?.role);
const name = normalized(fallback?.name);
if (!role && !name) return false;
if (role && normalized(ref.role) !== role) return false;
if (name && normalized(ref.name) !== name) return false;
return true;
}
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
return {
ref: ref.ref,
...(ref.role ? { role: ref.role } : {}),
...(ref.name ? { name: ref.name } : {}),
...(ref.ctx ? { ctx: ref.ctx } : {}),
};
}
export function matchObservationTarget(input: {
observation: RegisteredObservation;
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter(
(candidate) =>
candidate.ref &&
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
);
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
);
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
return unmatchedTarget(input.fallback);
}
const semanticMatches = input.observation.index
.refs(frameId)
.filter((ref) => matchesSemantics(ref, input.fallback));
return semanticMatches.length === 1
? descriptor(semanticMatches[0]!)
: unmatchedTarget(input.fallback);
}
@@ -0,0 +1,92 @@
import {
type StepV3,
type StopReason,
TRACE_VERSION_V3,
type TraceStateV3,
type TraceV3,
VOM_FORMAT_VERSION,
} from "@/transport/types";
import { resolveDraftStartUrl } from "./draft-policy";
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
import { formatTraceStateBody } from "./trace-state-body";
import type { RecordingDraftStep, StepAnnotation } from "./types";
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
const entries = registry.values();
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
return entries.filter((entry) => referenced.has(entry.id));
}
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
return [
...new Set(
draftIds.flatMap((id) => {
const stepId = stepIdByDraftId.get(id);
return stepId === undefined ? [] : [stepId];
}),
),
].sort((a, b) => a - b);
}
export function buildTraceV3(input: {
registry: RecordingStateRegistry;
drafts: RecordingDraftStep[];
annotations?: StepAnnotation[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, {
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
for (const annotation of input.annotations ?? []) {
const bucket = annotationsByState.get(annotation.stateId) ?? [];
bucket.push(annotation);
annotationsByState.set(annotation.stateId, bucket);
}
const steps = reduced.steps.map((step) => ({
...step,
state: publishedId.get(step.state) ?? step.state,
result: { state: publishedId.get(step.result.state) ?? step.result.state },
}));
const states: TraceStateV3[] = entries.map((entry) => {
const id = publishedId.get(entry.id) ?? entry.id;
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body: formatTraceStateBody({
stateId: id,
url: entry.url,
title: entry.title,
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
vomText: entry.vomText,
annotations: annotationsByState.get(entry.id) ?? [],
stepIdByDraftId: reduced.stepIdByDraftId,
}),
...(entry.truncated ? { truncated: true } : {}),
};
});
return {
version: TRACE_VERSION_V3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
states,
steps,
};
}
@@ -1,30 +1,10 @@
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
// Drop bare character typing — FillSession already records the value.
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
import type { RecordingDraftStep } from "./types";
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
const out: RecordingDraftStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
if (step.pageUrl) urls.push(step.pageUrl);
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
}
const seen = new Set<string>();
const unique: string[] = [];
@@ -58,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
}
function buildPageRegistry(
steps: DraftTraceStep[],
steps: RecordingDraftStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
@@ -81,28 +61,28 @@ function pageIdFor(
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.pageUrl ?? step.url;
if (step.pageUrl) return step.pageUrl;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): Step["effect"] {
): StepV2["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: Step, effect: Step["effect"]): Step {
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
@@ -110,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
}
function toV2Step(
step: DraftTraceStep,
step: RecordingDraftStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): Step | null {
): StepV2 | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
@@ -129,28 +109,23 @@ function toV2Step(
to: step.url,
};
case "click":
if (!step.captureTarget) return null;
return withEffect(
{
op: "click",
id,
page,
target: step.target,
target: step.captureTarget,
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "hover":
return {
op: "hover",
id,
page,
target: step.target,
};
case "fill":
if (!step.captureTarget) return null;
return {
op: "fill",
id,
page,
target: step.target,
target: step.captureTarget,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
@@ -161,44 +136,51 @@ function toV2Step(
id,
page,
key: step.key,
...(step.target ? { target: step.target } : {}),
...(step.captureTarget ? { target: step.captureTarget } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "select":
if (!step.captureTarget) return null;
return withEffect(
{
op: "select",
id,
page,
target: step.target,
target: step.captureTarget,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
// step variant and fail to decode the whole result if we emit it.
case "hover":
case "scroll":
case "switch_tab":
return null;
}
}
export interface ReducedTrace {
pages: PageRef[];
steps: Step[];
interface ReducedTrace {
pages: PageRefV2[];
steps: StepV2[];
}
/**
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
*/
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: Step[] = [];
const out: StepV2[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
else if (draft.pageUrl) lastUrl = draft.pageUrl;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
@@ -207,18 +189,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
return { pages, steps: out };
}
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
function resolveTraceStartUrl(
drafts: RecordingDraftStep[],
startUrl?: string,
pages?: PageRef[],
pages?: PageRefV2[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
return step.op === "navigate";
});
if (navigate) return navigate.url;
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return pages?.[0]?.url ?? "about:blank";
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
}
export function buildTraceV2(input: {
steps: RecordingDraftStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}): TraceV2 {
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
pages,
steps,
};
}
@@ -0,0 +1,154 @@
import type { NavigationCause, StepV3 } from "@/transport/types";
import { shouldIncludeDraft } from "./draft-policy";
import { hasRedirectQualifier } from "./navigation-policy";
import { unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep } from "./types";
interface CollapsedDraft {
draft: RecordingDraftStep;
draftIds: number[];
}
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
return hasRedirectQualifier(step.transitionQualifiers);
}
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
const output: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const previous = output[output.length - 1];
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
previous.draft = {
...previous.draft,
url: step.url,
postStateId: step.postStateId ?? previous.draft.postStateId,
};
previous.draftIds.push(index + 1);
return;
}
output.push({ draft: { ...step }, draftIds: [index + 1] });
});
return output;
}
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
if (step.cause) return step.cause;
const qualifiers = step.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.includes("from_address_bar")) return "user_typed";
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
}
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
interface ReduceDraftOptions {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
if (draft.op === "switch_tab") {
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
return {
op: "switch_tab",
id,
state: draft.preStateId,
result: { state: draft.postStateId },
};
}
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
const common = { id, state, result: { state: resultState } };
switch (draft.op) {
case "navigate":
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
case "click":
return {
op: "click",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "hover":
return {
op: "hover",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
const fillIsRedacted = options.redactValues || draft.redacted === true;
return {
op: "fill",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
value: fillIsRedacted ? "***" : draft.value,
commit: draft.commit ?? "blur",
...(fillIsRedacted ? { redacted: true } : {}),
};
case "press":
return {
op: "press",
...common,
key: draft.key,
...(draft.captureTarget || draft.matchedTarget
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
: {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return {
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTraceV3 {
steps: StepV3[];
stepIdByDraftId: Map<number, number>;
}
export function reduceTraceStepsV3(
steps: RecordingDraftStep[],
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, {
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
}
return { steps: output, stepIdByDraftId };
}
@@ -0,0 +1,39 @@
import type { StepAnnotation } from "./types";
function annotationText(annotation: StepAnnotation, stepId: number): string {
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
}
export function formatTraceStateBody(input: {
stateId: string;
url: string;
title?: string;
stepIds: number[];
vomText: string;
annotations: StepAnnotation[];
stepIdByDraftId: Map<number, number>;
}): string {
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
lines.push(`url: ${JSON.stringify(input.url)}`);
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
lines.push("---");
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
for (const annotation of input.annotations) {
const stepId = input.stepIdByDraftId.get(annotation.draftId);
if (stepId === undefined) continue;
const bucket = byLine.get(annotation.line) ?? [];
bucket.push({ annotation, stepId });
byLine.set(annotation.line, bucket);
}
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
let line = bodyLine;
for (const item of byLine.get(lineIndex) ?? []) {
line += annotationText(item.annotation, item.stepId);
}
lines.push(line);
});
return `${lines.join("\n")}\n`;
}
+85
View File
@@ -0,0 +1,85 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type {
FillCommit,
KeyModifier,
NavigationCause,
StepV3,
TargetDescriptorV3,
} from "@/transport/types";
export interface TargetGeometry {
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
rect: { x: number; y: number; w: number; h: number };
tag: string;
}
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means top frame; null means the source Document could not be resolved. */
frameId?: string | null;
geometrySpace?: "top" | "local";
}
export interface StepAnnotation {
draftId: number;
op: StepV3["op"];
line: number;
stateId: string;
detail?: string;
}
interface DraftStateLink {
pageUrl?: string;
preStateId?: string;
postStateId?: string;
}
interface DraftTarget {
captureTarget?: CaptureTargetDescriptor;
targetHint?: TargetMatchHint;
matchedTarget?: TargetDescriptorV3;
}
interface DraftNavigationEffect {
navigatedTo?: string;
}
export type RecordingDraftStep =
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
| ({ op: "hover" } & DraftStateLink & DraftTarget)
| ({
op: "fill";
value: string;
commit?: FillCommit;
redacted?: boolean;
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "press";
key: string;
modifiers?: KeyModifier[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "select";
values: string[];
labels?: string[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({
op: "navigate";
url: string;
cause?: NavigationCause;
transitionType?: string;
transitionQualifiers?: string[];
} & DraftStateLink);
export type TargetedRecordingDraft = Extract<
RecordingDraftStep,
{ op: "click" | "hover" | "fill" | "press" | "select" }
>;
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
const manager = new SessionManager({
agentWindow: {
create: vi.fn(async () => nextWindowId++),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
remove,
},
});
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
const manager = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
remove: vi.fn(() => removeGate),
},
});
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await manager.start("aa11");
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const ctx = await manager.start("aa11");
@@ -87,6 +87,7 @@ describe("attachSessionEventHandler", () => {
windowEvents: events.api,
});
manager.forgetClosedTab(7, { isWindowClosing: true });
events.emit(4242);
for (let i = 0; i < 4; i += 1) await Promise.resolve();
@@ -113,7 +114,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const transport = fakeTransport();
@@ -129,7 +130,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const transport = fakeTransport();
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
import { SessionManager } from "../manager";
import { isAgentControlledTab, SessionManager } from "../manager";
function fakeAgentWindow(): AgentWindowApi & {
createMock: ReturnType<typeof vi.fn>;
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
return id;
});
const removeMock = vi.fn(async (_id: number) => {});
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
return {
create: createMock,
remove: removeMock,
@@ -151,6 +151,76 @@ describe("SessionManager", () => {
expect(sm.list()).toEqual([]);
});
describe("explicit tab claims", () => {
it("claims the exact home tab returned by AgentWindowApi", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
expect(isAgentControlledTab(ctx, 0)).toBe(true);
});
it("keeps every other tab free without classification state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
expect(isAgentControlledTab(ctx, 99)).toBe(false);
expect(ctx).not.toHaveProperty("userTabs");
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
});
it("treats a committed borrow as an explicit claim", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
expect(isAgentControlledTab(ctx, 42)).toBe(true);
});
it("forgets an agent-created tab after Chrome removes it", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(42);
sm.forgetClosedTab(42);
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
expect(isAgentControlledTab(ctx, 0)).toBe(true);
});
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const a = await sm.start("aa11");
const b = await sm.start("bb22");
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
sm.forgetClosedTab(42);
sm.forgetClosedTab(42);
sm.forgetClosedTab(999);
expect(isAgentControlledTab(a, 42)).toBe(false);
expect(sm.findBorrowingSession(42, null)).toBeNull();
expect([...a.borrowedTabs.keys()]).toEqual([43]);
expect([...b.borrowedTabs.keys()]).toEqual([44]);
expect(sm.list()).toHaveLength(2);
});
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
sm.forgetClosedTab(42);
expect(() =>
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
).toThrow(/reservation disappeared/);
reservation.release();
expect(sm.findBorrowingSession(42, null)).toBeNull();
expect(ctx.borrowedTabs.has(42)).toBe(false);
});
});
describe("findBorrowingSession", () => {
it("returns null when no session has borrowed the tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
@@ -14,8 +14,11 @@ export interface AgentWindowApi {
* Guarantee the Agent Window has an active, CDP-navigable tab.
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
* so sessions bootstrap with `about:blank` instead.
*
* Resolves with the id of the activated (or newly created) tab, so callers
* can track the session's home tab without re-querying Chrome.
*/
ensureActiveTab(windowId: number, url: string): Promise<void>;
ensureActiveTab(windowId: number, url: string): Promise<number>;
}
/** Creation hints for a new Agent Window. */
@@ -49,15 +52,19 @@ export const chromeAgentWindowApi: AgentWindowApi = {
// cancellation success while the Agent Window remains open.
await chrome.windows.remove(windowId);
},
async ensureActiveTab(windowId: number, url: string): Promise<void> {
async ensureActiveTab(windowId: number, url: string): Promise<number> {
const tabs = await chrome.tabs.query({ windowId });
const first = tabs.find((t) => typeof t.id === "number");
if (first?.id) {
if (first?.id !== undefined) {
if (!first.active) {
await chrome.tabs.update(first.id, { active: true });
}
return;
return first.id;
}
await chrome.tabs.create({ windowId, url, active: true });
const created = await chrome.tabs.create({ windowId, url, active: true });
if (typeof created?.id !== "number") {
throw new Error("[bh] chrome.tabs.create returned no tab id");
}
return created.id;
},
};
+29 -1
View File
@@ -6,9 +6,20 @@ export interface SessionContext {
agentWindowId: number;
refStore: RefStore;
borrowedTabs: Map<number, BorrowedTab>;
/**
* Tabs explicitly claimed by the agent because it created them. This
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
* Tabs opened through Chrome UI never enter this set.
*/
agentCreatedTabs: Set<number>;
createdAtMs: number;
}
/** Whether this session has explicitly claimed control of `tabId`. */
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
}
export interface BorrowedTab {
tabId: number;
originalWindowId: number;
@@ -106,6 +117,19 @@ export class SessionManager {
return Array.from(this.sessions.values());
}
/**
* Forget a tab Chrome has removed, including any uncommitted borrow.
* Whole-window closures keep committed borrows until the window-removed
* handler reports which user tabs could not be returned.
*/
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
this.borrowReservations.delete(tabId);
for (const ctx of this.sessions.values()) {
ctx.agentCreatedTabs.delete(tabId);
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
}
}
/**
* Look up whether `tabId` is currently borrowed by some *other*
* session than the one calling. Used by M8 `tab_borrow` to refuse
@@ -179,7 +203,7 @@ export class SessionManager {
const { signal: _signal, ...createOptions } = opts;
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
throwIfSessionStartAborted(opts.signal);
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
throwIfSessionStartAborted(opts.signal);
const ctx: SessionContext = {
@@ -187,6 +211,10 @@ export class SessionManager {
agentWindowId: windowId,
refStore: new RefStore(),
borrowedTabs: new Map(),
// The home tab is the session's first explicit claim. Every other
// tab remains free until `tab_create` or `tab_borrow` identifies it
// by its concrete Chrome tab id.
agentCreatedTabs: new Set([homeTabId]),
createdAtMs: this.now(),
};
this.sessions.set(sessionId, ctx);
+42 -22
View File
@@ -1,19 +1,11 @@
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
* Each fresh `tool.snapshot` resets the store: M6 will call
* `replace(...)` with the new ref → node address pairs. A node address
* includes the owning flat CDP session/frame when the element lives in
* an OOPIF; tools resolve live geometry from that identity at call time.
*
* Refs are session-scoped (§7): looking up a ref in the wrong session
* returns `null`, never silently leaks. Storing values in different
* sessions is fine; they live in independent `RefStore` instances
* inside the `SessionContext`.
*/
import type { VisualCandidate } from "@/tools/vom/visual-discovery";
/** Session-local refs describe the latest observation. Reusing eN does not identify
* which observation a caller read; generation is internal bookkeeping only. */
export type BackendNodeId = number;
export interface RefEntry {
export interface DomRefEntry {
readonly kind: "dom";
backendNodeId: BackendNodeId;
tabId: number | null;
frameId?: string;
@@ -21,7 +13,15 @@ export interface RefEntry {
generation: number;
}
export interface VisualRefInput {
readonly kind: "visual-region";
readonly candidate: VisualCandidate;
}
export type RefEntry = DomRefEntry | (VisualRefInput & { readonly generation: number });
export type RefInput =
| VisualRefInput
| BackendNodeId
| {
backendNodeId: BackendNodeId;
@@ -31,7 +31,7 @@ export type RefInput =
};
export class RefStore {
private readonly map = new Map<string, RefEntry>();
private map = new Map<string, RefEntry>();
private generation = 0;
size(): number {
@@ -44,7 +44,7 @@ export class RefStore {
resolve(ref: string, opts: { tabId?: number } = {}): BackendNodeId | null {
const entry = this.map.get(normaliseRef(ref));
if (!entry) return null;
if (!entry || entry.kind !== "dom") return null;
if (opts.tabId !== undefined && entry.tabId !== opts.tabId) return null;
return entry.backendNodeId;
}
@@ -58,9 +58,11 @@ export class RefStore {
* Used after every fresh `tool.snapshot`.
*/
replace(entries: Iterable<readonly [string, RefInput]>): void {
this.map.clear();
this.generation += 1;
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
const generation = this.generation + 1;
const next = new Map<string, RefEntry>();
for (const [ref, input] of entries) next.set(normaliseRef(ref), this.entry(input, generation));
this.map = next;
this.generation = generation;
}
set(
@@ -73,6 +75,7 @@ export class RefStore {
} = {},
): void {
this.map.set(normaliseRef(ref), {
kind: "dom",
backendNodeId: id,
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
@@ -89,20 +92,37 @@ export class RefStore {
return this.map.entries();
}
private entry(input: RefInput): RefEntry {
private entry(input: RefInput, generation: number): RefEntry {
if (typeof input !== "number" && "kind" in input) {
const { document } = input.candidate;
if (
!document?.attachmentId ||
!document.frameId ||
!Number.isSafeInteger(document.target?.tabId) ||
!Number.isSafeInteger(document.documentElementBackendNodeId) ||
document.documentElementBackendNodeId <= 0 ||
!Number.isSafeInteger(input.candidate.backendNodeId) ||
input.candidate.backendNodeId <= 0
)
throw new TypeError("visual ref requires a verified DOM identity and anchor");
// Preserve the read-only evidence and shared clipping chain; do not clone ancestors per ref.
return { kind: "visual-region", candidate: input.candidate, generation };
}
if (typeof input === "number") {
return {
kind: "dom",
backendNodeId: input,
tabId: null,
generation: this.generation,
generation,
};
}
return {
kind: "dom",
backendNodeId: input.backendNodeId,
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
generation: this.generation,
generation,
};
}
}
@@ -0,0 +1,22 @@
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
const RECORDING_DOCUMENT_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
export function recordingDocumentMarkerValue(identity: string): string {
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
throw new TypeError("recording Document identity must be a random UUID");
}
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
}
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
export function readRecordingDocumentIdentity(
attrs: Readonly<Record<string, string>>,
): string | undefined {
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
}
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
expect(message.requestId.length).toBeGreaterThan(0);
});
it("proactively focuses the user window before sending the overlay message", async () => {
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
windows.getLastFocused.mockResolvedValueOnce(
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
);
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
const pending = requestBorrowConfirmation(42, {
deps: { tabs, windows, notifications },
});
await vi.runAllTimersAsync();
await pending;
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
});
it("still proceeds when proactive window focus fails", async () => {
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
windows.getLastFocused.mockResolvedValueOnce(
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
);
windows.update.mockRejectedValueOnce(new Error("window gone"));
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
const pending = requestBorrowConfirmation(42, {
deps: { tabs, windows, notifications },
});
await vi.runAllTimersAsync();
expect(await pending).toBe(true);
});
it("sends confirmation to the borrow target when it is the active tab", async () => {
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
windows.getLastFocused.mockResolvedValueOnce(
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -8,6 +8,7 @@ import type {
RequestFrame,
} from "@/transport/types";
import { ToolDispatcher } from "../dispatcher";
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
@@ -44,16 +45,120 @@ function makeRequest(method: string, params: unknown): RequestFrame {
describe("ToolDispatcher", () => {
afterEach(() => {
resetBrowserObservationForTests();
vi.unstubAllGlobals();
});
it.each([
"tool.snapshot",
"tool.evaluate",
"tool.get_html",
"tool.console",
])("annotates Chrome extension-access failures from %s", async (method) => {
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(),
},
});
await sessions.start("test");
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
const cdp = {
send: vi.fn().mockRejectedValue(error),
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
consoleEntriesSince: vi.fn(),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
dispatcher.start();
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({
error: {
code: "cdp_failed",
data: { reason: "cdp_extension_access_denied" },
},
});
dispatcher.stop();
});
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
const tab = {
id: 7,
windowId: 4242,
active: true,
status: "complete",
url: "https://example.com/start",
} as chrome.tabs.Tab;
const frameCoordinator = {
begin: vi.fn(),
armTab: vi.fn(async () => true),
sourceFor: vi.fn(() => null),
stop: vi.fn(async () => true),
cancel: vi.fn(),
};
const sendToTab = vi.fn(async () => ({ ok: true }));
setBrowserObservationAttachForTests(
() => () => {},
() => () => {},
);
const dispatcher = new ToolDispatcher({
transport,
sessions,
recording: {
tabsApi: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
frameCoordinator,
sendToTab,
},
});
dispatcher.start();
deliver(
makeRequest("tool.record_start", {
session_id: "aa11",
url: "https://example.com/start",
}),
);
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
});
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -76,7 +181,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create,
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -94,7 +199,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -108,6 +213,47 @@ describe("ToolDispatcher", () => {
expect(sent[0]).toEqual({ id: "r-1", result: {} });
});
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
// Regression guard for the deps that session_stop reads directly: when
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
// the agent-tab cleanup and the release decision silently no-op and the
// Agent Window is closed even though a user tab is still open (issue #57).
const removeTab = vi.fn(async () => {});
const closeWindow = vi.fn(async () => {});
// After cleanup the window still holds one tab (id 99) that is neither
// the home tab nor an agent-created tab — i.e. a genuine user tab.
vi.stubGlobal("chrome", {
tabs: {
remove: removeTab,
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: closeWindow,
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
// Register the exact tab id returned by the agent's creation path.
sessions.get("aa11")?.agentCreatedTabs.add(7);
const dispatcher = new ToolDispatcher({ transport, sessions });
dispatcher.start();
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
await flushMicrotasks();
// The agent tab is closed through the real chrome.tabs surface...
expect(removeTab).toHaveBeenCalledWith(7);
// ...and the window is released rather than closed, so the user's tab survives.
expect(closeWindow).not.toHaveBeenCalled();
expect(sessions.has("aa11")).toBe(false);
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
});
it("routes tool.console through the CDP console buffer", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -119,7 +265,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -168,13 +314,114 @@ describe("ToolDispatcher", () => {
});
});
it("bypasses and restores the control overlay for an upload trigger click", async () => {
const sendMessage = vi.fn(async () => undefined);
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })),
query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]),
sendMessage,
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 123, { tabId: 7 });
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T;
}
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
}
if (method === "DOM.resolveNode") {
return { object: { objectId: "trigger-object" } } as T;
}
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
if (method === "Runtime.callFunctionOn") {
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
if (declaration.includes("count: state.inputs.length")) {
return { result: { value: { count: 1, multiple: false } } } as T;
}
if (declaration.includes("inputs[0]")) {
return { result: { objectId: "input-object" } } as T;
}
return { result: { value: true } } as T;
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string }).expression ?? "";
if (expression.includes("overlayDetails")) {
return { result: { value: { hitIndex: 0 } } } as T;
}
if (expression.includes("overlayHostPresent")) {
return {
result: {
value: { overlayHostPresent: true, overlayHostConnected: true },
},
} as T;
}
if (expression.includes("count:")) {
return { result: { value: { count: 1, multiple: false } } } as T;
}
if (expression.includes("?.inputs[0]")) {
return { result: { objectId: "input-object" } } as T;
}
return { result: { value: true } } as T;
}
return {} as T;
});
const cdp = {
send,
detachSession: vi.fn(async () => {}),
ensureNetworkCapture: vi.fn(async () => {}),
networkEntriesSince: vi.fn(() => ({
tab_id: 7,
entries: [],
next_since: 0,
truncated: false,
})),
setDeviceMetricsOverride: vi.fn(async () => {}),
clearDeviceMetricsOverride: vi.fn(async () => {}),
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
dispatcher.start();
deliver(
makeRequest("tool.upload", {
session_id: "aa11",
ref: "@e1",
files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }],
}),
);
await flushMicrotasks();
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
type: "bh-automation-bypass",
enabled: true,
});
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
type: "bh-automation-bypass",
enabled: false,
});
});
it("detaches CDP state before stopping a session", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -210,7 +457,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove,
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -258,7 +505,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -278,7 +525,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -302,7 +549,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -319,7 +566,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -336,7 +583,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const onSessionsChanged = vi.fn();
@@ -352,17 +599,106 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
});
it.each([
"focus",
"blur",
"scroll_to",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
sendMessage: vi.fn(async () => undefined),
},
});
const sessions = new SessionManager({
agentWindow: {
create: async () => 4242,
remove: async () => {},
ensureActiveTab: async () => 7,
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 7 });
const { transport, sent, deliver } = fakeTransport();
const onBrowserControlResumed = vi.fn();
let resolveNode: ((value: object) => void) | undefined;
const cdp = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "DOM.resolveNode")
return new Promise((resolve) => {
resolveNode = resolve;
});
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
return {};
}),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed });
const hover = dispatcher as unknown as {
rememberHover: (sessionId: string, result: object) => void;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await hover.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" }));
await vi.waitFor(() => expect(resolveNode).toBeDefined());
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } });
resolveNode!({ object: { objectId: "focus-target" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
if (action === "scroll_to") {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", {
objectGroup: expect.any(String),
});
} else {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", {
objectId: "focus-target",
});
}
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(dispatcher.inflightAbortControllers.size).toBe(0);
dispatcher.stop();
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", {
tabs: {
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
id: 7,
windowId: props.windowId,
url: props.url,
active: props.active,
})),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const onBrowserControlResumed = vi.fn();
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
const onAgentTabClaimed = vi.fn();
const dispatcher = new ToolDispatcher({
transport,
sessions,
onBrowserControlResumed,
onAgentTabClaimed,
});
dispatcher.start();
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
@@ -379,6 +715,7 @@ describe("ToolDispatcher", () => {
});
await flushMicrotasks();
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
});
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
@@ -392,7 +729,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const order: string[] = [];
@@ -474,7 +811,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const cdp = {
@@ -524,6 +861,135 @@ describe("ToolDispatcher", () => {
);
});
it.each([
{ tab_id: 7, code: "not_found" },
{ tab_id: undefined, code: "invalid_params" },
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
const sendMessage = vi.fn(async () => undefined);
vi.stubGlobal("chrome", {
tabs: {
sendMessage,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
releaseSessionTab: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp: cdp as unknown as TestDispatcherCdp,
});
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await helpers.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ error: { code } });
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 10,
y: 20,
});
dispatcher.stop();
});
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
const sendMessage = vi.fn(async () => undefined);
const move = vi.fn(async () => {
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
return { id: 7, windowId: 200, index: 4 };
});
vi.stubGlobal("chrome", {
tabs: { sendMessage, move },
windows: { get: vi.fn(async () => ({ id: 200 })) },
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const ctx = await sessions.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
releaseSessionTab: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp: cdp as unknown as TestDispatcherCdp,
});
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
for (const [sessionId, tabId] of [
["aa11", 7],
["aa11", 8],
["bb22", 9],
] as const) {
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
await helpers.setHoverBypass(sessionId, tabId, true);
}
dispatcher.start();
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
expect(cdp.send).not.toHaveBeenCalled();
for (const [sessionId, tabId] of [
["aa11", 8],
["bb22", 9],
] as const) {
expect(sendMessage).not.toHaveBeenCalledWith(
tabId,
expect.objectContaining({ enabled: false }),
);
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 10,
y: 20,
});
}
dispatcher.stop();
});
it("transfers hover overlay bypass ownership between sessions", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -535,7 +1001,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -572,7 +1038,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -624,7 +1090,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -658,7 +1124,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
transport.send = () => {
@@ -680,7 +1146,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 5555),
remove,
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
transport.send = () => {
@@ -708,7 +1174,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(() => createPromise),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -759,7 +1225,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -780,7 +1246,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(() => createPromise),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import type { RpcError } from "@/transport/types";
import { cdpError, classifyCdpError } from "../errors";
const denied = "Cannot access a chrome-extension:// URL of different extension";
describe("CDP extension access errors", () => {
it("classifies both Chrome Error instances and string failures", () => {
for (const error of [new Error(denied), denied]) {
expect(cdpError(error)).toEqual({
code: "cdp_failed",
message: denied,
data: { reason: "cdp_extension_access_denied" },
});
}
});
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
const error: RpcError = {
code: "cdp_failed",
message: denied,
data: {
reason: "transfer_outcome_unknown",
effect_state: "unknown",
cleanup_state: "failed",
},
};
expect(classifyCdpError(error)).toBe(error);
});
it("leaves other error codes and unrelated debugger failures unchanged", () => {
for (const error of [
{ code: "permission_denied", message: denied },
{ code: "cdp_failed", message: "Another debugger is already attached" },
] satisfies RpcError[]) {
expect(classifyCdpError(error)).toBe(error);
}
});
});
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -0,0 +1,990 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { type DownloadsApi, handleDownload } from "../download";
import { captureBrowserDownload } from "../download-capture";
import { uploadThroughFileDrop } from "../file-drop-transaction";
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
import type { ResolvedActionTarget } from "../interaction";
import type { CdpRunner } from "../shared";
import { handleUpload } from "../upload";
function sessions() {
return new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
}
function tabsApi() {
return {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
}
function fakeEvent<T extends (...args: never[]) => unknown>() {
const listeners = new Set<T>();
return {
addListener: (listener: T) => listeners.add(listener),
removeListener: (listener: T) => listeners.delete(listener),
emit: (...args: Parameters<T>) => {
for (const listener of listeners) listener(...args);
},
};
}
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
return {
tab: { tabId: 4, windowId: 100, active: true },
backendNodeId: 123,
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
...(frameId ? { frameId } : {}),
usedRef: "e3",
};
}
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
return operation();
}
function uploadCdp(
options: {
inputCount?: number;
multiple?: boolean;
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
pendingResolve?: boolean;
} = {},
) {
const calls: Array<{ method: string; params?: object }> = [];
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.setInterceptFileChooserDialog") return {};
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
if (method === "DOM.resolveNode") {
if (options.pendingResolve) return new Promise<never>(() => {});
return { object: { objectId: "trigger-object" } };
}
if (method === "Runtime.callFunctionOn") {
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
if (declaration.includes("count: state.inputs.length")) {
return {
result: {
value: {
count: options.inputCount ?? 1,
multiple: options.multiple ?? true,
},
},
};
}
if (declaration.includes("inputs[0]")) {
return { result: { objectId: "input-object" } };
}
return { result: { value: true } };
}
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
if (
method === "Input.dispatchMouseEvent" &&
(params as { type?: string }).type === "mousePressed" &&
options.chooser
) {
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
}
return {};
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
return {
calls,
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
cdp,
};
}
function dropCdp(
options: {
targetContainsPoint?: boolean;
dispatchError?: (type: string) => Error | Promise<never> | undefined;
onCall?: (method: string) => void;
} = {},
) {
const calls: Array<{ method: string; params?: object }> = [];
const detach = vi.fn(async () => {});
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
options.onCall?.(method);
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
if (method === "DOM.resolveNode") {
const id = (params as { backendNodeId?: number }).backendNodeId;
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
}
if (method === "Runtime.callFunctionOn") {
return { result: { value: options.targetContainsPoint ?? true } };
}
if (method === "Input.dispatchDragEvent") {
const type = (params as { type?: string }).type ?? "";
const error = options.dispatchError?.(type);
if (error instanceof Promise) return error;
if (error) throw error;
return {};
}
if (method === "Runtime.releaseObjectGroup") return {};
throw new Error(`unexpected drop CDP command ${method}`);
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
detach,
};
return { calls, cdp, detach };
}
describe("file transfer tools", () => {
it("captures the file input activated by the requested click and injects only staged paths", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const { cdp, calls } = uploadCdp();
const result = await handleUpload(
manager,
{
session_id: "s1",
ref: "@e3",
files: [
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
],
},
{ cdp, tabsApi: tabsApi() },
);
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
expect(calls[0]).toEqual({
method: "Page.setInterceptFileChooserDialog",
params: { enabled: true },
});
expect(calls).toContainEqual({
method: "Page.setInterceptFileChooserDialog",
params: { enabled: false, cancel: true },
});
expect(calls).toContainEqual({
method: "DOM.setFileInputFiles",
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
});
});
it("fails immediately when the trigger does not activate a file input", async () => {
const { cdp } = uploadCdp({ inputCount: 0 });
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
});
expect(result).toMatchObject({
code: "unsupported",
data: { reason: "file_input_not_activated", phase: "resolve_input" },
});
});
it("fails before clicking when chooser interception is unavailable", async () => {
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.setInterceptFileChooserDialog") {
throw new Error("method unavailable");
}
return {};
}) as CdpRunner["send"],
};
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger,
});
expect(result).toMatchObject({
code: "cdp_failed",
data: { effect_state: "none", phase: "arm_interception" },
});
expect(trigger).not.toHaveBeenCalled();
});
it("uses an exact chooser event as an independent input-location signal", async () => {
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget("f1"),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => {
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({ multiple: false });
expect(calls).toContainEqual({
method: "DOM.setFileInputFiles",
params: { files: ["/private/stage/one"], backendNodeId: 789 },
});
});
it("reports a File System Access picker without waiting for a timeout", async () => {
const { cdp, emitChooser } = uploadCdp({
inputCount: 0,
});
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget("f1"),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => {
emitChooser({ frameId: "f1", mode: "selectSingle" });
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({
code: "unsupported",
message: "upload trigger invoked a non-input file picker",
data: { reason: "file_input_not_activated", phase: "resolve_input" },
});
});
it("bounds a stuck file-input probe", async () => {
const { cdp } = uploadCdp({ pendingResolve: true });
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 5,
trigger: vi.fn(),
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_input_probe_failed", phase: "arm_input_probe" },
});
});
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
const fixture = uploadCdp();
const originalSend = fixture.cdp.send;
const detach = vi.fn(async () => {});
fixture.cdp.detach = detach;
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
return originalSend(tabId, method, params);
}) as CdpRunner["send"];
const result = await uploadThroughActivatedFileInput({
cdp: fixture.cdp,
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 10,
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
});
expect(result).toMatchObject({
code: "timeout",
data: { effect_state: "unknown", phase: "set_files" },
});
expect(detach).toHaveBeenCalledWith(4);
});
it("delivers staged files through one explicit native drop without clicking", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const lifecycle: string[] = [];
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
const overlayPhases: string[] = [];
const result = await handleUpload(
manager,
{
session_id: "s1",
ref: "@e3",
mode: "drop",
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
},
{
cdp,
tabsApi: tabsApi(),
sendToTab: vi.fn(async (_tabId, message) => {
overlayPhases.push(message.phase);
lifecycle.push(`overlay:${message.phase}`);
}),
},
);
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
{
method: "Input.dispatchDragEvent",
params: {
type: "dragEnter",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
{
method: "Input.dispatchDragEvent",
params: {
type: "dragOver",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
{
method: "Input.dispatchDragEvent",
params: {
type: "drop",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
]);
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
expect(overlayPhases).toEqual(["begin", "end"]);
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
lifecycle.indexOf("overlay:begin"),
);
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
lifecycle.indexOf("DOM.getNodeForLocation"),
);
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
lifecycle.indexOf("overlay:end"),
);
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
lifecycle.indexOf("Runtime.releaseObjectGroup"),
);
});
it("fails closed before dragging when another element owns the action point", async () => {
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "permission_denied",
data: {
reason: "file_drop_target_unavailable",
mechanism: "drop",
effect_state: "none",
phase: "resolve_target",
},
});
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
});
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
const rootCalls: Array<{ method: string; params?: object }> = [];
const childCalls: Array<{ method: string; params?: object }> = [];
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
rootCalls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method, params) => {
childCalls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "Runtime.evaluate")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
}
if (method === "DOM.getNodeForLocation") {
return { backendNodeId: 124, frameId: "child" };
}
if (method === "DOM.resolveNode") {
const id = (params as { backendNodeId?: number }).backendNodeId;
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
}
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
return {};
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget("child", "child-session"),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({ x: 110, y: 70 });
expect(
childCalls
.filter((call) => call.method === "Input.dispatchDragEvent")
.map((call) => call.params),
).toEqual([
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
]);
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
});
it("reports a missing native drag command as unsupported before commit", async () => {
const { cdp } = dropCdp({
dispatchError: (type) =>
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "unsupported",
data: {
reason: "upload_mechanism_unsupported",
effect_state: "none",
phase: "drag_enter",
},
});
});
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
const { cdp, calls } = dropCdp({
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 10,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
});
expect(calls).toContainEqual({
method: "Input.dispatchDragEvent",
params: expect.objectContaining({ type: "dragCancel" }),
});
});
it("marks an unresolved drop unknown and detaches browser state", async () => {
const { cdp, detach } = dropCdp({
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 10,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
});
expect(detach).toHaveBeenCalledWith(4);
});
it("routes one exact-target download through a browser-relative capability", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const initial = {
id: 9,
url: "https://example.test/result.zip",
finalUrl: "https://example.test/result.zip",
filename: "result.zip",
state: "in_progress",
fileSize: -1,
totalBytes: 12,
mime: "application/zip",
danger: "safe",
} as chrome.downloads.DownloadItem;
const completed = {
...initial,
filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip",
state: "complete",
fileSize: 12,
} as chrome.downloads.DownloadItem;
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => [completed]),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
if (
method === "Input.dispatchMouseEvent" &&
(params as { type?: string }).type === "mousePressed"
) {
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: initial.url,
suggestedFilename: "result.zip",
});
await new Promise<void>((resolve) => {
onDeterminingFilename.emit(initial, (value) => {
suggested = value;
resolve();
});
});
onCreated.emit(completed);
}
return {};
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
const result = await handleDownload(
manager,
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
{ cdp, tabsApi: tabsApi(), downloads },
);
expect(suggested).toEqual({
filename: "BrowserSkill/tr_1/result.zip",
conflictAction: "overwrite",
});
expect(result).toMatchObject({
tab_id: 4,
suggested_filename: "result.zip",
byte_size: 12,
browser_path: completed.filename,
});
});
it("does not claim a download without an intent from the exact target", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => []),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as unknown as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
const unrelated = {
id: 17,
url: "https://example.test/unrelated.zip",
finalUrl: "https://example.test/unrelated.zip",
filename: "unrelated.zip",
state: "in_progress",
} as chrome.downloads.DownloadItem;
let defaultSuggestionCalled = false;
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4, sessionId: "expected-child" },
downloads,
browserRelativeDir: "BrowserSkill/tr_1",
timeoutMs: 5,
trigger: async () => {
cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", {
url: unrelated.url,
suggestedFilename: unrelated.filename,
});
onDeterminingFilename.emit(unrelated, (suggestion) => {
defaultSuggestionCalled = suggestion === undefined;
});
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(defaultSuggestionCalled).toBe(true);
expect(downloads.cancel).not.toHaveBeenCalled();
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "download_capture_failed" },
});
});
it("correlates a filename candidate that arrives before the CDP intent", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const initial = {
id: 21,
url: "https://example.test/candidate-first.bin",
finalUrl: "https://example.test/candidate-first.bin",
filename: "candidate-first.bin",
state: "in_progress",
fileSize: -1,
totalBytes: 4,
bytesReceived: 0,
} as chrome.downloads.DownloadItem;
const complete = {
...initial,
state: "complete",
fileSize: 4,
} as chrome.downloads.DownloadItem;
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => [complete]),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4 },
downloads,
browserRelativeDir: "BrowserSkill/tr_21",
timeoutMs: 1_000,
trigger: async () => {
const suggested = new Promise<void>((resolve) => {
onDeterminingFilename.emit(initial, (value) => {
suggestion = value;
resolve();
});
});
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: initial.url,
suggestedFilename: initial.filename,
});
await suggested;
onCreated.emit(complete);
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(suggestion).toEqual({
filename: "BrowserSkill/tr_21/candidate-first.bin",
conflictAction: "overwrite",
});
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
});
it("removes a completed download when the final size exceeds the limit", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const initial = {
id: 22,
url: "https://example.test/oversized.bin",
finalUrl: "https://example.test/oversized.bin",
filename: "oversized.bin",
state: "in_progress",
fileSize: -1,
totalBytes: -1,
bytesReceived: 0,
} as chrome.downloads.DownloadItem;
const complete = {
...initial,
filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin",
state: "complete",
fileSize: 8,
totalBytes: 8,
bytesReceived: 8,
} as chrome.downloads.DownloadItem;
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => [complete]),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4 },
downloads,
browserRelativeDir: "BrowserSkill/tr_22",
maxByteSize: 4,
timeoutMs: 1_000,
trigger: async () => {
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: initial.url,
suggestedFilename: initial.filename,
});
await new Promise<void>((resolve) => {
onDeterminingFilename.emit(initial, () => resolve());
});
onCreated.emit(initial);
onChanged.emit({ id: initial.id, state: { current: "complete" } });
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "download_capture_failed", effect_state: "committed" },
});
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
expect(downloads.cancel).not.toHaveBeenCalled();
});
it("reconciles a download that completes while cancellation is being requested", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const initial = {
id: 23,
url: "https://example.test/racing.bin",
finalUrl: "https://example.test/racing.bin",
filename: "racing.bin",
state: "in_progress",
fileSize: -1,
totalBytes: -1,
bytesReceived: 0,
} as chrome.downloads.DownloadItem;
const complete = {
...initial,
filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin",
state: "complete",
fileSize: 4,
totalBytes: 4,
bytesReceived: 4,
} as chrome.downloads.DownloadItem;
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
cancel: vi.fn(async () => {
throw new Error("download already complete");
}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4 },
downloads,
browserRelativeDir: "BrowserSkill/tr_23",
timeoutMs: 80,
trigger: async () => {
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: initial.url,
suggestedFilename: initial.filename,
});
await new Promise<void>((resolve) => {
onDeterminingFilename.emit(initial, () => resolve());
});
onCreated.emit(initial);
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "download_capture_failed", effect_state: "committed" },
});
expect(downloads.cancel).toHaveBeenCalledWith(initial.id);
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } });
});
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => []),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
const defaults: number[] = [];
const candidate = (id: number) =>
({
id,
url: "https://example.test/same.bin",
finalUrl: "https://example.test/same.bin",
filename: "same.bin",
state: "in_progress",
}) as chrome.downloads.DownloadItem;
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4 },
downloads,
browserRelativeDir: "BrowserSkill/tr_ambiguous",
timeoutMs: 100,
trigger: async () => {
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: "https://example.test/same.bin",
suggestedFilename: "same.bin",
});
for (const id of [31, 32]) {
onDeterminingFilename.emit(candidate(id), (value) => {
if (value === undefined) defaults.push(id);
});
}
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(defaults.sort()).toEqual([31, 32]);
expect(downloads.cancel).not.toHaveBeenCalled();
expect(result).toMatchObject({
code: "cdp_failed",
data: { effect_state: "unknown", phase: "attribution" },
});
});
});
@@ -0,0 +1,573 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import { handleFill } from "../interaction";
interface ScriptParams {
functionDeclaration: string;
arguments?: Array<{ value: unknown }>;
}
async function setup(markup = '<input value="old">') {
document.body.innerHTML = markup;
const element = document.body.firstElementChild as HTMLInputElement;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 4 });
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return {
result: {
value: await fn.apply(
element,
(params.arguments ?? []).map((arg) => arg.value),
),
},
};
} catch {
return { exceptionDetails: { text: "test page script failed" } };
}
});
// Number/email controls have a browser caret even though their selection
// getters return null. Model that caret instead of assuming it is at the end.
const hiddenCaret = { start: 0, end: 0 };
const moveToEnd = vi.fn(async () => {
if (typeof element.selectionStart === "number") {
element.setSelectionRange(element.value.length, element.value.length);
} else {
hiddenCaret.start = hiddenCaret.end = element.value.length;
}
});
const insert = vi.fn(async (text: string) => {
const focused = document.activeElement as HTMLInputElement;
if (focused.readOnly || focused.disabled) return;
const start = focused.selectionStart ?? hiddenCaret.start;
const end = focused.selectionEnd ?? hiddenCaret.end;
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
focused.dispatchEvent(new Event("input", { bubbles: true }));
});
const release = vi.fn(async () => ({}));
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
switch (method) {
case "DOM.describeNode":
return {
node: {
nodeName: element.tagName,
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
},
};
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.focus":
element.focus();
return {};
case "DOM.resolveNode":
return { object: { objectId: "fill-target" } };
case "Runtime.callFunctionOn":
return script(params as ScriptParams);
case "Input.insertText":
await insert((params as { text: string }).text);
return {};
case "Input.dispatchKeyEvent":
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
await moveToEnd();
}
return {};
case "Runtime.releaseObject":
return release();
default:
throw new Error(`unexpected ${method}`);
}
});
return {
element,
hiddenCaret,
moveToEnd,
script,
insert,
release,
send,
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
handleFill(
manager,
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
{
cdp: { send: send as CdpRunner["send"] },
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
signal,
},
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("fill result verification", () => {
it.each([
"readonly",
"disabled",
])("preserves a %s field before rejecting it", async (attribute) => {
const h = await setup(`<input ${attribute} value="old">`);
expect(await h.fill()).toMatchObject({
code: "invalid_params",
data: { reason: "target_not_fillable" },
});
expect(h.element.value).toBe("old");
expect(h.insert).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"checkbox",
"radio",
"range",
"date",
"file",
])("rejects an unsupported %s input before clearing", async (type) => {
const h = await setup(`<input type="${type}">`);
const before = h.element.value;
expect(await h.fill()).toMatchObject({
code: "invalid_params",
data: { reason: "target_not_fillable" },
});
expect(h.element.value).toBe(before);
expect(h.insert).not.toHaveBeenCalled();
});
it("rejects an overlong value before clearing", async () => {
const h = await setup('<input maxlength="3" value="old">');
expect(await h.fill()).toMatchObject({
code: "invalid_params",
message: /maxlength/,
data: { reason: "fill_value_invalid" },
});
expect(h.element.value).toBe("old");
expect(h.insert).not.toHaveBeenCalled();
});
it.each([
true,
false,
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
const send = h.send.getMockImplementation()!;
h.send.mockImplementation(async (tabId, method, params) => {
if (method === "DOM.focus") throw new Error("Element is not focusable");
return send(tabId, method, params);
});
expect(await h.fill()).toMatchObject({
code: disabled ? "invalid_params" : "cdp_failed",
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
});
expect(h.element.value).toBe("old");
expect(h.insert).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it("fills a background document without requiring foreground browser focus", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.fill()).toMatchObject({ value_length: 5 });
expect(h.element.value).toBe("hello");
});
it("restores focus once after clearing without typing into another field", async () => {
const h = await setup();
const other = document.createElement("input");
document.body.append(other);
h.element.addEventListener("input", () => other.focus());
expect(await h.fill()).toMatchObject({ value_length: 5 });
expect(h.element.value).toBe("hello");
expect(other.value).toBe("");
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
});
it.each([
"focus",
"value",
])("rechecks %s after restoring focus, without retrying again", async (action) => {
const h = await setup();
const other = document.createElement("input");
document.body.append(other);
h.element.addEventListener(
"input",
() => {
other.focus();
h.element.addEventListener("focus", () => {
if (action === "focus") queueMicrotask(() => other.focus());
else h.element.value = "changed";
});
},
{ once: true },
);
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
});
expect(other.value).toBe("");
expect(h.insert).not.toHaveBeenCalled();
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
});
it.each([
"remove",
"readonly",
"restore",
"microtask-restore",
])("stops when clearing causes the page to %s the target", async (action) => {
const h = await setup();
h.element.addEventListener("input", () => {
if (action === "remove") h.element.remove();
if (action === "readonly") h.element.readOnly = true;
if (action === "restore") h.element.value = "restored";
if (action === "microtask-restore")
queueMicrotask(() => {
h.element.value = "restored";
});
});
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_target_changed" },
});
expect(h.insert).not.toHaveBeenCalled();
});
it("fails when insertText is acknowledged without changing the value", async () => {
const h = await setup();
h.insert.mockImplementation(async () => {});
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.release).toHaveBeenCalledOnce();
});
it("detects truncation when maxlength changes after preparation", async () => {
const h = await setup();
h.element.addEventListener(
"input",
() => {
h.element.maxLength = 3;
},
{ once: true },
);
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.element.value).toBe("hel");
});
it.each([
"input",
"change",
"microtask-change",
])("detects a value reverted by the %s handler", async (event) => {
const h = await setup();
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
if (event === "microtask-change")
queueMicrotask(() => {
h.element.value = "";
});
else h.element.value = "";
});
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
});
it("compares content rather than length without including field values in the error", async () => {
const h = await setup('<input type="password">');
h.element.addEventListener("change", () => {
h.element.value = "other-secret";
});
const result = await h.fill("given-secret");
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
});
it("does not report success for a target replaced by a change handler", async () => {
const h = await setup();
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_target_changed" },
});
});
it.each([
1, 2, 3, 4, 5,
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
const run = h.script.getMockImplementation()!;
let count = 0;
h.script.mockImplementation(async (params) =>
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
);
const result = await h.fill();
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
expect(JSON.stringify(result)).not.toContain("page secret");
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
const run = h.script.getMockImplementation()!;
let count = 0;
h.script.mockImplementation(async (params) =>
++count === phase ? { result: { type: "undefined" } } : run(params),
);
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
});
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
const h = await setup();
h.element.addEventListener(event, () => {
queueMicrotask(() =>
queueMicrotask(() => {
h.element.value = "reverted";
}),
);
});
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
});
it.each([
"<input>",
'<input type="password">',
"<textarea></textarea>",
])("verifies successful text in %s", async (markup) => {
const h = await setup(markup);
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
expect(h.element.value).toBe("你好🙂");
expect(h.release).toHaveBeenCalledOnce();
});
it("verifies append against the old value plus the requested text", async () => {
const h = await setup();
h.element.focus();
h.element.setSelectionRange(3, 3);
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
expect(h.element.value).toBe("old!");
});
it.each([
["<textarea></textarea>", "a\r\nb", "a\nb"],
["<input>", "a\nb", "a b"],
["<input>", "a\r\nb", "a b"],
[
'<input type="email" multiple>',
" a@example.com , b@example.com ",
"a@example.com,b@example.com",
],
])("accepts native normalization for %s", async (markup, requested, normalized) => {
if (markup.startsWith("<textarea")) {
// Happy DOM does not implement Chrome's textarea line-ending normalization.
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
this: HTMLTextAreaElement,
value,
) {
setter.call(this, value.replace(/\r\n?/g, "\n"));
});
}
const h = await setup(markup);
h.insert.mockImplementation(async () => {
h.element.value = normalized;
});
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
});
it("does not count an editable padding break as an extra typed character", async () => {
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
// These two layout/editability properties are missing in Happy DOM.
let innerText = "";
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
Object.defineProperty(h.element, "innerText", { get: () => innerText });
h.insert.mockImplementation(async () => {
h.element.innerHTML = "one<div><br></div>";
innerText = "one\n\n";
});
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
});
it.each([
[0, 0],
[1, 1],
[0, 3],
])("appends after moving selection %s:%s to the end", async (start, end) => {
const h = await setup();
h.element.focus();
h.element.setSelectionRange(start, end);
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
expect(h.element.value).toBe("old🙂");
});
it.each(
["number", "email"].flatMap((type) =>
[
[0, 0],
[1, 1],
[0, 2],
].map(([start, end]) => ({ type, start, end })),
),
)("appends to $type from selection $start:$end without using its unsupported API", async ({
type,
start,
end,
}) => {
const h = await setup(`<input type="${type}" value="12">`);
Object.assign(h.hiddenCaret, { start, end });
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
throw new DOMException("unsupported", "InvalidStateError");
});
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
expect(h.element.value).toBe("123");
expect(selection).not.toHaveBeenCalled();
});
it.each([
"number",
"email",
])("does not report successful %s append when the caret command is ignored", async (type) => {
const h = await setup(`<input type="${type}" value="12">`);
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
h.moveToEnd.mockResolvedValueOnce(undefined);
expect(await h.fill("3", false)).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.element.value).toBe("312");
expect(h.insert).toHaveBeenCalledOnce();
});
it("rejects invalid number text before clearing", async () => {
const h = await setup('<input type="number" value="12">');
expect(await h.fill("abc")).toMatchObject({
code: "invalid_params",
data: { reason: "fill_value_invalid" },
});
expect(h.element.value).toBe("12");
expect(h.insert).not.toHaveBeenCalled();
});
it("reports page formatting as unconfirmed without retrying the write", async () => {
const h = await setup();
h.element.addEventListener("change", () => {
h.element.value = h.element.value.toUpperCase();
});
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.element.value).toBe("HELLO");
expect(h.insert).toHaveBeenCalledOnce();
});
it.each([
["focus", false],
["focus", true],
["value", false],
["value", true],
["cancel", false],
["cancel", true],
["throw", false],
["throw", true],
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
const controller = new AbortController();
const other = document.createElement("input");
document.body.append(other);
const send = h.send.getMockImplementation()!;
h.send.mockImplementation(async (tabId, method, params) => {
const reply = await send(tabId, method, params);
if (
method === "Input.dispatchKeyEvent" &&
(params as { type: string }).type === "rawKeyDown"
) {
if (action === "focus") other.focus();
if (action === "value") h.element.value = "changed";
if (action === "cancel") controller.abort();
if (action === "throw") throw new Error("dispatch failed");
}
return reply;
});
const result = await h.fill("!", clearBefore, controller.signal);
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
expect(h.insert).not.toHaveBeenCalled();
expect(other.value).toBe("");
expect(h.send).toHaveBeenCalledWith(
4,
"Input.dispatchKeyEvent",
expect.objectContaining({ type: "keyUp" }),
);
expect(h.release).toHaveBeenCalledOnce();
});
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
const h = await setup();
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
expect(h.element.value).toBe(clearBefore ? "" : "old");
expect(h.insert).not.toHaveBeenCalled();
});
it("cancels after insertion without sending change events", async () => {
const h = await setup();
const controller = new AbortController();
h.insert.mockImplementation(async () => controller.abort());
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
expect(h.script).toHaveBeenCalledTimes(2);
expect(h.release).toHaveBeenCalledOnce();
});
it("cancels after resolving the remote object without clearing the field", async () => {
const h = await setup();
const controller = new AbortController();
const send = h.send.getMockImplementation()!;
h.send.mockImplementation(async (tabId, method, params) => {
const result = await send(tabId, method, params);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
expect(h.element.value).toBe("old");
expect(h.script).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it("honors cancellation during verification", async () => {
const h = await setup();
const controller = new AbortController();
h.element.addEventListener("change", () => controller.abort());
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
expect(h.release).toHaveBeenCalledOnce();
});
it("keeps the verified outcome when releasing the remote object fails", async () => {
const h = await setup();
h.release.mockRejectedValueOnce(new Error("target closed"));
expect(await h.fill()).toMatchObject({ value_length: 5 });
});
});
@@ -0,0 +1,166 @@
// @vitest-environment node
// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile.
import { describe, expect, it } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
async function withFocusBrowser(
run: (h: {
evaluate: (expression: string) => Promise<unknown>;
ref: () => Promise<void>;
focus: (signal?: AbortSignal) => ReturnType<typeof handleFocus>;
blur: (signal?: AbortSignal) => ReturnType<typeof handleBlur>;
calls: string[];
afterCommand: { current: (method: string) => void };
}) => Promise<void>,
) {
// Reuse the existing isolated Chrome launcher; no new browser dependency.
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, sessionId);
const evaluate = async (expression: string) => {
const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
await evaluate(
`document.body.innerHTML = '<input id="target"><input id="other">'; window.probe = document.querySelector('#target');`,
);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: string[] = [];
const afterCommand = { current: (_method: string) => {} };
const cdp: CdpRunner = {
send: async (_tabId, method, params) => {
calls.push(method);
const result = await send(method, params, sessionId);
afterCommand.current(method);
return result as never;
},
};
const deps = {
cdp,
tabsApi: {
get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
};
await run({
evaluate,
calls,
afterCommand,
ref: async () => {
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
sessionId,
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionId,
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId);
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
},
focus: (signal) =>
handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
blur: (signal) =>
handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
});
},
);
}
describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => {
it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => {
await withFocusBrowser(async (h) => {
if (mode !== "light")
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested);
const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input);
window.probe = input;
})()`);
await h.ref();
expect(await h.focus()).toMatchObject({ focused: true });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(await h.blur()).toMatchObject({ was_focused: true, focused: false });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false);
expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2);
});
});
it("detects focus changes from event-handler microtasks", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate(
"window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})",
);
expect(await h.focus()).toMatchObject({ code: "cdp_failed" });
expect(await h.focus()).toMatchObject({ focused: true });
await h.evaluate(
"window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})",
);
expect(await h.blur()).toMatchObject({ code: "cdp_failed" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
});
});
it("cancels before blur while still releasing the resolved object", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.focus();
const abort = new AbortController();
h.afterCommand.current = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
it("reports a page exception with its cause", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }");
expect(await h.blur()).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
});
@@ -0,0 +1,293 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
interface ScriptParams {
functionDeclaration: string;
}
interface Call {
target: CdpTarget;
method: string;
}
async function setup(element?: HTMLElement, childSession = false) {
if (!element) {
document.body.innerHTML = '<input id="target"><input id="other">';
element = document.querySelector<HTMLInputElement>("#target")!;
}
const targetElement = element;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, {
tabId: 4,
...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}),
});
const calls: Call[] = [];
const afterCommand = vi.fn<(call: Call) => void>();
const script = vi.fn(async (params: ScriptParams) => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return { result: { value: await fn.call(targetElement) } };
} catch (error) {
return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } };
}
});
const focus = vi.fn(() => targetElement.focus());
const release = vi.fn(async () => ({}));
const send = async (target: CdpTarget, method: string, params?: object) => {
const call = { target, method };
calls.push(call);
try {
switch (method) {
case "DOM.getDocument":
return { root: { nodeId: 1 } };
case "DOM.querySelector":
return { nodeId: 2 };
case "DOM.describeNode":
return { node: { backendNodeId: 12 } };
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.resolveNode":
return { object: { objectId: "focus-target" } };
case "DOM.focus":
focus();
return {};
case "Runtime.callFunctionOn":
return await script(params as ScriptParams);
case "Runtime.releaseObject":
return await release();
default:
throw new Error(`unexpected ${method}`);
}
} finally {
afterCommand(call);
}
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
...(childSession
? [
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
]
: []),
],
})),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
return {
element: targetElement,
calls,
afterCommand,
script,
focus,
release,
cdp,
tabsApi,
run: (
action: "focus" | "blur",
signal?: AbortSignal,
params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" },
) =>
(action === "focus" ? handleFocus : handleBlur)(
manager,
{ session_id: "aa11", ...params },
{ cdp, tabsApi, signal },
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("focus and blur state verification", () => {
it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => {
const host = document.createElement("div");
document.body.append(host);
const outer = host.attachShadow({ mode });
const innerHost = document.createElement("div");
outer.append(innerHost);
const inner = innerHost.attachShadow({ mode });
const element = document.createElement("input");
inner.append(element);
const h = await setup(element);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(inner.activeElement).toBe(element);
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
expect(inner.activeElement).toBeNull();
expect(h.release).toHaveBeenCalledTimes(2);
});
it("keeps DOM focus usable when the document is in the background", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
});
it("blurring an unfocused target preserves the other element's focus", async () => {
const h = await setup();
const other = document.querySelector<HTMLInputElement>("#other")!;
other.focus();
expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false });
expect(document.activeElement).toBe(other);
});
it("detects focus redirected by a focus handler's microtask", async () => {
const h = await setup();
h.element.addEventListener("focus", () =>
queueMicrotask(() => document.querySelector<HTMLInputElement>("#other")!.focus()),
);
expect(await h.run("focus")).toMatchObject({
code: "cdp_failed",
message: "target element did not become focused",
});
expect(h.release).toHaveBeenCalledOnce();
});
it("does not report blur success when a microtask restores focus", async () => {
const h = await setup();
h.element.focus();
h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus()));
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: "target element remained focused after blur()",
});
expect(document.activeElement).toBe(h.element);
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => {
const h = await setup();
h.element.remove();
expect(await h.run(action)).toMatchObject({ code: "cdp_failed" });
expect(h.release).toHaveBeenCalledOnce();
});
it("preserves a script exception's cause", async () => {
const h = await setup();
vi.spyOn(h.element, "blur").mockImplementation(() => {
throw new Error("page blur failed");
});
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => {
const h = await setup();
h.script.mockResolvedValueOnce({ result: { value: null } });
expect(await h.run(action)).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("unexpected result"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"focus",
"blur",
] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => {
const h = await setup(undefined, true);
if (action === "blur") h.element.focus();
expect(await h.run(action)).toMatchObject({ focused: action === "focus" });
const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded");
expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true);
expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject");
});
it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => {
const h = await setup();
h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({
code: "permission_denied",
});
expect(h.calls).toEqual([]);
});
it("does not replace a successful result when navigation disposed the remote object", async () => {
const h = await setup();
h.release.mockRejectedValue(new Error("Cannot find context"));
expect(await h.run("focus")).toMatchObject({ focused: true });
});
});
describe("focus and blur cancellation", () => {
const boundaries = [
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
"DOM.resolveNode",
"Runtime.callFunctionOn",
];
for (const action of ["focus", "blur"] as const) {
it.each(
action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries,
)(`${action} stops after cancellation during %s`, async (method) => {
const h = await setup();
const abort = new AbortController();
let cancelledAfter = 0;
h.afterCommand.mockImplementation((call) => {
if (call.method === method && !abort.signal.aborted) {
cancelledAfter = h.calls.length;
abort.abort();
}
});
expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(abort.signal.aborted).toBe(true);
expect(
h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"),
).toBe(true);
if (h.calls.some((call) => call.method === "DOM.resolveNode"))
expect(h.release).toHaveBeenCalledOnce();
});
it(`${action} stops if cancellation arrives while resolving the tab`, async () => {
const h = await setup();
const abort = new AbortController();
h.tabsApi.query.mockImplementation(async () => {
abort.abort();
return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab];
});
expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls).toEqual([]);
});
}
it("does not scroll frame owners after cancellation in an OOPIF", async () => {
const h = await setup(undefined, true);
const abort = new AbortController();
h.afterCommand.mockImplementation((call) => {
if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort();
});
expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
expect(h.focus).not.toHaveBeenCalled();
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
import { resolveNodeGeometry } from "../frame-geometry";
import {
clipPolygon,
polygonArea,
@@ -10,9 +10,141 @@ import {
rectPolygon,
regionBounds,
} from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
function scrollbarDriver(
visible = { width: 185, height: 89 },
frameSize: unknown = { width: 200, height: 100 },
) {
const send = vi.fn(async (target: { sessionId?: string }, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: frameSize } };
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: {
clientWidth: target.sessionId ? visible.width : 800,
clientHeight: target.sessionId ? visible.height : 600,
},
};
if (method === "DOM.getBoxModel")
return {
model: {
content: target.sessionId
? [0, 0, 200, 0, 200, 100, 0, 100]
: [100, 100, 500, 100, 500, 300, 100, 300],
},
};
if (method === "DOM.getContentQuads")
return { quads: [[180, 80, 230, 80, 230, 120, 180, 120]] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: (tabId, method) => send({ tabId } as never, method) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child" },
},
{
frameId: "same-child",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "child" },
},
],
}),
};
return { cdp, send };
}
describe("frame geometry projection", () => {
it.each([
{ width: 200, height: 100 },
{ width: 185, height: 100 },
{ width: 200, height: 89 },
{ width: 185, height: 89 },
])("preserves scale and clips scrollbar strips for visible viewport $width × $height", async (visible) => {
const { cdp, send } = scrollbarDriver(visible);
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: {
x: 460,
y: 260,
width: (visible.width - 180) * 2,
height: (visible.height - 80) * 2,
},
actionPoint: { x: 460 + visible.width - 180, y: 260 + visible.height - 80 },
targetActionPoint: { x: (180 + visible.width) / 2, y: (80 + visible.height) / 2 },
});
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(send.mock.calls.filter(([, method]) => method === "Page.getLayoutMetrics")).toHaveLength(
2,
);
});
it("shares the full viewport read across same-target frames, and refreshes it next operation", async () => {
const { cdp, send } = scrollbarDriver();
const context = new GeometryContext(cdp, 4);
await Promise.all([context.targetProjection("child"), context.targetProjection("same-child")]);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(2);
});
it.each([
null,
{},
{ width: 0, height: 100 },
{ width: 200, height: NaN },
])("rejects unavailable full viewport dimensions: %j", async (size) => {
const { cdp } = scrollbarDriver(undefined, size);
expect(await new GeometryContext(cdp, 4).targetProjection("child")).toBeNull();
});
it("clips an inner OOPIF against an intermediate OOPIF's occupied scrollbar strips", async () => {
const { cdp } = scrollbarDriver();
const outer = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "DOM.getBoxModel")
return { model: { content: [150, 60, 250, 60, 250, 120, 150, 120] } } as never;
if (target.sessionId !== "inner") return outer(target, method, params);
if (method === "Runtime.evaluate")
return { result: { value: { width: 100, height: 60 } } } as never;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 90, clientHeight: 50 } } as never;
if (method === "DOM.getContentQuads")
return { quads: [[0, 0, 100, 0, 100, 60, 0, 60]] } as never;
throw new Error(`unexpected ${method}`);
};
const graph = await cdp.getFrameGraph!(4);
graph.frames[2] = {
frameId: "inner",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "inner" },
};
cdp.getFrameGraph = async () => graph;
expect(
await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "inner" },
frameId: "inner",
backendNodeId: 101,
}),
).toMatchObject({
topBounds: { x: 400, y: 220, width: 70, height: 58 },
});
});
it("keeps region bounds separate from polygon area", () => {
const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
@@ -76,34 +208,32 @@ describe("frame geometry projection", () => {
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
};
const projection = await resolveFrameProjection(
cdp,
{
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
},
"oopif",
);
const projection = await new GeometryContext(cdp, 4, {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
}).targetProjection("oopif");
expect(projection?.edges).toEqual([
{
@@ -138,6 +268,8 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -169,6 +301,7 @@ describe("frame geometry projection", () => {
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
targetActionPoint: { x: 100, y: 50 },
});
});
@@ -198,6 +331,8 @@ describe("frame geometry projection", () => {
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -261,6 +396,7 @@ describe("frame geometry projection", () => {
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
targetActionPoint: { x: 200, y: 150 },
});
});
@@ -0,0 +1,464 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target },
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
],
};
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 300, height: 200 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
});
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
}
describe("measurement geometry context", () => {
it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => {
const measured = snapshotLayoutScale({
visualViewport: { clientWidth: 997.25, zoom: 1.1 },
cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 },
// Rounded layout dimensions must not supply the scale.
layoutViewport: { clientWidth: 997 },
cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) },
});
expect(measured).toBeCloseTo(scale, 12);
const coordinates = snapshotCoordinates(
{ scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale },
measured,
{ x: 999, y: 999 },
);
expect(
snapshotViewportRect(
[160 * scale, 320 * scale, 120 * scale, 40 * scale],
{ target },
coordinates,
)?.rect,
).toEqual({ x: 80, y: 120, width: 120, height: 40 });
});
it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => {
const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 });
expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({
x: 60,
y: 300,
width: 120,
height: 40,
});
expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull();
expect(
snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }),
).toBeNull();
});
it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => {
expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull();
for (const raw of [0, -1, NaN, Infinity]) {
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: raw },
cssVisualViewport: { clientWidth: 800 },
}),
).toBeNull();
}
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: 0, clientHeight: 600 },
cssVisualViewport: { clientWidth: 0, clientHeight: 300 },
}),
).toBe(2);
expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull();
expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull();
});
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const results = await Promise.all(
Array.from({ length: 20 }, () => context.targetProjection("child")),
);
expect(results.every((value) => value === results[0])).toBe(true);
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"Page.getLayoutMetrics",
"DOM.getBoxModel",
]);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
expect(cdp.calls.mock.calls).toHaveLength(4);
});
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const source = { target, frameId: "child" };
const projection = await context.snapshotProjection(source, 10, [], {
width: 1000,
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 0 },
});
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 10 },
});
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
1,
);
expect(
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
).toHaveLength(1);
});
it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => {
const cdp = driver();
const original = cdp.send;
const calls: string[] = [];
cdp.send = async (tabId, method, params) => {
calls.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 10 } },
{ type: "string", value: '{"width":300,"height":200}' },
],
},
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 11 } },
{ type: "string", value: '{"width":0,"height":200}' },
],
},
],
},
},
} as never;
if (method === "Runtime.releaseObjectGroup") return {} as never;
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4);
context.registerSnapshotOwners(target, new Set([10, 11]));
const values = await Promise.all(
[10, 11].map((id) =>
context.snapshotProjection({ target, frameId: String(id) }, id, [], {
width: 1000,
height: 1000,
}),
),
);
expect(values.every((value) => value.status === "available")).toBe(true);
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1);
expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
});
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2);
});
it("waits for batch object cleanup and propagates cancellation", async () => {
const cdp = driver();
const original = cdp.send;
const controller = new AbortController();
let release!: () => void;
let cleanup!: () => void;
const started = new Promise<void>((resolve) => {
cleanup = resolve;
});
cdp.send = async (tabId, method, params) => {
if (method === "Runtime.evaluate") return {} as never;
if (method === "Runtime.releaseObjectGroup") {
cleanup();
await new Promise<void>((resolve) => {
release = resolve;
});
return {} as never;
}
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
context.registerSnapshotOwners(target, new Set([10, 11]));
let settled = false;
const operation = context
.snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 })
.finally(() => {
settled = true;
});
const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" });
await started;
controller.abort();
expect(settled).toBe(false);
release();
await rejected;
expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } },
)!;
const projection = {
source: { target, frameId: "main" },
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
};
expect(projectSnapshotRect(input, projection)).toEqual({
x: 20,
y: 30,
width: 120,
height: 40,
});
expect(
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
).toBeNull();
expect(
projectSnapshotRect(input, {
...projection,
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, NaN, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, 10, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } },
),
).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
})) as CdpRunner["send"],
};
expect(
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 100,
height: 100,
}),
).toEqual({
status: "unavailable",
source: { target, frameId: "child" },
ownerBackendNodeId: 10,
});
});
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
const cdp = driver();
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
{ target, frameId: "nested" },
10,
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
{ width: 1000, height: 1000 },
);
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
y: 620,
width: 100,
height: 100,
});
});
it("rejects malformed frame ancestry without looping", async () => {
const cdp = driver();
const cycle: CdpFrameGraph = {
rootFrameId: "a",
frames: [
{ frameId: "a", parentFrameId: "b", target },
{ frameId: "b", parentFrameId: "a", target },
],
};
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
expect(cdp.calls).not.toHaveBeenCalled();
const orphan: CdpFrameGraph = {
rootFrameId: "a",
frames: [{ frameId: "a", parentFrameId: "missing", target }],
};
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
});
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
const cdp = driver();
expect(
await resolveNodeGeometry(
cdp,
4,
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ code: "cdp_failed" });
expect(cdp.calls).not.toHaveBeenCalled();
});
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
let active = 0;
let peak = 0;
const release: Array<() => void> = [];
const send = vi.fn(async () => {
active++;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 100, clientHeight: 100 },
};
});
const controller = new AbortController();
const context = new GeometryContext(
{ send: send as CdpRunner["send"] },
4,
undefined,
controller.signal,
);
const tasks = Array.from({ length: 8 }, (_, index) =>
context.viewport({ tabId: 4, sessionId: String(index) }),
);
const settled = Promise.allSettled(tasks);
expect(send).toHaveBeenCalledTimes(4);
controller.abort();
for (const done of release) done();
const results = await settled;
expect(peak).toBe(4);
expect(send).toHaveBeenCalledTimes(4);
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
});
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
const controller = new AbortController();
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
cdp.calls.mockImplementation(async (tab, method) => {
const result = await original(tab, method);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
await expect(
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
}),
).rejects.toMatchObject({ name: "AbortError" });
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.releaseObject",
]);
});
it("reads live owner geometry only after scrolling", async () => {
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
let scrolled = false;
cdp.calls.mockImplementation(async (tab, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
scrolled = true;
return {};
}
if (method === "DOM.getContentQuads") {
expect(scrolled).toBe(true);
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
}
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
return original(tab, method);
});
expect(
await resolveNodeGeometry(
cdp,
4,
{ target, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
1,
);
});
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
).toBeNull();
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
).toBeNull();
});
it("retries a rejected measurement only in a new operation", async () => {
const cdp = driver();
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
const context = new GeometryContext(cdp, 4);
await expect(context.viewport(target)).rejects.toThrow("unavailable");
await expect(context.viewport(target)).rejects.toThrow("unavailable");
expect(cdp.calls).toHaveBeenCalledTimes(1);
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
width: 1000,
height: 1000,
});
});
});
@@ -405,6 +405,7 @@ describe("handleRequestHelp", () => {
agentWindowId: 99,
refStore: {
resolveEntry: () => ({
kind: "dom",
backendNodeId: 42,
tabId: 5,
frameId: "child",
@@ -433,6 +434,7 @@ describe("handleRequestHelp", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
@@ -21,7 +23,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -201,6 +203,7 @@ describe("handleClick", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -220,6 +223,7 @@ describe("handleClick", () => {
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
{ sessionId: "child-session", method: "Runtime.evaluate" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
@@ -540,6 +544,176 @@ describe("handleHover", () => {
});
});
describe("handleFocus and handleBlur", () => {
it("focuses a ref and verifies the deep active element", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({ result: { value: true } }),
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true });
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"DOM.focus",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("focuses an OOPIF ref in its CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.focus" || method === "Runtime.releaseObject") return {};
if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } };
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.focused).toBe(true);
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.resolveNode" },
{ sessionId: "child-session", method: "DOM.focus" },
{ sessionId: "child-session", method: "Runtime.callFunctionOn" },
{ sessionId: "child-session", method: "Runtime.releaseObject" },
]);
});
it("does not focus after cancellation during scrolling", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const abort = new AbortController();
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
abort.abort();
return {};
},
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false);
});
it("blurs a ref and returns its previous focus state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": vi
.fn()
.mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } })
.mockResolvedValueOnce({ result: { value: false } }),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({
tab_id: 4,
used_ref: "e3",
was_focused: true,
focused: false,
});
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("rejects a target that does not implement blur", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({
result: { value: { ok: false, was_focused: false, focused: false } },
}),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ });
});
});
function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? [];
return {
result: {
value:
args.length === 2
? { before: "", expected: args[0].value }
: script.functionDeclaration.startsWith("function(expected)")
? { connected: true, matches: true, valueLength: String(args[0].value).length }
: "ready",
},
};
}
describe("handleFill", () => {
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
@@ -586,7 +760,8 @@ describe("handleFill", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
"Runtime.callFunctionOn": successfulFillScript,
"Runtime.releaseObject": () => ({}),
"Input.insertText": () => ({}),
});
const res = await handleFill(
@@ -600,13 +775,12 @@ describe("handleFill", () => {
expect(res.used_ref).toBe("e1");
const insert = fake.sent.find((c) => c.method === "Input.insertText");
expect(insert?.params).toEqual({ text: "hello" });
// clear_before defaults to true → callFunctionOn invoked once to
// clear, once to fire input/change after typing.
// Foreground replacement needs no extra caret-positioning round trip.
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
expect(callFns.length).toBeGreaterThanOrEqual(2);
expect(callFns).toHaveLength(4);
});
it("clear_before=false skips the wipe call", async () => {
it("passes clear_before=false to preparation and verifies the result", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 1, { tabId: 4 });
@@ -618,12 +792,12 @@ describe("handleFill", () => {
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
"Runtime.callFunctionOn": (p) => {
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
// No "this.value = ''" clearing on a no-clear path; only the
// post-input dispatchEvent.
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
return { result: { type: "undefined" } };
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
if (args.length === 2) expect(args[1].value).toBe(false);
return successfulFillScript(p);
},
"Input.dispatchKeyEvent": () => ({}),
"Runtime.releaseObject": () => ({}),
"Input.insertText": () => ({}),
});
const res = await handleFill(
@@ -650,7 +824,8 @@ describe("handleFill", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
"Runtime.callFunctionOn": successfulFillScript,
"Runtime.releaseObject": () => ({}),
"Input.insertText": () => ({}),
});
const res = await handleFill(
@@ -0,0 +1,266 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { BrowserNavigationApi } from "../browser-navigation";
import { handleNavigate, handleReload } from "../navigation";
import type { CdpRunner } from "../shared";
const denied = "Cannot access a chrome-extension:// URL of different extension";
const url = "https://example.test/recovered";
function event<T extends (...args: never[]) => void>() {
const listeners = new Set<T>();
return {
listeners,
addListener: (listener: T) => listeners.add(listener),
removeListener: (listener: T) => listeners.delete(listener),
fire: (...args: Parameters<T>) => {
for (const listener of [...listeners]) listener(...args);
},
};
}
async function fixture() {
const manager = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 4),
},
});
await manager.start("test");
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
const events = {
onCommitted: event<Listener>(),
onDOMContentLoaded: event<Listener>(),
onCompleted: event<Listener>(),
onErrorOccurred: event<Listener>(),
};
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
const send = vi.fn(async (_tabId: number, method: string) => {
if (state.blocked) throw new Error(denied);
if (method === "Page.getFrameTree") {
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
}
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
frameId: "main",
loaderId: "new-loader",
name: "networkIdle",
});
}
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
return {};
});
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
onEvent: (listener) => {
cdpEvents.addListener(listener);
return { dispose: () => cdpEvents.removeListener(listener) };
},
};
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
const navigate = async () => {
state.blocked = state.destinationBlocked;
events.onCommitted.fire(main);
events.onDOMContentLoaded.fire(main);
events.onCompleted.fire(main);
};
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
const expectCleanedUp = () => {
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
expect(cdpEvents.listeners.size).toBe(0);
};
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
}
describe("navigation after Chrome denies extension-frame access", () => {
it.each([
"load",
"commit",
"domcontentloaded",
"networkidle",
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
const f = await fixture();
const result = await handleNavigate(
f.manager,
{
session_id: "test",
url,
wait_until: waitUntil,
},
f.deps,
);
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
f.expectCleanedUp();
});
it("recovers reload through tabs.reload and preserves hard reload", async () => {
const f = await fixture();
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
bypassCache: true,
});
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("reports the restriction when reload leaves the conflicting frame present", async () => {
const f = await fixture();
f.state.destinationBlocked = true;
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "cdp_extension_access_denied" },
});
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
f.expectCleanedUp();
});
it("does not fall back for unrelated debugger failures", async () => {
const f = await fixture();
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
expect(result).toMatchObject({
code: "cdp_failed",
message: "Another debugger is already attached",
});
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("does not replay navigation when a dispatched CDP command fails", async () => {
const f = await fixture();
f.state.blocked = false;
const send = f.send.getMockImplementation()!;
f.send.mockImplementation(async (tabId, method) => {
if (method === "Page.navigate") throw new Error(denied);
return send(tabId, method);
});
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "cdp_extension_access_denied" },
});
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("keeps the Agent Window guard in front of the recovery", async () => {
const f = await fixture();
f.deps.tabsApi.get.mockResolvedValue({
id: 4,
windowId: 200,
active: true,
url,
} as chrome.tabs.Tab);
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
expect(result).toMatchObject({ code: "permission_denied" });
expect(f.send).not.toHaveBeenCalled();
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
});
it("ignores old-document, subframe, and other-tab completion events", async () => {
const f = await fixture();
f.deps.browserNavigation.update.mockImplementation(async () => {});
let settled = false;
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
settled = true;
return result;
});
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
f.events.onCompleted.fire(f.main);
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
f.events.onCompleted.fire(f.main);
await Promise.resolve();
expect(settled).toBe(false);
f.events.onCommitted.fire(f.main);
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
await Promise.resolve();
expect(settled).toBe(false);
f.state.blocked = false;
f.events.onCompleted.fire(f.main);
expect(await work).toMatchObject({ reached: "load" });
f.expectCleanedUp();
});
it("does not infer network idle from a browser load event", async () => {
const f = await fixture();
f.state.emitIdle = false;
const result = await handleNavigate(
f.manager,
{
session_id: "test",
url,
wait_until: "networkidle",
timeout_ms: 10,
},
f.deps,
);
expect(result).toMatchObject({ reached: "timeout" });
f.expectCleanedUp();
});
it("cleans up browser listeners on timeout", async () => {
const f = await fixture();
f.deps.browserNavigation.update.mockImplementation(async () => {});
const result = await handleNavigate(
f.manager,
{ session_id: "test", url, timeout_ms: 10 },
f.deps,
);
expect(result).toMatchObject({ reached: "timeout" });
f.expectCleanedUp();
});
it("does not navigate an already-cancelled request", async () => {
const f = await fixture();
const result = await handleNavigate(
f.manager,
{ session_id: "test", url },
{
...f.deps,
signal: AbortSignal.abort(),
},
);
expect(result).toMatchObject({ code: "cancelled" });
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("cleans up browser listeners when cancelled after dispatch", async () => {
const f = await fixture();
const ac = new AbortController();
f.deps.browserNavigation.update.mockImplementation(async () => {
ac.abort();
});
const result = await handleNavigate(
f.manager,
{ session_id: "test", url },
{ ...f.deps, signal: ac.signal },
);
expect(result).toMatchObject({ code: "cancelled" });
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
f.expectCleanedUp();
});
it("reports browser navigation errors and releases listeners", async () => {
const f = await fixture();
f.deps.browserNavigation.update.mockImplementation(async () => {
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
});
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
f.expectCleanedUp();
});
});
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,348 @@
// @vitest-environment node
// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile.
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
async function withBrowser(
run: (h: Awaited<ReturnType<typeof harness>>) => Promise<void>,
zoom = 1,
) {
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom },
async (send: Send) => run(await harness(send)),
);
}
async function harness(send: Send) {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, rootSession);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: { method: string; sessionId?: string }[] = [];
const after = { command: (_method: string) => {} };
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
const forward = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
calls.push({ method, sessionId: target.sessionId });
const result = await send<T>(method, params, sessionFor(target));
after.command(method);
return result;
};
const cdp: CdpRunner = {
send: (_tab, method, params) => forward({ tabId: 4 }, method, params),
sendToTarget: forward,
};
const evaluate = async <T>(expression: string, frame?: CdpFrame): Promise<T> => {
const session = frame ? sessionFor(frame.target) : rootSession;
const world = frame
? await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
session,
)
: undefined;
const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{
expression,
contextId: world?.executionContextId,
returnByValue: true,
awaitPromise: true,
},
session,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
const rememberRef = (frame: CdpFrame, backendNodeId: number) =>
ctx.refStore.set("e1", backendNodeId, {
tabId: 4,
frameId: frame.frameId,
cdpSessionId: frame.target.sessionId,
});
const ref = async (frame: CdpFrame, selector = "#probe") => {
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
sessionFor(frame.target),
);
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{
expression: `document.querySelector(${JSON.stringify(selector)})`,
contextId: executionContextId,
},
sessionFor(frame.target),
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionFor(frame.target),
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target));
rememberRef(frame, node.backendNodeId);
};
const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) =>
handleScrollTo(
manager,
{ session_id: "aa11", ...target },
{
cdp,
signal,
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
},
);
const loadFrames = async () => {
const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targetInfos.filter((t) => t.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
const frames: (CdpFrame & { name: string })[] = [];
for (const session of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session);
const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) };
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" });
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((f) => f.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
cdp.getFrameGraph = async () => graph;
return frames;
};
return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after };
}
function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null {
const x = Math.max(a.x, b.x),
y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.width, b.x + b.width),
bottom = Math.min(a.y + a.height, b.y + b.height);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
}
function expectRect(actual: unknown, expected: ViewportRect) {
expect(actual).not.toHaveProperty("code");
for (const key of ["x", "y", "width", "height"] as const)
expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2);
}
describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => {
it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => {
await withBrowser(async (h) => {
for (const html of [
'<div style="height:2200px"></div><div id="target" style="width:100px;height:40px"></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto"><div id="target" style="width:100px;height:400px"></div></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto;transform:scale(1.25);transform-origin:0 0"><div id="target" style="width:100px;height:400px"></div></div>',
]) {
await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`);
const result = await h.scroll({ selector: "#target" });
const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({
rect: document.querySelector('#target').getBoundingClientRect().toJSON(),
clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON()
})`);
expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect);
}
});
});
it.each([
"visibility:hidden",
"opacity:0",
"display:none",
"content-visibility:hidden",
])("rejects %s targets", async (style) => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="${style}"><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
});
});
it("rejects targets fully outside an overflow:clip ancestor", async () => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="height:80px;overflow:clip"><div style="height:150px"></div><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" });
});
});
it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => {
await withBrowser(async (h) => {
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const root = host.attachShadow({mode:'${mode}'});
root.innerHTML = '<div style="height:60px;overflow:auto"><button id="probe" style="height:200px;width:100px">Target</button></div>';
window.probe = root.querySelector('#probe');
})()`);
const frames = await h.loadFrames();
const { result } = await h.send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
h.rootSession,
);
// Build the ref from the actual closed-root node, as an observation does.
const { node } = await h.send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
h.rootSession,
);
await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession);
h.rememberRef(frames[0], node.backendNodeId);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 });
});
});
it("cancels after scrolling and releases the measurement object without follow-up reads", async () => {
await withBrowser(async (h) => {
await h.evaluate("document.body.innerHTML = '<button id=target>Target</button>'");
const abort = new AbortController();
h.after.command = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({
code: "cancelled",
});
expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
});
});
it.each([
{ fixture: "snapshot-coordinates", zoom: 1 },
{ fixture: "snapshot-coordinates", zoom: 1.25 },
{ fixture: "oopif-scrollbars", zoom: 1 },
{ fixture: "oopif-scrollbars", zoom: 0.8 },
])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => {
const { createEvalServer } = await import(
new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withBrowser(async (h) => {
await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession);
await expect
.poll(
() =>
server
.snapshot("scroll")
.events.some(
(e: { type: string; data: { root?: boolean } }) =>
e.type === "geometry.ready" && e.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const frames = await h.loadFrames();
expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3);
for (const frame of frames) {
await h.ref(frame);
const result = await h.scroll({ ref: "e1" });
// Independent DOM oracle: CSS border boxes plus this fixture's known
// axis-aligned iframe scales. Re-read after each scroll changes layout.
let rect = await h.evaluate<ViewportRect>(
"document.querySelector('#probe').getBoundingClientRect().toJSON()",
frame,
);
let current = frame;
while (true) {
const viewport = await h.evaluate<ViewportRect>(
"({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })",
current,
);
rect = intersection(rect, viewport)!;
if (!current.parentFrameId) break;
const parent = frames.find((f) => f.frameId === current.parentFrameId)!;
const owner = await h.evaluate<{ x: number; y: number; scale: number }>(
`(() => {
const iframe = document.querySelector('iframe[name="${current.name}"]');
const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe);
const scale = r.width / iframe.offsetWidth;
return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale,
y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale };
})()`,
parent,
);
rect = {
x: owner.x + rect.x * owner.scale,
y: owner.y + rect.y * owner.scale,
width: rect.width * owner.scale,
height: rect.height * owner.scale,
};
current = parent;
}
expectRect(result, rect);
}
const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!;
await h.ref(child);
await h.evaluate(
`document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`,
);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" });
}, zoom);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -0,0 +1,310 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
interface Call {
target: CdpTarget;
method: string;
params?: Record<string, unknown>;
aborted: boolean;
}
async function fixture(frame: "top" | "same-target" | "oopif" = "top") {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
...(frame !== "top" ? { frameId: "child" } : {}),
...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}),
});
const abort = new AbortController();
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) },
},
],
};
const rectangles = new Map<number, ViewportRect | null>([
[1234, { x: 10, y: 20, width: 100, height: 40 }],
[99, { x: 204, y: 306, width: 400, height: 200 }],
]);
const calls: Call[] = [];
const hooks = {
after: (_call: Call) => {},
reply: (_call: Call): object | undefined => undefined,
};
const send = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
const call = {
target,
method,
params: params as Call["params"],
aborted: abort.signal.aborted,
};
calls.push(call);
const override = hooks.reply(call);
hooks.after(call);
if (override) return override as T;
const args = call.params ?? {};
const replies: Record<string, unknown> = {
"DOM.getDocument": { root: { nodeId: 1 } },
"DOM.querySelector": { nodeId: 2 },
"DOM.describeNode": { node: { backendNodeId: 1234 } },
"DOM.scrollIntoViewIfNeeded": {},
"DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } },
"Runtime.callFunctionOn": {
result: {
value: String(args.functionDeclaration).includes("IntersectionObserver")
? rectangles.get(Number(args.objectId))
: { width: 200, height: 100 },
},
},
"Runtime.releaseObject": {},
"Runtime.releaseObjectGroup": {},
"Runtime.evaluate": { result: { value: { width: 200, height: 100 } } },
"Page.getLayoutMetrics": {
cssLayoutViewport: {
clientWidth: target.sessionId ? 185 : 1280,
clientHeight: target.sessionId ? 89 : 720,
},
},
"DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } },
};
if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`);
return replies[method] as T;
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
getFrameGraph: vi.fn(async () => {
hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted });
return graph;
}),
trackSessionTab: vi.fn(),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
const run = (params: Partial<ScrollToParams> = {}) =>
handleScrollTo(
manager,
{ session_id: "aa11", ref: "@e3", ...params },
{ cdp, tabsApi, signal: abort.signal },
);
return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run };
}
afterEach(() => vi.restoreAllMocks());
describe("handleScrollTo", () => {
it("scrolls a ref and returns renderer-clipped bounds", async () => {
const f = await fixture();
expect(await f.run()).toMatchObject({
tab_id: 4,
used_ref: "e3",
x: 10,
y: 20,
width: 100,
height: 40,
});
expect(f.calls[0]).toMatchObject({
method: "DOM.scrollIntoViewIfNeeded",
params: { backendNodeId: 1234 },
});
const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup;
expect(f.calls.at(-1)).toMatchObject({
method: "Runtime.releaseObjectGroup",
params: { objectGroup: group },
});
});
it("resolves a selector and tracks the attached tab even when it is missing", async () => {
const f = await fixture();
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
used_selector: "#target",
});
f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined);
expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({
code: "not_found",
data: { reason: "selector_not_found" },
});
expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4);
});
it.each([
"same-target",
"oopif",
] as const)("projects %s refs and scrolls their owners", async (frame) => {
const f = await fixture(frame);
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
expect(
f.calls
.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")
.map((call) => call.params!.backendNodeId),
).toEqual([99, 1234]);
expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1);
if (frame === "oopif")
expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe(
"child-session",
);
});
it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => {
const f = await fixture("oopif");
f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 });
expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 });
});
it("clips against the iframe owner's ordinary DOM ancestors", async () => {
const f = await fixture("oopif");
f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 });
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 });
});
it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => {
const f = await fixture("oopif");
f.rectangles.set(node, null);
expect(await f.run()).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it.each([
{ ref: undefined },
{ selector: "#target" },
{ timeout_ms: 0 },
])("rejects invalid params %j", async (params) => {
const f = await fixture();
expect(await f.run(params)).toMatchObject({ code: "invalid_params" });
expect(f.calls).toEqual([]);
});
it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => {
const f = await fixture();
if (mode === "stale") f.ctx.refStore.clear();
else f.ctx.refStore.set("e3", 1234, { tabId: 8 });
expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } });
expect(f.calls).toEqual([]);
});
it("enforces the Agent Window boundary", async () => {
const f = await fixture();
f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]);
f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" });
expect(f.calls).toEqual([]);
});
it.each([
"getFrameGraph",
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Page.getLayoutMetrics",
"Runtime.evaluate",
"DOM.getBoxModel",
])("honors cancellation during %s and only cleans up afterwards", async (method) => {
const f = await fixture("oopif");
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(
f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"),
).toEqual([]);
});
it.each([
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
])("cancels selector resolution during %s", async (method) => {
const f = await fixture();
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(f.calls.filter((call) => call.aborted)).toEqual([]);
});
it("does not start after early cancellation", async () => {
const f = await fixture();
f.abort.abort();
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls).toEqual([]);
expect(f.tabsApi.query).not.toHaveBeenCalled();
});
it("does not issue fallback scrolling after cancellation", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
f.abort.abort();
throw new Error("scroll failed");
}
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]);
});
it("stops target scrolling after a deadline expires while scrolling its owner", async () => {
const f = await fixture("oopif");
const now = vi.spyOn(Date, "now").mockReturnValue(100);
f.hooks.after = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201);
};
expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" });
expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
});
it("reports measurement exceptions and releases objects even if cleanup fails", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "Runtime.callFunctionOn")
return { exceptionDetails: { text: "measurement failed" } };
if (method === "Runtime.releaseObjectGroup") throw new Error("target closed");
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" });
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it("attaches dialogs observed during scrolling", async () => {
const f = await fixture();
f.cdp.dialogCursor = () => 5;
const dialogs: JavaScriptDialogInfo[] = [
{
sequence: 6,
tab_id: 4,
type: "alert",
message: "scrolled",
handled: "accepted",
url: "https://example.test",
},
];
f.cdp.dialogsSince = vi.fn(() => dialogs);
expect(await f.run()).toHaveProperty("dialogs", dialogs);
expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5);
});
});
@@ -1,8 +1,26 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleSessionStop } from "../session";
import type { ChromeTabsApi } from "../shared";
import { type AgentOverlayResetApi, type ChromeWindowsApi, type TabMutationApi } from "../tabs";
/** Build a read-only query api over a FakeState. */
function makeQuery(state: FakeState): ChromeTabsApi {
return {
get: vi.fn(async (id) => {
const t = state.tabs.get(id);
if (!t) throw new Error(`tab ${id} not found`);
return t;
}),
query: vi.fn(async (q: chrome.tabs.QueryInfo) => {
const w = q.windowId;
return Array.from(state.tabs.values()).filter(
(t) => typeof w !== "number" || t.windowId === w,
);
}),
};
}
function fakeAgentWindow(ids: number[]) {
let i = 0;
const create = vi.fn(async () => {
@@ -11,7 +29,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
});
const remove = vi.fn(async () => {});
const ensureActiveTab = vi.fn(async () => {});
const ensureActiveTab = vi.fn(async () => 0);
return { create, remove, ensureActiveTab };
}
@@ -30,7 +48,9 @@ function makeApis(
} {
const tabs: TabMutationApi = {
create: vi.fn(),
remove: vi.fn(async () => {}),
remove: vi.fn(async (id: number) => {
state.tabs.delete(id);
}),
update: vi.fn(async (_id, _p) => undefined),
get: vi.fn(async (id) => {
const t = state.tabs.get(id);
@@ -237,11 +257,15 @@ describe("handleSessionStop with auto-return", () => {
moves: [],
};
const { tabs, windows } = makeApis(state, { moveThrowsFor: new Set([1]) });
const cdp = {
detachSession: vi.fn(async () => {}),
releaseSessionTab: vi.fn(async () => {}),
};
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows } },
{ cdp, tabManagement: { tabs, windows } },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.return_failures?.map((f) => f.tab_id)).toEqual([1]);
@@ -249,6 +273,8 @@ describe("handleSessionStop with auto-return", () => {
expect(sm.has("aa11")).toBe(true);
expect(ctx.borrowedTabs.has(1)).toBe(true);
expect(ctx.borrowedTabs.has(2)).toBe(false);
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 2);
expect(cdp.detachSession).not.toHaveBeenCalled();
expect(aw.remove).not.toHaveBeenCalled();
});
@@ -267,4 +293,275 @@ describe("handleSessionStop with auto-return", () => {
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
expect(ctx.refStore.isEmpty()).toBe(true);
});
it.each([
"before teardown",
"during its return",
"after its return failed",
"before its turn in the return loop",
])("finishes stopping when a borrowed tab closes %s", async (timing) => {
const aw = fakeAgentWindow([100]);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(1, { tabId: 1, originalWindowId: 200, originalIndex: 0 });
ctx.borrowedTabs.set(2, { tabId: 2, originalWindowId: 200, originalIndex: 1 });
const state: FakeState = {
tabs: new Map([
[1, { id: 1, windowId: 100 } as chrome.tabs.Tab],
[2, { id: 2, windowId: 100 } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const cdp = { detachSession: vi.fn(async () => {}) };
const closeTab = (tabId: number) => {
state.tabs.delete(tabId);
sm.forgetClosedTab(tabId);
};
const baseMove = tabs.move;
tabs.move = vi.fn(async (tabId, props) => {
if (tabId === 1 && timing === "during its return") closeTab(1);
if (tabId === 1 && timing === "after its return failed") {
throw new Error("simulated move failure");
}
if (tabId === 2 && timing === "after its return failed") closeTab(1);
if (tabId === 1 && timing === "before its turn in the return loop") closeTab(2);
if (!state.tabs.has(tabId)) throw new Error(`tab ${tabId} not found`);
return baseMove(tabId, props);
});
if (timing === "before teardown") closeTab(1);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{
cdp,
tabManagement: {
tabs,
windows,
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
},
tabsQuery: makeQuery(state),
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const closedTabId = timing === "before its turn in the return loop" ? 2 : 1;
expect(res.returned_tab_ids).toEqual([closedTabId === 1 ? 2 : 1]);
expect(res.return_failures).toBeUndefined();
expect(ctx.borrowedTabs.size).toBe(0);
expect(sm.has("aa11")).toBe(false);
expect(cdp.detachSession).toHaveBeenCalledWith("aa11");
expect(aw.remove).toHaveBeenCalledWith(100);
if (timing === "before teardown" || timing === "before its turn in the return loop") {
expect(tabs.move).not.toHaveBeenCalledWith(closedTabId, expect.anything());
}
});
});
describe("handleSessionStop window release (issue #57)", () => {
const agentWindowId = 100;
it("releases the window when user-created tabs remain", async () => {
const aw = fakeAgentWindow([agentWindowId]);
// ensureActiveTab returns the home tab id (10). We override the fake to
// return a known id.
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
ctx.agentCreatedTabs.add(12);
// user tab 99 (created via Chrome UI, NOT in agentCreatedTabs)
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
[12, { id: 12, windowId: agentWindowId } as chrome.tabs.Tab],
[99, { id: 99, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.window_released).toBe(true);
// agent tabs + home removed, user tab 99 kept
expect((tabs.remove as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0]).sort()).toEqual([
10, 11, 12,
]);
expect(state.tabs.has(99)).toBe(true);
// window released (dropOnly), not closed
expect(aw.remove).not.toHaveBeenCalled();
expect(sm.has("aa11")).toBe(false);
});
it("closes the window (not release) when no user tabs remain", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
expect(sm.has("aa11")).toBe(false);
});
it("is non-fatal when an agent tab is already gone", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11); // no longer in state → remove throws
const state: FakeState = {
tabs: new Map([[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab]]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
if (id === 11) throw new Error("tab 11 already closed");
state.tabs.delete(id); // home (10) removed normally
});
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// user tab 10? no — 10 is home, removed. window would be empty → close.
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalled();
});
it("degrades to close-window when tabsQuery is missing", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows } }, // no tabsQuery
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// No query available → conservatively close the window.
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
});
it("degrades to close-window when tabsQuery.query throws", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
const state: FakeState = {
tabs: new Map([[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab]]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const query = makeQuery(state);
(query.query as ReturnType<typeof vi.fn>).mockImplementation(async () => {
throw new Error("query failed");
});
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalled();
});
it("tracks agentCreatedTabs across tab_create / tab_close", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
// Simulate handleTabCreate adding, then handleTabClose removing.
ctx.agentCreatedTabs.add(11);
expect(ctx.agentCreatedTabs.has(11)).toBe(true);
ctx.agentCreatedTabs.delete(11); // as handleTabClose does on success
expect(ctx.agentCreatedTabs.has(11)).toBe(false);
// Unknown id delete is a safe no-op (user-closed tab).
expect(() => ctx.agentCreatedTabs.delete(999)).not.toThrow();
});
it("closes the window (not release) when an agent tab fails to close", async () => {
// Regression: a leaked agent tab (still in agentCreatedTabs, but
// still present because Step 4 remove() threw) must NOT be mistaken
// for a user tab. Otherwise the window would be released (dropOnly)
// and the agent tab would leak (issue #57 regression).
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11); // Step 4 remove() will throw for this tab
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
if (id === 11) throw new Error("tab 11 failed to close");
state.tabs.delete(id); // home (10) removed normally; 11 lingers
});
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// The leaked agent tab must NOT keep the window open.
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
expect(sm.has("aa11")).toBe(false);
});
});
@@ -12,7 +12,7 @@ function fakeAgentWindow() {
return {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -0,0 +1,355 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import type { CdpRunner } from "../shared";
import { captureObservationFacts } from "../vom/capture-coordinator";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = {
probes: Record<string, Rect>;
viewport: { width: number; height: number };
owners: Record<string, { x: number; y: number; scale: number }>;
};
function clip(rect: Rect, viewport: Oracle["viewport"]): Rect | null {
const x = Math.max(0, rect.x),
y = Math.max(0, rect.y);
const right = Math.min(viewport.width, rect.x + rect.w);
const bottom = Math.min(viewport.height, rect.y + rect.h);
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
}
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const probes = {};
for (const node of document.querySelectorAll('[data-geometry-probe]')) {
const box = node.getBoundingClientRect();
probes[node.id] = { x: box.x, y: box.y, w: box.width, h: box.height };
}
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
const style = getComputedStyle(frame);
const scale = rect.width / frame.offsetWidth;
owners[frame.name] = {
x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale,
y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale,
scale,
};
}
return { probes, viewport: { width: document.documentElement.clientWidth, height: document.documentElement.clientHeight }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each(
[
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
]
.flatMap((configuration) => [
{ ...configuration, fixture: "snapshot-coordinates", scrollbars: "none" },
{ ...configuration, fixture: "oopif-scrollbars", scrollbars: "both" },
])
.concat(
["vertical", "horizontal", "none"].map((scrollbars) => ({
deviceScale: 1,
zoom: 1,
fixture: "oopif-scrollbars",
scrollbars,
})),
),
)("$fixture: device scale $deviceScale, zoom $zoom, scrollbars $scrollbars", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withChrome(
{ executable: process.env.BSK_GEOMETRY_CHROME, ...configuration },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true },
);
await send(
"Page.navigate",
{
url: `${baseUrl}/${configuration.fixture}?run=coordinates&scrollbars=${configuration.scrollbars}`,
},
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.some(
(event: { type: string; data: { root?: boolean } }) =>
event.type === "geometry.ready" && event.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(configuration.fixture === "oopif-scrollbars" ? 3 : 2);
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
for (const sessionId of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>(
"Page.getFrameTree",
{},
sessionId,
);
const target: CdpTarget = {
tabId: 1,
...(sessionId === rootSession ? {} : { sessionId }),
};
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target });
names.set(tree.frame.id, tree.frame.name ?? "");
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
expect(frames).toHaveLength(configuration.fixture === "oopif-scrollbars" ? 3 : 5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
const calls: string[] = [];
const cdp: CdpRunner = {
send: (tabId, method, params) => {
calls.push(`${tabId}:${method}`);
return send(method, params, rootSession);
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
if (
method === "Runtime.evaluate" &&
(params as { expression?: string })?.expression?.includes("window.innerWidth")
)
calls.push(`${target.sessionId ?? target.tabId}:viewport-size`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
getAttachmentId: () => rootSession,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
const session = sessionFor(frame.target);
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{ frameId: frame.frameId, worldName: "coordinate-oracle" },
session,
);
await expect
.poll(
async () => {
const ready = await send<{ result: { value: boolean } }>(
"Runtime.evaluate",
{
expression: 'document.documentElement.dataset.geometryReady === "true"',
contextId: executionContextId,
returnByValue: true,
},
session,
);
return ready.result.value;
},
{ timeout: 5_000 },
)
.toBe(true);
// Establish scroll after cross-process target creation/layout settles.
const scroll = frame.parentFrameId
? names.get(frame.frameId) === "nested"
? [10, 20]
: [40, 100]
: [80, 240];
await send(
"Runtime.evaluate",
{
expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`,
contextId: executionContextId,
awaitPromise: true,
},
session,
);
const reply = await send<{ result: { value: Oracle } }>(
"Runtime.evaluate",
{ expression: oracleExpression, contextId: executionContextId, returnByValue: true },
session,
);
oracles.set(frame.frameId, reply.result.value);
}
const childViewport = await send<{
result: {
value: { width: number; height: number; clientWidth: number; clientHeight: number };
};
}>(
"Runtime.evaluate",
{
expression:
"({ width: innerWidth, height: innerHeight, clientWidth: document.documentElement.clientWidth, clientHeight: document.documentElement.clientHeight })",
returnByValue: true,
},
sessions[1],
);
for (const dimension of ["Width", "Height"] as const) {
const occupied =
childViewport.result.value[dimension === "Width" ? "width" : "height"] -
childViewport.result.value[`client${dimension}`];
if (
configuration.scrollbars === "both" ||
configuration.scrollbars === (dimension === "Width" ? "vertical" : "horizontal")
)
expect(occupied).toBeGreaterThan(0);
else expect(occupied).toBe(0);
}
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
// Identity verification also uses Runtime.evaluate; count the full viewport reads separately.
const viewportReadsBeforeLive = calls.filter((call) => call.endsWith(":viewport-size"));
expect(new Set(viewportReadsBeforeLive).size).toBe(viewportReadsBeforeLive.length);
expect(viewportReadsBeforeLive.length).toBeLessThanOrEqual(sessions.length - 1);
for (const frame of frames) {
for (const [id, local] of Object.entries(oracles.get(frame.frameId)!.probes)) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === id);
expect(node, `missing ${id} in ${names.get(frame.frameId) || "root"}`).toBeDefined();
let top = clip(local, oracles.get(frame.frameId)!.viewport);
let current = frame;
while (top && current.parentFrameId) {
const parentOracle = oracles.get(current.parentFrameId)!;
const owner = parentOracle.owners[names.get(current.frameId)!];
top = clip(
{
x: owner.x + top.x * owner.scale,
y: owner.y + top.y * owner.scale,
w: top.w * owner.scale,
h: top.h * owner.scale,
},
parentOracle.viewport,
);
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} ${id} local ${key}`,
).toBeLessThan(2);
if (!top) {
expect(node!.rect, `${id} must be clipped out`).toBeNull();
if (configuration.fixture === "oopif-scrollbars")
expect(
await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
}),
).toMatchObject({ code: "permission_denied" });
continue;
}
expect(node!.rect).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} ${id} top ${key}`,
).toBeLessThan(2);
if (configuration.fixture !== "oopif-scrollbars") continue;
const live = await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
});
if ("code" in live) throw new Error(live.message);
for (const [key, liveKey] of [
["x", "x"],
["y", "y"],
["w", "width"],
["h", "height"],
] as const)
expect(
Math.abs(live.topBounds[liveKey] - top[key]),
`${id} live ${key}`,
).toBeLessThan(2);
const before = server.snapshot("coordinates").events.length;
for (const type of ["mousePressed", "mouseReleased"])
await send(
"Input.dispatchMouseEvent",
{ type, ...live.actionPoint, button: "left", clickCount: 1 },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.slice(before)
.some(
(event: { type: string; path: string; data: { probe?: string } }) =>
event.type === "geometry.clicked" &&
event.data.probe === id &&
event.path ===
(frame.parentFrameId
? names.get(frame.frameId) === "nested"
? "/oopif-scrollbars/nested"
: "/oopif-scrollbars/frame"
: "/oopif-scrollbars"),
),
{ timeout: 3000 },
)
.toBe(true);
}
}
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
}>("Page.getLayoutMetrics", {}, rootSession);
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4);
},
);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
},
remove: async () => {},
ensureActiveTab: async () => {},
ensureActiveTab: async () => 1,
};
}
+181 -1
View File
@@ -1,5 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { SessionManager } from "@/session-manager/manager";
import { handleHover } from "../interaction";
import {
type AgentOverlayResetApi,
type ChromeWindowsApi,
@@ -23,7 +25,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -221,6 +223,7 @@ describe("handleTabCreate", () => {
active: true,
});
expect(res).toMatchObject({ tab_id: 10, window_id: 100 });
expect(sm.get("aa11")?.agentCreatedTabs.has(10)).toBe(true);
});
it("forwards explicit url + index + active=false", async () => {
@@ -256,6 +259,41 @@ describe("handleTabCreate", () => {
const res = await handleTabCreate(sm, { session_id: "aa11", index: -1 }, { tabs: api });
expect(res).toMatchObject({ code: "invalid_params" });
});
it("does not claim any tab when chrome.tabs.create fails", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
const { api, spies } = makeTabMutationApi(state);
spies.create.mockRejectedValueOnce(new Error("chrome.tabs.create failed"));
const res = await handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
expect(res).toMatchObject({ code: "protocol_error" });
expect(sm.get("aa11")?.agentCreatedTabs).toEqual(new Set([1]));
});
it("claims the returned id even when a user tab appears during tab creation", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
const { api, spies } = makeTabMutationApi(state);
let resolveCreate!: (tab: chrome.tabs.Tab) => void;
const createResult = new Promise<chrome.tabs.Tab>((resolve) => {
resolveCreate = resolve;
});
spies.create.mockImplementationOnce(() => createResult);
const pending = handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
// A user can create an unrelated same-window tab while Chrome is still
// resolving the agent request. There is no pending slot for it to steal.
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
resolveCreate({ id: 10, windowId: 100, active: true } as chrome.tabs.Tab);
await expect(pending).resolves.toMatchObject({ tab_id: 10 });
expect(ctx.agentCreatedTabs.has(10)).toBe(true);
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
});
});
describe("handleTabClose", () => {
@@ -511,6 +549,148 @@ describe("handleTabBorrow", () => {
});
describe("handleTabReturn", () => {
it.each([
"not_found",
"cdp_failed",
])("detaches after selector resolution fails with %s", async (code) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const state: FakeTabState = {
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
nextTabId: 50,
windowsClosed: new Set(),
};
const { api } = makeTabMutationApi(state);
const { api: windows } = makeWindowsApi(state);
const debuggerApi = {
attach: vi.fn(async () => {}),
detach: vi.fn(async () => {}),
sendCommand: vi.fn(async (_target: unknown, method: string) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.querySelector") {
if (code === "cdp_failed") throw new Error("selector query failed");
return { nodeId: 0 };
}
return {};
}),
onEvent: { addListener: vi.fn(), removeListener: vi.fn() },
onDetach: { addListener: vi.fn(), removeListener: vi.fn() },
};
const cdp = new ChromiumCdp(debuggerApi as unknown as CdpDebuggerApi);
const hover = await handleHover(
sm,
{ session_id: "aa11", tab_id: 7, selector: "#missing" },
{
cdp,
tabsApi: { get: api.get, query: vi.fn(async () => []) },
},
);
expect(hover).toMatchObject({ code });
expect(cdp.isAttached(7)).toBe(true);
const returned = await handleTabReturn(
sm,
{ session_id: "aa11", tab_id: 7 },
{
tabs: api,
windows,
cdp,
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
},
);
expect(returned).toMatchObject({ returned_to_window_id: 200 });
expect(ctx.borrowedTabs.has(7)).toBe(false);
expect(cdp.isAttached(7)).toBe(false);
expect(debuggerApi.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 7 });
cdp.dispose();
});
it("honours cancellation while preparing a validated return", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const state: FakeTabState = {
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
nextTabId: 50,
windowsClosed: new Set(),
};
const { api, spies } = makeTabMutationApi(state);
const { api: windows } = makeWindowsApi(state);
const controller = new AbortController();
const beforeReturn = vi.fn(async () => {
controller.abort();
});
const result = await handleTabReturn(
sm,
{ session_id: "aa11", tab_id: 7 },
{
tabs: api,
windows,
signal: controller.signal,
beforeReturn,
},
);
expect(result).toMatchObject({ code: "cancelled" });
expect(beforeReturn).toHaveBeenCalledExactlyOnceWith("aa11", 7);
expect(spies.move).not.toHaveBeenCalled();
expect(ctx.borrowedTabs.has(7)).toBe(true);
});
it.each([
"original",
"fallback",
"cleanup_failed",
"failed",
"cancelled",
])("releases CDP only after a successful return (%s)", async (mode) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const state: FakeTabState = {
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
nextTabId: 50,
windowsClosed: new Set(),
};
const { api, spies } = makeTabMutationApi(state);
const { api: windowsApi } = makeWindowsApi(state, { lastFocused: 500 });
const success = mode !== "failed" && mode !== "cancelled";
if (mode === "fallback") spies.move.mockRejectedValueOnce(new Error("original move failed"));
if (mode === "failed") spies.move.mockRejectedValue(new Error("move failed"));
let windowAtRelease: number | undefined;
const cdp = {
releaseSessionTab: vi.fn(async () => {
windowAtRelease = state.tabs.get(7)?.windowId;
if (mode === "cleanup_failed") throw new Error("debugger release failed");
}),
};
const controller = new AbortController();
if (mode === "cancelled") controller.abort();
const result = await handleTabReturn(
sm,
{ session_id: "aa11", tab_id: 7 },
{
tabs: api,
windows: windowsApi,
cdp,
signal: controller.signal,
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
},
);
expect("code" in result).toBe(!success);
expect(ctx.borrowedTabs.has(7)).toBe(!success);
expect(cdp.releaseSessionTab).toHaveBeenCalledTimes(success ? 1 : 0);
if (success) {
expect(cdp.releaseSessionTab).toHaveBeenCalledWith("aa11", 7);
expect(windowAtRelease).toBe(mode === "fallback" ? 500 : 200);
expect(spies.move).toHaveBeenCalledTimes(mode === "fallback" ? 2 : 1);
}
});
it("returns the tab to its original window/index and clears borrowedTabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -0,0 +1,208 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { RefStore, type VisualRefInput } from "@/session-manager/ref-store";
import { handleDownload } from "../download";
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
import { handleGetHtml, handleScreenshot } from "../observation";
import type { CdpRunner } from "../shared";
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
import { handleUpload } from "../upload";
function visual(): VisualRefInput {
return {
kind: "visual-region",
candidate: {
document: {
attachmentId: "a",
target: { tabId: 4, sessionId: "child" },
frameId: "frame",
documentElementBackendNodeId: 1,
},
backendNodeId: 42,
parentBackendNodeId: 1,
region: {
status: "available",
borderBox: { x: 0, y: 0, width: 100, height: 50 },
crop: { x: 0, y: 0, width: 100, height: 50 },
},
},
};
}
async function setup() {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("test");
ctx.refStore.replace([
["e1", visual()],
["e2", { backendNodeId: 43, tabId: 4 }],
]);
const send = vi.fn(async () => {
throw new Error("unexpected target operation");
});
const cdp = { send, sendToTarget: send, trackSessionTab: vi.fn() } as unknown as CdpRunner;
const tab = { id: 4, windowId: 100, active: true, url: "https://example.com" } as chrome.tabs.Tab;
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
return { manager, ctx, send, cdp, tabsApi };
}
describe("typed visual refs", () => {
afterEach(() => resetHelpLifecycleForTests());
it("keeps candidate evidence and cannot resolve a visual anchor as a bare DOM node", () => {
const store = new RefStore();
const input = visual();
store.replace([
["@e1", input],
["e2", { backendNodeId: 43, tabId: 4 }],
]);
const entry = store.resolveEntry("e1");
expect(entry?.kind).toBe("visual-region");
if (entry?.kind !== "visual-region") throw new Error("missing visual ref");
expect(entry.candidate).toBe(input.candidate);
expect(entry.generation).toBe(1);
expect(store.resolve("@e1")).toBeNull();
expect(store.resolve("e2")).toBe(43);
// The latest observation replaces both target kinds; eN strings carry no generation.
store.replace([["e1", { backendNodeId: 99, tabId: 4 }]]);
expect(store.resolve("e1")).toBe(99);
expect(store.resolveEntry("e1")).toMatchObject({ kind: "dom", generation: 2 });
expect(store.resolveEntry("e2")).toBeNull();
});
it("rejects missing visual identity without partially publishing a replacement", () => {
const store = new RefStore();
store.set("e1", 10, { tabId: 4 });
const invalid = visual();
invalid.candidate.document.attachmentId = "";
expect(() =>
store.replace([
["e2", 20],
["e3", invalid],
]),
).toThrow(TypeError);
expect(store.resolveEntry("e1")).toMatchObject({ generation: 0, backendNodeId: 10 });
expect(store.size()).toBe(1);
store.replace([["e1", visual()]]);
expect(store.resolveEntry("e1")?.generation).toBe(1);
});
it("isolates tabs/sessions and distinguishes unsupported targets from missing refs", async () => {
const { ctx } = await setup();
expect(lookupRefTarget(ctx, "@e1", 4)?.kind).toBe("visual-region");
expect(lookupRefTarget(ctx, "e1", 5)).toBeNull();
expect(lookupSnapshotRef(ctx, "e1", 4)).toBeNull();
expect(resolveSnapshotRef(ctx, "e1", 4)).toMatchObject({
code: "unsupported",
data: { reason: "ref_kind_unsupported" },
});
expect(resolveSnapshotRef(ctx, "e1", 5)).toMatchObject({
code: "not_found",
data: { reason: "ref_not_found" },
});
expect(lookupRefTarget({ ...ctx, refStore: new RefStore() }, "e1", 4)).toBeNull();
expect(lookupSnapshotRef(ctx, "e2", 4)).toMatchObject({ backendNodeId: 43 });
});
it.each([
"click",
"fill",
"hover",
"press",
"select",
"get_html",
"screenshot",
])("rejects visual refs in %s before target effects", async (tool) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
const deps = { cdp, tabsApi };
const captureVisibleTab = vi.fn();
let result: unknown;
switch (tool) {
case "click":
result = await handleClick(manager, params, deps);
break;
case "fill":
result = await handleFill(manager, { ...params, value: "hello" }, deps);
break;
case "hover":
result = await handleHover(manager, params, deps);
break;
case "press":
result = await handlePress(manager, { ...params, key: "Enter" }, deps);
break;
case "select":
result = await handleSelect(manager, { ...params, values: ["one"] }, deps);
break;
case "get_html":
result = await handleGetHtml(manager, params, deps);
break;
case "screenshot":
result = await handleScreenshot(manager, params, {
...deps,
captureApi: { ...tabsApi, captureVisibleTab },
});
break;
}
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
expect(captureVisibleTab).not.toHaveBeenCalled();
});
it.each([
"input",
"drop",
"download",
] as const)("rejects visual refs before %s file-transfer setup", async (mode) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "e1" };
const result =
mode === "download"
? await handleDownload(
manager,
{ ...params, browser_relative_dir: "test" },
{ cdp, tabsApi },
)
: await handleUpload(
manager,
{
...params,
mode,
files: [{ transfer_id: "test", name: "test.txt", staged_path: "/unused/test.txt" }],
},
{ cdp, tabsApi },
);
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
});
it("does not scroll or highlight a visual anchor in human help", async () => {
const { manager, cdp, tabsApi, send } = await setup();
const sendToTab = vi.fn(async (_tab: number, _message: unknown) => ({
type: "bsk-help-response",
outcome: "continued",
}));
const result = await handleRequestHelp(
manager,
{ session_id: "test", tab_id: 4, prompt: "help", targets: [{ ref: "@e1" }] },
{
cdp,
tabsApi,
sendToTab,
windows: { update: vi.fn(async () => ({}) as never) },
activateTab: vi.fn(async () => {}),
notifications: null,
autoAttachLifecycle: false,
},
);
expect(result).toMatchObject({ resolved_targets: [{ ref: "@e1", matched: false }] });
expect(send).not.toHaveBeenCalled();
expect(sendToTab.mock.calls[0][1]).toMatchObject({ rects: [], selectors: [] });
});
});

Some files were not shown because too many files have changed in this diff Show More