mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
216
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
90b0ff301b | ||
|
|
e74f354e90 | ||
|
|
05c036518a | ||
|
|
020fb3212e | ||
|
|
0822c53c65 | ||
|
|
4450fbe28c | ||
|
|
a3bb81bfab | ||
|
|
caea98cd52 | ||
|
|
187b088013 | ||
|
|
5046b5ab44 | ||
|
|
e8137b42cc | ||
|
|
cb5dc148d5 | ||
|
|
3c5f838c56 | ||
|
|
9337bd557b | ||
|
|
162a31ae92 | ||
|
|
2f84234a31 | ||
|
|
72c10aab75 | ||
|
|
1eeb55af02 | ||
|
|
4992a54864 | ||
|
|
8b195eb0df | ||
|
|
eae4db1729 | ||
|
|
e961395be9 | ||
|
|
a6d1d02acc | ||
|
|
5ebe609c40 | ||
|
|
321494d93f | ||
|
|
f2184e74c5 | ||
|
|
eeb633d68e | ||
|
|
6d181105f2 | ||
|
|
ebad0ed1b5 | ||
|
|
498f8fb4a7 | ||
|
|
cef59137e4 | ||
|
|
f70978bfcf | ||
|
|
1498cd425c | ||
|
|
e831d65db3 | ||
|
|
47ac947f18 | ||
|
|
69d64c4eb1 | ||
|
|
99c997d6d5 | ||
|
|
93853ebc65 | ||
|
|
248bb90a2d | ||
|
|
6e6c178434 | ||
|
|
386ef91c24 | ||
|
|
0dc11b30ea | ||
|
|
759eebd4b6 | ||
|
|
fd3ce202a1 | ||
|
|
618bd6a0df | ||
|
|
c455a09fc3 | ||
|
|
b6be356e40 | ||
|
|
945bf1523d | ||
|
|
a928a9fb76 | ||
|
|
0b2d7e95cc | ||
|
|
556bea1645 | ||
|
|
0302d5592b | ||
|
|
eb34ec0dbf | ||
|
|
43067b6bfe | ||
|
|
6cc0d38489 | ||
|
|
0b71963344 | ||
|
|
59e2c3224e | ||
|
|
d70df753f6 | ||
|
|
66ae555e40 | ||
|
|
fee7c0983c | ||
|
|
51473c01ff | ||
|
|
49744233f7 | ||
|
|
95571f8b70 | ||
|
|
8344e311fa | ||
|
|
63338cd98e | ||
|
|
bd193ceb5d | ||
|
|
5ed828736f | ||
|
|
ee620acf64 | ||
|
|
05a36075b8 | ||
|
|
1bd13d5c3c | ||
|
|
59d6533eff | ||
|
|
536d3a9844 | ||
|
|
a77353bce8 | ||
|
|
91621b2545 | ||
|
|
641d934337 | ||
|
|
a4fed136e3 | ||
|
|
3c77ef7b01 | ||
|
|
3b21f2f0a7 | ||
|
|
405433ab65 | ||
|
|
58c7f38a05 | ||
|
|
e0c88295c7 | ||
|
|
dc36e38cb9 | ||
|
|
d5de9b0154 | ||
|
|
995061514c | ||
|
|
406bb21b7e | ||
|
|
f06a514bd2 | ||
|
|
1e53ee63ae | ||
|
|
120dc441bb | ||
|
|
710b1219f4 | ||
|
|
177ecd3ebf | ||
|
|
0e6b9c4542 | ||
|
|
506adbe708 | ||
|
|
984d017b0f | ||
|
|
5df93143c5 | ||
|
|
7163f2f9b0 | ||
|
|
e9f0584af3 | ||
|
|
2997f495db | ||
|
|
f96bd72c3c | ||
|
|
329c68054b | ||
|
|
5cd43553ed | ||
|
|
9ad33801c7 | ||
|
|
7aaa5a2d01 | ||
|
|
210b21b637 | ||
|
|
48f64ec0df | ||
|
|
c48a53269e | ||
|
|
3bbd6ab19f | ||
|
|
1a70d2200e | ||
|
|
554861d6b1 | ||
|
|
0d7f976335 | ||
|
|
d1d3eb6a94 | ||
|
|
38fd7d3f4c | ||
|
|
b5062ac19b | ||
|
|
41cb576256 | ||
|
|
f060a7c074 | ||
|
|
60b62d5420 | ||
|
|
70b851a3e8 | ||
|
|
da3534d066 | ||
|
|
a004291848 | ||
|
|
b1b1ae1d48 | ||
|
|
8822bba9a5 | ||
|
|
c74c119a82 | ||
|
|
60fbf30e24 | ||
|
|
b801bc1f62 | ||
|
|
bf59753aa3 | ||
|
|
a86cbf724f | ||
|
|
76d447caa0 | ||
|
|
14e5bbe4fc | ||
|
|
bef722107f | ||
|
|
6800f8fdfe | ||
|
|
4b6cdde168 | ||
|
|
4aade4b94f | ||
|
|
616af1099f | ||
|
|
3656f0dd3c | ||
|
|
9742d83213 | ||
|
|
615e866608 | ||
|
|
c08fed5de3 | ||
|
|
887d1ed0b9 | ||
|
|
1a3d3b7de8 | ||
|
|
72834980d2 | ||
|
|
38b0debba9 | ||
|
|
4677149e5e | ||
|
|
8ea5477e11 | ||
|
|
17af1af65a | ||
|
|
6300eda0aa | ||
|
|
87cdcfe5ca | ||
|
|
21476c3884 | ||
|
|
c2a225377b | ||
|
|
e64f4cac9f | ||
|
|
368507b07d | ||
|
|
b750eed5d4 | ||
|
|
3d159c0254 | ||
|
|
524516cb56 | ||
|
|
f98198949a | ||
|
|
d05279f991 | ||
|
|
a4b6a61d29 | ||
|
|
cad0b5cbc9 | ||
|
|
2719e76438 | ||
|
|
dd9fc9edba | ||
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
8352c3c67a | ||
|
|
4eedd98c1d | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
6b78211e79 | ||
|
|
eebf334b40 | ||
|
|
a284ffc5ae | ||
|
|
e720f1ffe5 | ||
|
|
2b7e10ca2b | ||
|
|
1ef93463dd | ||
|
|
c7252529f4 | ||
|
|
0b65b4b295 | ||
|
|
94ec7e73fc | ||
|
|
42403e55f2 | ||
|
|
22019cdf07 | ||
|
|
8d7c0aa66c | ||
|
|
a9489282fa | ||
|
|
d4cddd6224 | ||
|
|
8e4de6cf4b | ||
|
|
fea66ce404 | ||
|
|
8bae85127a | ||
|
|
edb149611b | ||
|
|
14c5884cbc | ||
|
|
e7aad9bc16 | ||
|
|
3fb9e9f581 | ||
|
|
9854ee2f45 | ||
|
|
7035e46024 | ||
|
|
51085b960b | ||
|
|
51df868008 | ||
|
|
43ebfab7ba | ||
|
|
ce11325de2 | ||
|
|
4d1dcd122f | ||
|
|
a4e668542d | ||
|
|
52aa356153 | ||
|
|
d1001e14b2 | ||
|
|
eb71ddc1da | ||
|
|
b07b8fec65 | ||
|
|
47070fdc14 | ||
|
|
916229dc62 | ||
|
|
dde707b558 | ||
|
|
e2d07574d7 | ||
|
|
461d204ca3 | ||
|
|
bc5d9610a3 | ||
|
|
2955260d11 | ||
|
|
ce89e95211 | ||
|
|
24faa10659 | ||
|
|
a7bc8bacf6 | ||
|
|
3d050e0f26 | ||
|
|
afdc398730 | ||
|
|
6b96fcca32 | ||
|
|
0771292336 | ||
|
|
54869cabd0 | ||
|
|
2de7c5c6ff | ||
|
|
703fc64399 |
@@ -37,6 +37,35 @@ jobs:
|
|||||||
- name: Run Rust tests
|
- name: Run Rust tests
|
||||||
run: cargo test --workspace --locked
|
run: cargo test --workspace --locked
|
||||||
|
|
||||||
|
windows-daemon:
|
||||||
|
name: Windows daemon, installer and cancellation regression tests
|
||||||
|
runs-on: windows-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
|
||||||
|
- name: Run Windows named-pipe tests
|
||||||
|
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
|
||||||
|
|
||||||
|
- name: Run Windows process liveness tests
|
||||||
|
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
|
||||||
|
|
||||||
|
- name: Run Windows update tests
|
||||||
|
run: cargo test -p bsk --lib --locked cli::update::tests
|
||||||
|
|
||||||
|
- name: Run Windows update and cancellation process tests
|
||||||
|
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
|
||||||
|
|
||||||
|
- name: Run installer tests (Windows PowerShell 5.1)
|
||||||
|
shell: powershell
|
||||||
|
run: ./scripts/install-windows.test.ps1
|
||||||
|
|
||||||
|
- name: Run installer tests (PowerShell 7)
|
||||||
|
shell: pwsh
|
||||||
|
run: ./scripts/install-windows.test.ps1
|
||||||
|
|
||||||
frontend:
|
frontend:
|
||||||
name: Frontend lint, typecheck, tests, build
|
name: Frontend lint, typecheck, tests, build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
name: Release dsh plugin
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- dsh-plugin-v*
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: release-dsh-plugin-${{ github.ref }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||||
|
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
resolve:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
version: ${{ steps.version.outputs.version }}
|
||||||
|
tag: ${{ steps.version.outputs.tag }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Resolve plugin version
|
||||||
|
id: version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||||
|
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||||
|
else
|
||||||
|
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||||
|
fi
|
||||||
|
VERSION="${VERSION#v}"
|
||||||
|
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
guard:
|
||||||
|
needs: resolve
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Verify tag matches package.json name and version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||||
|
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||||
|
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||||
|
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||||
|
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||||
|
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||||
|
|
||||||
|
publish:
|
||||||
|
needs: [resolve, guard]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||||
|
environment: npm-publish
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- uses: pnpm/action-setup@v6
|
||||||
|
|
||||||
|
- uses: actions/setup-node@v6
|
||||||
|
with:
|
||||||
|
node-version: 22
|
||||||
|
cache: pnpm
|
||||||
|
registry-url: https://registry.npmjs.org
|
||||||
|
scope: "@wxg-prc-cpg"
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Typecheck plugin
|
||||||
|
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||||
|
|
||||||
|
- name: Test plugin
|
||||||
|
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||||
|
|
||||||
|
- name: Publish to npm
|
||||||
|
working-directory: ${{ env.PACKAGE_DIR }}
|
||||||
|
env:
|
||||||
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||||
|
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
pnpm publish --access public --no-git-checks
|
||||||
|
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||||
@@ -325,3 +325,180 @@ jobs:
|
|||||||
|
|
||||||
echo "Chrome Web Store submission created successfully:"
|
echo "Chrome Web Store submission created successfully:"
|
||||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||||
|
|
||||||
|
publish-edge-addons:
|
||||||
|
needs: [resolve, build-extension]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: edge-addons
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Download build artifact
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: extension-zip
|
||||||
|
path: dist
|
||||||
|
|
||||||
|
- name: Validate Microsoft Edge Add-ons configuration
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||||
|
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||||
|
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "$API_KEY" ]; then
|
||||||
|
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$CLIENT_ID" ]; then
|
||||||
|
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$PRODUCT_ID" ]; then
|
||||||
|
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
||||||
|
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||||
|
if [ -z "$ARCHIVE" ]; then
|
||||||
|
echo "::error::No extension zip found in the downloaded artifact"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Upload and publish Microsoft Edge Add-ons extension
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||||
|
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||||
|
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||||
|
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
||||||
|
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
||||||
|
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
||||||
|
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
||||||
|
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
||||||
|
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
||||||
|
|
||||||
|
operation_id_from_headers() {
|
||||||
|
local headers="$1"
|
||||||
|
local location
|
||||||
|
location="$(awk '
|
||||||
|
tolower($1) == "location:" {
|
||||||
|
sub(/\r$/, "")
|
||||||
|
sub(/^[^:]*:[[:space:]]*/, "")
|
||||||
|
print
|
||||||
|
exit
|
||||||
|
}
|
||||||
|
' "$headers")"
|
||||||
|
location="${location%%\?*}"
|
||||||
|
printf '%s' "${location##*/}"
|
||||||
|
}
|
||||||
|
|
||||||
|
poll_operation() {
|
||||||
|
local operation_name="$1"
|
||||||
|
local status_url="$2"
|
||||||
|
local status="InProgress"
|
||||||
|
local http_status
|
||||||
|
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
sleep 5
|
||||||
|
http_status="$(curl --silent --show-error \
|
||||||
|
--output "$STATUS_RESPONSE" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
"$status_url")"
|
||||||
|
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
||||||
|
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
||||||
|
echo "$operation_name status check $attempt/30: $status"
|
||||||
|
case "$status" in
|
||||||
|
Succeeded)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
InProgress)
|
||||||
|
;;
|
||||||
|
Failed)
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$UPLOAD_RESPONSE" \
|
||||||
|
--dump-header "$UPLOAD_HEADERS" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
--header "Content-Type: application/zip" \
|
||||||
|
--data-binary "@$ARCHIVE" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
||||||
|
if [ "$HTTP_STATUS" != "202" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
||||||
|
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
||||||
|
cat "$UPLOAD_HEADERS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
poll_operation \
|
||||||
|
"package upload" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$PUBLISH_RESPONSE" \
|
||||||
|
--dump-header "$PUBLISH_HEADERS" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
--header "Content-Type: application/json" \
|
||||||
|
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
||||||
|
if [ "$HTTP_STATUS" != "202" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
||||||
|
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
||||||
|
cat "$PUBLISH_HEADERS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
poll_operation \
|
||||||
|
"submission" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
||||||
|
|
||||||
|
echo "Microsoft Edge Add-ons submission created successfully:"
|
||||||
|
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
||||||
|
|||||||
@@ -7,10 +7,18 @@ dist/
|
|||||||
.output/
|
.output/
|
||||||
.wxt/
|
.wxt/
|
||||||
.web-extension-id
|
.web-extension-id
|
||||||
|
.pnpm-store/
|
||||||
|
|
||||||
|
# Git worktrees
|
||||||
|
.worktrees/
|
||||||
|
|
||||||
# Logs
|
# Logs
|
||||||
*.log
|
*.log
|
||||||
|
|
||||||
|
# Local browser evaluation artifacts
|
||||||
|
evals/browser/results/
|
||||||
|
evals/browser/agents.local.json
|
||||||
|
|
||||||
# macOS
|
# macOS
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
|||||||
@@ -3,3 +3,6 @@
|
|||||||
**/.output/**
|
**/.output/**
|
||||||
**/target/**
|
**/target/**
|
||||||
**/node_modules/**
|
**/node_modules/**
|
||||||
|
# Generated client CSS (tailwind compile output); built by
|
||||||
|
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||||
|
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||||
|
|||||||
+9
-5
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
|
|||||||
## 3. Open the extension install page
|
## 3. Open the extension install page
|
||||||
|
|
||||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||||
has not installed or enabled the browser extension yet. Open the
|
has not installed or enabled the browser extension yet. Open the install page
|
||||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
matching the user's browser —
|
||||||
in the user's browser, then tell the user:
|
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
for Chrome and other Chromium browsers,
|
||||||
|
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||||
|
for Microsoft Edge — then tell the user:
|
||||||
|
|
||||||
> Install **BrowserSkill** from the
|
> Install **BrowserSkill** from the
|
||||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
> open the popup, and wait until it turns green. Reply when done.
|
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||||
|
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||||
|
|
||||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||||
once more. All `ok`/`na` → tell the user it's ready.
|
once more. All `ok`/`na` → tell the user it's ready.
|
||||||
|
|||||||
+109
@@ -0,0 +1,109 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to BrowserSkill will be documented in this file.
|
||||||
|
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
||||||
|
|
||||||
|
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
|
||||||
|
|
||||||
|
## [0.2.1] - 2026-09-09
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
|
||||||
|
- CLI: Windows named-pipe connection continuity and stale daemon process detection
|
||||||
|
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
|
||||||
|
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
|
||||||
|
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
|
||||||
|
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
|
||||||
|
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
|
||||||
|
- Extension: match observed form state to the correct element after page changes
|
||||||
|
- Extension: UUID generation fallback for recording and overlays on insecure origins
|
||||||
|
- Browser locale matching for regional language variants
|
||||||
|
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
|
||||||
|
- DSH Plugin: removed the obsolete client runtime host dependency
|
||||||
|
|
||||||
|
## [0.2.0] - 2026-09-02
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- File transfer: upload and download support across CLI, Extension, and DSH Plugin
|
||||||
|
- File transfer: drag-and-drop upload (`drop-to-upload`)
|
||||||
|
- VOM semantic graph, name enrichment, and hover perception modules
|
||||||
|
- VOM hover probing (opt-in via `observe` parameter)
|
||||||
|
- DSH Plugin: browser tool parity with CLI commands
|
||||||
|
- Edge Add-ons automated publishing in CI
|
||||||
|
- Protocol upgrade reminder when CLI / Extension protocol versions differ
|
||||||
|
- Browser evaluation harness (`evals/browser/`)
|
||||||
|
- Unified release script (`scripts/release.mjs`)
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- **Version scheme**: all three components now share the same semver
|
||||||
|
- VOM rendering algorithm optimizations
|
||||||
|
- Leaner SKILL.md agent instructions
|
||||||
|
- DSH Plugin: simplified browser commands
|
||||||
|
- Borrow confirmation UX — proactive focus and longer timeout
|
||||||
|
- PiP window now has a close button
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Screenshot media type detection (was hard-coded to `image/png`)
|
||||||
|
- DSH Plugin session lifecycle stability
|
||||||
|
- DSH Plugin Cordis package ID mismatch
|
||||||
|
- Observation thumbnail media type sniffing
|
||||||
|
- Upload/download race conditions and layout bypass issues
|
||||||
|
- VOM repeated name and safety policy issues
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Previous releases used independent version numbers per component.*
|
||||||
|
|
||||||
|
## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- DSH Plugin sidebar integration, session lifecycle, and archive cleanup
|
||||||
|
- Recorder iframe and OOPIF support
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- VOM functional refactor
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Recorder safety policy and bug fixes
|
||||||
|
|
||||||
|
## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- VOM observation recording and settled-state detection
|
||||||
|
- Record overlay timer
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Browser keepalive disconnect handling
|
||||||
|
|
||||||
|
## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- CLI auto-update mechanism
|
||||||
|
- Trace v3 protocol and recorder
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- MV3 keepalive disconnect
|
||||||
|
|
||||||
|
## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- More browser interaction actions
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Windows named-pipe hash-only path issue
|
||||||
|
|
||||||
|
## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07
|
||||||
|
|
||||||
|
Initial public release pair.
|
||||||
|
|
||||||
|
## CLI 0.1.6 — 2026-06-30
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Minor CLI fixes
|
||||||
|
|
||||||
|
## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22
|
||||||
|
|
||||||
|
First tagged releases.
|
||||||
Generated
+3
-2
@@ -134,9 +134,10 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk"
|
name = "bsk"
|
||||||
version = "0.1.10"
|
version = "0.2.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"base64",
|
||||||
"bsk-protocol",
|
"bsk-protocol",
|
||||||
"clap",
|
"clap",
|
||||||
"console",
|
"console",
|
||||||
@@ -168,7 +169,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk-protocol"
|
name = "bsk-protocol"
|
||||||
version = "0.1.10"
|
version = "0.2.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"schemars",
|
"schemars",
|
||||||
"semver",
|
"semver",
|
||||||
|
|||||||
+3
-2
@@ -7,7 +7,7 @@ members = [
|
|||||||
default-members = ["crates/bsk-cli"]
|
default-members = ["crates/bsk-cli"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.1.10"
|
version = "0.2.1"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
@@ -36,13 +36,14 @@ flate2 = "1.1.9"
|
|||||||
tar = "0.4.46"
|
tar = "0.4.46"
|
||||||
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
||||||
sha2 = "0.11.0"
|
sha2 = "0.11.0"
|
||||||
|
base64 = "0.22"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
nix = { version = "0.29", features = ["signal", "process"] }
|
nix = { version = "0.29", features = ["signal", "process"] }
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
uuid = { version = "1", features = ["v4", "serde"] }
|
uuid = { version = "1", features = ["v4", "serde"] }
|
||||||
|
|
||||||
# Workspace crates (version required for `cargo publish` of dependents)
|
# Workspace crates (version required for `cargo publish` of dependents)
|
||||||
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
|
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
strip = "symbols"
|
strip = "symbols"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||||
browser.
|
browser.
|
||||||
|
|
||||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||||
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||||
|
|
||||||
#### 1. Install the `bsk` CLI
|
#### 1. Install the `bsk` CLI
|
||||||
|
|
||||||
@@ -91,7 +92,14 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. Install the browser extension
|
#### 2. Install the browser extension
|
||||||
|
|
||||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
Install BrowserSkill from your browser's store:
|
||||||
|
|
||||||
|
| Browser | Store listing |
|
||||||
|
| --- | --- |
|
||||||
|
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||||
|
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||||
|
|
||||||
|
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||||
|
|
||||||
#### 3. Install the skill
|
#### 3. Install the skill
|
||||||
|
|
||||||
@@ -123,7 +131,8 @@ internal variants and install paths.
|
|||||||
|
|
||||||
Other shell-capable agent harnesses are supported too. Copy
|
Other shell-capable agent harnesses are supported too. Copy
|
||||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||||
`browser-skill/SKILL.md` to install the skill manually.
|
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||||
|
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -133,6 +142,32 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## DeepSeek Harness plugin
|
||||||
|
|
||||||
|
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||||
|
BrowserSkill ships a first-class dsh plugin on npm as
|
||||||
|
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||||
|
It gives the agent native `browser_*` tools and a live view of its browser sessions
|
||||||
|
in the Web UI. The plugin runs `bsk` on the agent's behalf.
|
||||||
|
|
||||||
|
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
|
||||||
|
to a dsh profile and start it (replace `web` with your profile name):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||||
|
dsh --profile web
|
||||||
|
```
|
||||||
|
|
||||||
|
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
|
||||||
|
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||||
|
```
|
||||||
|
|
||||||
|
Restart the profile after upgrading. See the
|
||||||
|
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
|
||||||
|
|
||||||
## How It Works
|
## How It Works
|
||||||
|
|
||||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||||
@@ -166,7 +201,9 @@ flowchart TB
|
|||||||
|
|
||||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||||
browser task; the local daemon routes that request to the extension; the
|
browser task; the local daemon routes that request to the extension; the
|
||||||
extension runs it in an Agent Window.
|
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||||
|
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||||
|
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||||
|
|
||||||
## For Developers
|
## For Developers
|
||||||
|
|
||||||
@@ -176,6 +213,8 @@ The repository is a Cargo + pnpm workspace:
|
|||||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||||
- `apps/extension` — browser extension
|
- `apps/extension` — browser extension
|
||||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||||
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+39
-7
@@ -12,7 +12,7 @@
|
|||||||
<a href="README.md">English</a> · 中文
|
<a href="README.md">English</a> · 中文
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||||
|
|
||||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||||
|
|
||||||
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||||
|
|
||||||
#### 1. 安装 `bsk` CLI
|
#### 1. 安装 `bsk` CLI
|
||||||
|
|
||||||
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||||
```
|
```
|
||||||
|
|
||||||
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
||||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
|
||||||
|
```powershell
|
||||||
|
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||||
|
```
|
||||||
|
|
||||||
验证二进制:
|
验证二进制:
|
||||||
|
|
||||||
@@ -75,7 +79,14 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. 安装浏览器扩展
|
#### 2. 安装浏览器扩展
|
||||||
|
|
||||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
在对应浏览器的商店安装 BrowserSkill:
|
||||||
|
|
||||||
|
| 浏览器 | 商店页面 |
|
||||||
|
| --- | --- |
|
||||||
|
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||||
|
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||||
|
|
||||||
|
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||||
|
|
||||||
#### 3. 安装 skill
|
#### 3. 安装 skill
|
||||||
|
|
||||||
@@ -102,7 +113,7 @@ bsk install-skill
|
|||||||
|
|
||||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||||
|
|
||||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -112,6 +123,25 @@ bsk install-skill
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## DeepSeek Harness 插件
|
||||||
|
|
||||||
|
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具,由插件代为调用 `bsk`,并在 Web UI 中实时展示浏览器会话。
|
||||||
|
|
||||||
|
先安装 `bsk` CLI 并连接浏览器扩展,再将插件装进 dsh profile 并启动(将 `web` 替换为你的 profile 名称):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||||
|
dsh --profile web
|
||||||
|
```
|
||||||
|
|
||||||
|
插件自带 `browser-skill` skill,所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新;升级此插件请运行:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||||
|
```
|
||||||
|
|
||||||
|
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||||
|
|
||||||
## 工作原理
|
## 工作原理
|
||||||
|
|
||||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||||
@@ -143,7 +173,7 @@ flowchart TB
|
|||||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||||
```
|
```
|
||||||
|
|
||||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
||||||
|
|
||||||
## 面向开发者
|
## 面向开发者
|
||||||
|
|
||||||
@@ -153,6 +183,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
|||||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||||
- `apps/extension` — 浏览器扩展
|
- `apps/extension` — 浏览器扩展
|
||||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||||
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
||||||
|
|
||||||
## 许可证
|
## 许可证
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# BrowserSkill — Privacy Policy
|
# BrowserSkill — Privacy Policy
|
||||||
|
|
||||||
**Last updated:** May 25, 2026
|
**Last updated:** August 20, 2026
|
||||||
|
|
||||||
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
|
|||||||
|
|
||||||
## 2. Single Purpose
|
## 2. Single Purpose
|
||||||
|
|
||||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||||
|
|
||||||
## 3. Data the Extension Accesses
|
## 3. Data the Extension Accesses
|
||||||
|
|
||||||
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
|
|||||||
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
||||||
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
||||||
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
||||||
|
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
|
||||||
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
||||||
|
|
||||||
## 4. Data the Extension Does **Not** Collect
|
## 4. Data the Extension Does **Not** Collect
|
||||||
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
|
|||||||
|
|
||||||
- Send any data to remote servers, the Extension's authors, or any third party.
|
- Send any data to remote servers, the Extension's authors, or any third party.
|
||||||
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
||||||
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
|
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
|
||||||
- Use webcam, microphone, geolocation, or any device sensor.
|
- Use webcam, microphone, geolocation, or any device sensor.
|
||||||
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
||||||
- Track users across websites or across sessions.
|
- Track users across websites or across sessions.
|
||||||
@@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
|
|||||||
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
||||||
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
|
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
|
||||||
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
||||||
|
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
|
||||||
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
||||||
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
||||||
|
|
||||||
@@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi
|
|||||||
|
|
||||||
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
||||||
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
||||||
|
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
|
||||||
|
|
||||||
## 8. User Control
|
## 8. User Control
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>BrowserSkill file-transfer fixture</title>
|
||||||
|
<style>
|
||||||
|
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
|
||||||
|
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
|
||||||
|
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
|
||||||
|
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
|
||||||
|
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
|
||||||
|
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
|
||||||
|
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Upload then download the same image</h1>
|
||||||
|
<p id="status">No image selected</p>
|
||||||
|
<div class="actions">
|
||||||
|
<button id="upload" type="button">Upload through file input</button>
|
||||||
|
<input id="file" type="file" accept="image/png" hidden />
|
||||||
|
<a id="download" aria-disabled="true">Download uploaded image</a>
|
||||||
|
</div>
|
||||||
|
<div id="drop" role="button" tabindex="0">Drop an image here</div>
|
||||||
|
<img id="preview" alt="Uploaded image preview" hidden />
|
||||||
|
<script>
|
||||||
|
const input = document.querySelector('#file');
|
||||||
|
const upload = document.querySelector('#upload');
|
||||||
|
const drop = document.querySelector('#drop');
|
||||||
|
const download = document.querySelector('#download');
|
||||||
|
const preview = document.querySelector('#preview');
|
||||||
|
const status = document.querySelector('#status');
|
||||||
|
let objectUrl;
|
||||||
|
|
||||||
|
window.__bskTransferFixture = { events: [], result: null };
|
||||||
|
|
||||||
|
function accept(file, mechanism) {
|
||||||
|
if (!file) return;
|
||||||
|
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||||
|
objectUrl = URL.createObjectURL(file);
|
||||||
|
preview.src = objectUrl;
|
||||||
|
preview.hidden = false;
|
||||||
|
download.href = objectUrl;
|
||||||
|
download.download = file.name;
|
||||||
|
download.removeAttribute('aria-disabled');
|
||||||
|
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
|
||||||
|
window.__bskTransferFixture.result = {
|
||||||
|
mechanism,
|
||||||
|
name: file.name,
|
||||||
|
size: file.size,
|
||||||
|
type: file.type,
|
||||||
|
hasPathProperty: 'path' in file,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
upload.addEventListener('click', () => input.click());
|
||||||
|
input.addEventListener('change', () => accept(input.files[0], 'input'));
|
||||||
|
|
||||||
|
for (const type of ['dragenter', 'dragover']) {
|
||||||
|
drop.addEventListener(type, event => {
|
||||||
|
const types = [...event.dataTransfer.types];
|
||||||
|
window.__bskTransferFixture.events.push({ type, types });
|
||||||
|
if (!types.includes('Files')) return;
|
||||||
|
event.preventDefault();
|
||||||
|
drop.classList.add('dragging');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
|
||||||
|
drop.addEventListener('drop', event => {
|
||||||
|
window.__bskTransferFixture.events.push({
|
||||||
|
type: 'drop',
|
||||||
|
types: [...event.dataTransfer.types],
|
||||||
|
fileCount: event.dataTransfer.files.length,
|
||||||
|
});
|
||||||
|
event.preventDefault();
|
||||||
|
drop.classList.remove('dragging');
|
||||||
|
accept(event.dataTransfer.files[0], 'drop');
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@browser-skill/extension",
|
"name": "@browser-skill/extension",
|
||||||
"version": "0.1.6",
|
"version": "0.2.1",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||||
|
|
||||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||||
const listeners = new Set<(...args: TArgs) => void>();
|
const listeners = new Set<(...args: TArgs) => void>();
|
||||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeApi() {
|
function fakeApi() {
|
||||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||||
const api: CdpDebuggerApi = {
|
const api: CdpDebuggerApi = {
|
||||||
attach: vi.fn(async () => {}),
|
attach: vi.fn(async () => {}),
|
||||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("ChromiumCdp", () => {
|
describe("ChromiumCdp", () => {
|
||||||
|
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (
|
||||||
|
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||||
|
method: string,
|
||||||
|
params?: { frameId?: string },
|
||||||
|
) => {
|
||||||
|
if (method === "DOM.getFrameOwner") {
|
||||||
|
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||||
|
}
|
||||||
|
if (method !== "Page.getFrameTree") return {};
|
||||||
|
if (target.sessionId === "right-session") {
|
||||||
|
return {
|
||||||
|
frameTree: {
|
||||||
|
frame: { id: "right", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (target.sessionId === "nested-session") {
|
||||||
|
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
frameTree: {
|
||||||
|
frame: { id: "main", url: "https://app.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||||
|
{
|
||||||
|
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
await cdp.ensureAttached(4);
|
||||||
|
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||||
|
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||||
|
sessionId: "nested-session",
|
||||||
|
});
|
||||||
|
|
||||||
|
const graph = await cdp.getFrameGraph(4);
|
||||||
|
|
||||||
|
expect(graph.frames).toHaveLength(4);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||||
|
"right-session",
|
||||||
|
);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||||
|
"nested-session",
|
||||||
|
);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 4, sessionId: "right-session" },
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: "nested" },
|
||||||
|
);
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 4, sessionId: "nested-session" },
|
||||||
|
"Target.setAutoAttach",
|
||||||
|
expect.objectContaining({ flatten: true }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
const depth = 6;
|
||||||
|
const scheduled = new Set<string>();
|
||||||
|
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||||
|
frame: {
|
||||||
|
id: index === 0 ? "main" : `frame-${index}`,
|
||||||
|
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||||
|
},
|
||||||
|
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||||
|
});
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||||
|
if (method === "Target.setAutoAttach") {
|
||||||
|
const parentIndex = target.sessionId
|
||||||
|
? Number(target.sessionId.replace("session-", ""))
|
||||||
|
: 0;
|
||||||
|
const nextIndex = parentIndex + 1;
|
||||||
|
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||||
|
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||||
|
scheduled.add(key);
|
||||||
|
setTimeout(() => {
|
||||||
|
onEvent.fire(
|
||||||
|
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||||
|
"Target.attachedToTarget",
|
||||||
|
{
|
||||||
|
sessionId: `session-${nextIndex}`,
|
||||||
|
targetInfo: { type: "iframe" },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}, 5);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (method === "Page.getFrameTree") {
|
||||||
|
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||||
|
const index = Number(target.sessionId.replace("session-", ""));
|
||||||
|
return { frameTree: nestedTree(index) };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getFrameOwner") {
|
||||||
|
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
|
||||||
|
const graph = await cdp.getFrameGraph(4);
|
||||||
|
|
||||||
|
expect(graph.frames).toHaveLength(depth + 1);
|
||||||
|
for (let index = 1; index <= depth; index += 1) {
|
||||||
|
expect(
|
||||||
|
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||||
|
).toBe(`session-${index}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
let releaseAttach!: () => void;
|
let releaseAttach!: () => void;
|
||||||
@@ -125,6 +254,69 @@ describe("ChromiumCdp", () => {
|
|||||||
expect(cdp.isAttached(7)).toBe(false);
|
expect(cdp.isAttached(7)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
const attached = new Set<number>();
|
||||||
|
let finishDetach!: () => void;
|
||||||
|
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
|
||||||
|
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
|
||||||
|
attached.add(tabId!);
|
||||||
|
});
|
||||||
|
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
|
||||||
|
attached.delete(tabId!);
|
||||||
|
});
|
||||||
|
vi.mocked(api.detach).mockImplementationOnce(
|
||||||
|
({ tabId }) =>
|
||||||
|
new Promise<void>((resolve, reject) => {
|
||||||
|
finishDetach = () => {
|
||||||
|
attached.delete(tabId!);
|
||||||
|
if (rejectDetach) reject(new Error("tab already detached"));
|
||||||
|
else resolve();
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
cdp.trackSessionTab("aa11", 7);
|
||||||
|
await cdp.ensureAttached(7);
|
||||||
|
const returning = cdp.releaseSessionTab("aa11", 7);
|
||||||
|
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
|
||||||
|
let duplicateDetachFinished = false;
|
||||||
|
const duplicateDetach = cdp.detach(7).then(() => {
|
||||||
|
duplicateDetachFinished = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
cdp.trackSessionTab("bb22", 7);
|
||||||
|
// Register rejection handlers immediately; the pre-fix driver rejects
|
||||||
|
// both commands while the browser still has the previous attachment.
|
||||||
|
const pending = Promise.allSettled([
|
||||||
|
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
|
||||||
|
cdp.send(7, "DOM.getDocument"),
|
||||||
|
]);
|
||||||
|
await cdp.send(8, "DOM.getDocument");
|
||||||
|
const finishedEarly = duplicateDetachFinished;
|
||||||
|
finishDetach();
|
||||||
|
await Promise.all([returning, duplicateDetach]);
|
||||||
|
|
||||||
|
expect(await pending).toEqual([
|
||||||
|
{ status: "fulfilled", value: { ok: true } },
|
||||||
|
{ status: "fulfilled", value: { ok: true } },
|
||||||
|
]);
|
||||||
|
expect(finishedEarly).toBe(false);
|
||||||
|
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
expect(cdp.isAttached(7)).toBe(true);
|
||||||
|
expect(cdp.isAttached(8)).toBe(true);
|
||||||
|
await cdp.detachSession("bb22");
|
||||||
|
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||||
|
expect(cdp.isAttached(7)).toBe(false);
|
||||||
|
expect(attached.has(7)).toBe(false);
|
||||||
|
expect(cdp.isAttached(8)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("detachAll() iterates every cached tab", async () => {
|
it("detachAll() iterates every cached tab", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
const cdp = new ChromiumCdp(api);
|
const cdp = new ChromiumCdp(api);
|
||||||
@@ -522,6 +714,155 @@ describe("ChromiumCdp", () => {
|
|||||||
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
|
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("releases only the returning session's claim on a tab", async () => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
await cdp.ensureAttached(1);
|
||||||
|
await cdp.ensureAttached(2);
|
||||||
|
cdp.trackSessionTab("aa11", 1);
|
||||||
|
cdp.trackSessionTab("bb22", 1);
|
||||||
|
cdp.trackSessionTab("aa11", 2);
|
||||||
|
|
||||||
|
await cdp.releaseSessionTab("unknown", 1);
|
||||||
|
await cdp.releaseSessionTab("aa11", 1);
|
||||||
|
expect(api.detach).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
await cdp.releaseSessionTab("bb22", 1);
|
||||||
|
await cdp.releaseSessionTab("bb22", 1);
|
||||||
|
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
|
||||||
|
expect(cdp.isAttached(2)).toBe(true);
|
||||||
|
|
||||||
|
await cdp.detachSession("aa11");
|
||||||
|
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||||
|
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
let finishAttach!: () => void;
|
||||||
|
vi.mocked(api.attach).mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise<void>((resolve) => {
|
||||||
|
finishAttach = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
cdp.trackSessionTab("aa11", 1);
|
||||||
|
const attaching = cdp.ensureAttached(1);
|
||||||
|
const releasing = cdp.releaseSessionTab("aa11", 1);
|
||||||
|
if (newOwner) cdp.trackSessionTab("bb22", 1);
|
||||||
|
finishAttach();
|
||||||
|
await Promise.all([attaching, releasing]);
|
||||||
|
|
||||||
|
expect(cdp.isAttached(1)).toBe(newOwner);
|
||||||
|
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
let controlled = true;
|
||||||
|
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
|
||||||
|
cdp.trackSessionTab("aa11", 1);
|
||||||
|
cdp.trackSessionTab("bb22", 1);
|
||||||
|
await cdp.ensureAttached(1);
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||||
|
type: "confirm",
|
||||||
|
message: "before return",
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
|
||||||
|
vi.mocked(api.sendCommand).mockClear();
|
||||||
|
|
||||||
|
controlled = false;
|
||||||
|
await cdp.releaseSessionTab("aa11", 1);
|
||||||
|
expect(cdp.isAttached(1)).toBe(true);
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||||
|
type: "confirm",
|
||||||
|
message: "after return",
|
||||||
|
});
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||||
|
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
|
||||||
|
|
||||||
|
controlled = true;
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||||
|
type: "alert",
|
||||||
|
message: "borrowed again",
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
|
||||||
|
{ tabId: 1 },
|
||||||
|
"Page.handleJavaScriptDialog",
|
||||||
|
{ accept: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores dialog events after detach, including a pending eligibility check", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
let resolveEligibility!: (value: boolean) => void;
|
||||||
|
const shouldAutoAcceptDialog = vi.fn(
|
||||||
|
() =>
|
||||||
|
new Promise<boolean>((resolve) => {
|
||||||
|
resolveEligibility = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||||
|
await cdp.ensureAttached(1);
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||||
|
await cdp.detach(1);
|
||||||
|
resolveEligibility(true);
|
||||||
|
await Promise.resolve();
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||||
|
|
||||||
|
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
|
||||||
|
expect(api.sendCommand).not.toHaveBeenCalledWith(
|
||||||
|
expect.anything(),
|
||||||
|
"Page.handleJavaScriptDialog",
|
||||||
|
expect.anything(),
|
||||||
|
);
|
||||||
|
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
const shouldAutoAcceptDialog = vi.fn(async () => {
|
||||||
|
throw new Error("tab is gone");
|
||||||
|
});
|
||||||
|
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||||
|
await cdp.ensureAttached(1);
|
||||||
|
vi.mocked(api.sendCommand).mockClear();
|
||||||
|
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
|
||||||
|
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||||
|
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not restore dialog state when an acceptance completes after detach", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
await cdp.ensureAttached(1);
|
||||||
|
let finishAccept!: () => void;
|
||||||
|
vi.mocked(api.sendCommand).mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise<void>((resolve) => {
|
||||||
|
finishAccept = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||||
|
await cdp.detach(1);
|
||||||
|
finishAccept();
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||||
|
expect(cdp.dialogCursor(1)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
it("detachSession only detaches tabs no other session owns", async () => {
|
it("detachSession only detaches tabs no other session owns", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
const cdp = new ChromiumCdp(api);
|
const cdp = new ChromiumCdp(api);
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||||
|
|
||||||
|
describe("buildFrameGraph", () => {
|
||||||
|
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||||
|
const graph = buildFrameGraph([
|
||||||
|
{
|
||||||
|
target: { tabId: 4 },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "main", url: "https://app.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||||
|
{
|
||||||
|
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "right", url: "https://right.test" },
|
||||||
|
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "nested-session" },
|
||||||
|
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(graph?.frames).toHaveLength(4);
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
});
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||||
|
parentFrameId: "right",
|
||||||
|
target: { tabId: 4, sessionId: "nested-session" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("propagates each target boundary to its same-process descendants", () => {
|
||||||
|
const sources: CdpFrameTreeSource[] = [
|
||||||
|
{
|
||||||
|
target: { tabId: 4 },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "main" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "oopif-a", parentId: "main" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||||
|
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "session-a" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "oopif-a" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||||
|
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "session-c" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "oopif-c" },
|
||||||
|
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const graph = buildFrameGraph(sources);
|
||||||
|
|
||||||
|
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||||
|
["main", undefined],
|
||||||
|
["oopif-a", "session-a"],
|
||||||
|
["same-process-b", "session-a"],
|
||||||
|
["oopif-c", "session-c"],
|
||||||
|
["same-process-d", "session-c"],
|
||||||
|
]);
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||||
|
"same-process-b",
|
||||||
|
);
|
||||||
|
|
||||||
|
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||||
|
expect(reordered).toEqual(graph);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||||
|
const depth = 5_000;
|
||||||
|
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||||
|
let parent = root;
|
||||||
|
for (let index = 1; index <= depth; index += 1) {
|
||||||
|
const child: CdpFrameTreeNode = {
|
||||||
|
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||||
|
};
|
||||||
|
parent.childFrames = [child];
|
||||||
|
parent = child;
|
||||||
|
}
|
||||||
|
|
||||||
|
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||||
|
|
||||||
|
expect(graph?.frames).toHaveLength(depth + 1);
|
||||||
|
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||||
|
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||||
|
root.childFrames = [root];
|
||||||
|
|
||||||
|
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -31,6 +31,15 @@ import type {
|
|||||||
NetworkEntryKind,
|
NetworkEntryKind,
|
||||||
NetworkResult,
|
NetworkResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
buildFrameGraph,
|
||||||
|
type CdpFrameGraph,
|
||||||
|
type CdpFrameTreeNode,
|
||||||
|
type CdpFrameTreeSource,
|
||||||
|
type CdpTarget,
|
||||||
|
} from "./frame-graph";
|
||||||
|
|
||||||
|
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||||
@@ -38,21 +47,15 @@ import type {
|
|||||||
* fake without monkey-patching the real `chrome` global.
|
* fake without monkey-patching the real `chrome` global.
|
||||||
*/
|
*/
|
||||||
export interface CdpDebuggerApi {
|
export interface CdpDebuggerApi {
|
||||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
detach(target: CdpDebuggee): Promise<void>;
|
||||||
sendCommand(
|
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||||
target: chrome.debugger.Debuggee,
|
|
||||||
method: string,
|
|
||||||
commandParams?: object,
|
|
||||||
): Promise<unknown>;
|
|
||||||
/**
|
/**
|
||||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||||
* …). The first callback argument is the source debuggee; the second
|
* …). The first callback argument is the source debuggee; the second
|
||||||
* is the CDP method name; the third is the payload.
|
* is the CDP method name; the third is the payload.
|
||||||
*/
|
*/
|
||||||
onEvent: chrome.events.Event<
|
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
|
||||||
>;
|
|
||||||
/**
|
/**
|
||||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||||
* the tab navigated to a chrome:// URL or the user clicked
|
* the tab navigated to a chrome:// URL or the user clicked
|
||||||
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
|||||||
const MAX_NETWORK_BUFFER = 200;
|
const MAX_NETWORK_BUFFER = 200;
|
||||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||||
const MAX_NETWORK_REQUEST_META = 1024;
|
const MAX_NETWORK_REQUEST_META = 1024;
|
||||||
|
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||||
|
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||||
|
|
||||||
interface ParsedDialogOpening {
|
interface ParsedDialogOpening {
|
||||||
type: JavaScriptDialogType;
|
type: JavaScriptDialogType;
|
||||||
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
|
|||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FrameDiscoveryState {
|
||||||
|
sessions: Set<string>;
|
||||||
|
pending: Set<Promise<void>>;
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||||
|
if (promises.length === 0) return true;
|
||||||
|
const remaining = deadline - Date.now();
|
||||||
|
if (remaining <= 0) return false;
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const timedOut = await Promise.race([
|
||||||
|
Promise.allSettled(promises).then(() => false),
|
||||||
|
new Promise<boolean>((resolve) => {
|
||||||
|
timer = setTimeout(() => resolve(true), remaining);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
return !timedOut;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||||
* tabId" cache and exposes typed `send<T>()`.
|
* tabId" cache and exposes typed `send<T>()`.
|
||||||
@@ -139,6 +165,7 @@ export class ChromiumCdp {
|
|||||||
private readonly api: CdpDebuggerApi;
|
private readonly api: CdpDebuggerApi;
|
||||||
private readonly attachedTabs = new Set<number>();
|
private readonly attachedTabs = new Set<number>();
|
||||||
private readonly attachInFlight = new Map<number, Promise<void>>();
|
private readonly attachInFlight = new Map<number, Promise<void>>();
|
||||||
|
private readonly detachInFlight = new Map<number, Promise<void>>();
|
||||||
private readonly tabOwners = new Map<number, Set<string>>();
|
private readonly tabOwners = new Map<number, Set<string>>();
|
||||||
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
|
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
|
||||||
private readonly dialogSequences = new Map<number, number>();
|
private readonly dialogSequences = new Map<number, number>();
|
||||||
@@ -149,21 +176,34 @@ export class ChromiumCdp {
|
|||||||
private readonly networkSequences = new Map<number, number>();
|
private readonly networkSequences = new Map<number, number>();
|
||||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||||
|
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||||
private detachSubscription: { dispose(): void } | null = null;
|
private detachSubscription: { dispose(): void } | null = null;
|
||||||
private dialogSubscription: { dispose(): void } | null = null;
|
private dialogSubscription: { dispose(): void } | null = null;
|
||||||
private consoleSubscription: { dispose(): void } | null = null;
|
private consoleSubscription: { dispose(): void } | null = null;
|
||||||
private networkSubscription: { dispose(): void } | null = null;
|
private networkSubscription: { dispose(): void } | null = null;
|
||||||
|
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||||
|
|
||||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
constructor(
|
||||||
|
api: CdpDebuggerApi = chromeDebuggerApi,
|
||||||
|
private readonly options: {
|
||||||
|
/** CDP observation alone does not authorize dismissing user dialogs. */
|
||||||
|
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
this.api = api;
|
this.api = api;
|
||||||
this.bindAutoDetach();
|
this.bindAutoDetach();
|
||||||
this.bindDialogHandler();
|
this.bindDialogHandler();
|
||||||
this.bindConsoleHandler();
|
this.bindConsoleHandler();
|
||||||
this.bindNetworkHandler();
|
this.bindNetworkHandler();
|
||||||
|
this.bindFrameTargetHandler();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Attach to `tabId` if we haven't already in this driver. */
|
/** Attach to `tabId` if we haven't already in this driver. */
|
||||||
async ensureAttached(tabId: number): Promise<void> {
|
async ensureAttached(tabId: number): Promise<void> {
|
||||||
|
// Returning a tab clears the cache before Chrome finishes detaching.
|
||||||
|
// New observers must wait before opening the next connection to that tab.
|
||||||
|
const detaching = this.detachInFlight.get(tabId);
|
||||||
|
if (detaching) await detaching;
|
||||||
if (this.attachedTabs.has(tabId)) return;
|
if (this.attachedTabs.has(tabId)) return;
|
||||||
const existing = this.attachInFlight.get(tabId);
|
const existing = this.attachInFlight.get(tabId);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
@@ -177,6 +217,9 @@ export class ChromiumCdp {
|
|||||||
await this.enableConsoleDomains(tabId);
|
await this.enableConsoleDomains(tabId);
|
||||||
await this.enableNetworkDomainBestEffort(tabId);
|
await this.enableNetworkDomainBestEffort(tabId);
|
||||||
this.attachedTabs.add(tabId);
|
this.attachedTabs.add(tabId);
|
||||||
|
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||||
|
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// A CDP domain enable failed after the raw attach succeeded
|
// A CDP domain enable failed after the raw attach succeeded
|
||||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||||
@@ -222,6 +265,73 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||||
|
if (!this.attachedTabs.has(target.tabId)) {
|
||||||
|
await this.ensureAttached(target.tabId);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||||
|
} catch (err) {
|
||||||
|
throw normalizeError(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||||
|
await this.ensureAttached(tabId);
|
||||||
|
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||||
|
await this.drainFrameAttachTasks(tabId);
|
||||||
|
|
||||||
|
const sources: CdpFrameTreeSource[] = [];
|
||||||
|
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||||
|
{ tabId },
|
||||||
|
"Page.getFrameTree",
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||||
|
|
||||||
|
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||||
|
const childTrees = await Promise.all(
|
||||||
|
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||||
|
const target = { tabId, sessionId };
|
||||||
|
try {
|
||||||
|
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||||
|
target,
|
||||||
|
"Page.getFrameTree",
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
for (const source of childTrees) {
|
||||||
|
if (source) sources.push(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
const graph = buildFrameGraph(sources);
|
||||||
|
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||||
|
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
await Promise.all(
|
||||||
|
graph.frames.map(async (frame) => {
|
||||||
|
if (!frame.parentFrameId) return;
|
||||||
|
const parent = frameById.get(frame.parentFrameId);
|
||||||
|
if (!parent) return;
|
||||||
|
try {
|
||||||
|
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||||
|
parent.target,
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: frame.frameId },
|
||||||
|
);
|
||||||
|
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||||
|
} catch {
|
||||||
|
// The frame may have navigated between tree capture and owner lookup.
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return graph;
|
||||||
|
}
|
||||||
|
|
||||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||||
dialogCursor(tabId: number): DialogCursor {
|
dialogCursor(tabId: number): DialogCursor {
|
||||||
return this.dialogSequences.get(tabId) ?? 0;
|
return this.dialogSequences.get(tabId) ?? 0;
|
||||||
@@ -326,19 +436,31 @@ export class ChromiumCdp {
|
|||||||
|
|
||||||
/** Detach if attached; never throws. */
|
/** Detach if attached; never throws. */
|
||||||
async detach(tabId: number): Promise<void> {
|
async detach(tabId: number): Promise<void> {
|
||||||
|
const existing = this.detachInFlight.get(tabId);
|
||||||
|
if (existing) {
|
||||||
|
await existing;
|
||||||
|
return;
|
||||||
|
}
|
||||||
this.attachInFlight.delete(tabId);
|
this.attachInFlight.delete(tabId);
|
||||||
if (!this.attachedTabs.has(tabId)) return;
|
if (!this.attachedTabs.has(tabId)) return;
|
||||||
this.attachedTabs.delete(tabId);
|
this.attachedTabs.delete(tabId);
|
||||||
this.clearDialogState(tabId);
|
this.clearDialogState(tabId);
|
||||||
this.clearConsoleState(tabId);
|
this.clearConsoleState(tabId);
|
||||||
this.clearNetworkState(tabId);
|
this.clearNetworkState(tabId);
|
||||||
try {
|
this.clearFrameState(tabId);
|
||||||
await this.api.detach({ tabId });
|
const detach = (async () => {
|
||||||
} catch (err) {
|
try {
|
||||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
await this.api.detach({ tabId });
|
||||||
// at debug so production builds aren't noisy.
|
} catch (err) {
|
||||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||||
}
|
// at debug so production builds aren't noisy.
|
||||||
|
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||||
|
}
|
||||||
|
})().finally(() => {
|
||||||
|
this.detachInFlight.delete(tabId);
|
||||||
|
});
|
||||||
|
this.detachInFlight.set(tabId, detach);
|
||||||
|
await detach;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
|
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
|
||||||
@@ -353,8 +475,19 @@ export class ChromiumCdp {
|
|||||||
this.tabOwners.set(tabId, owners);
|
this.tabOwners.set(tabId, owners);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Release one session's claim, preserving attachments still used by another. */
|
||||||
|
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
|
||||||
|
const owners = this.tabOwners.get(tabId);
|
||||||
|
if (!owners?.delete(sessionId) || owners.size > 0) return;
|
||||||
|
this.tabOwners.delete(tabId);
|
||||||
|
// An observation may still be attaching when the tab is returned. Wait
|
||||||
|
// for it so detach cannot miss the attachment or remove a new owner's claim.
|
||||||
|
await this.attachInFlight.get(tabId)?.catch(() => {});
|
||||||
|
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
} {
|
} {
|
||||||
this.api.onEvent.addListener(handler);
|
this.api.onEvent.addListener(handler);
|
||||||
@@ -378,6 +511,7 @@ export class ChromiumCdp {
|
|||||||
this.networkSequences.clear();
|
this.networkSequences.clear();
|
||||||
this.networkDomainsEnabledTabs.clear();
|
this.networkDomainsEnabledTabs.clear();
|
||||||
this.networkRequestMeta.clear();
|
this.networkRequestMeta.clear();
|
||||||
|
this.frameDiscovery.clear();
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
tabs.map(async (tabId) => {
|
tabs.map(async (tabId) => {
|
||||||
try {
|
try {
|
||||||
@@ -428,9 +562,104 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||||
|
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||||
|
autoAttach: true,
|
||||||
|
waitForDebuggerOnStart: false,
|
||||||
|
flatten: true,
|
||||||
|
filter: [{ type: "iframe", exclude: false }],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private bindFrameTargetHandler(): void {
|
||||||
|
if (this.frameTargetSubscription) return;
|
||||||
|
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||||
|
const tabId = source.tabId;
|
||||||
|
if (typeof tabId !== "number") return;
|
||||||
|
const raw = (params ?? {}) as Record<string, unknown>;
|
||||||
|
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||||
|
if (!sessionId) return;
|
||||||
|
|
||||||
|
if (method === "Target.detachedFromTarget") {
|
||||||
|
const state = this.frameDiscovery.get(tabId);
|
||||||
|
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (method !== "Target.attachedToTarget") return;
|
||||||
|
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||||
|
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||||
|
|
||||||
|
const state = this.frameDiscoveryState(tabId);
|
||||||
|
if (state.sessions.has(sessionId)) return;
|
||||||
|
state.sessions.add(sessionId);
|
||||||
|
state.generation += 1;
|
||||||
|
|
||||||
|
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||||
|
state.pending.add(task);
|
||||||
|
void task.finally(() => {
|
||||||
|
state.pending.delete(task);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
this.api.onEvent.addListener(listener);
|
||||||
|
this.frameTargetSubscription = {
|
||||||
|
dispose: () => this.api.onEvent.removeListener(listener),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.enableFrameDiscovery(target);
|
||||||
|
} catch (err) {
|
||||||
|
const state = this.frameDiscovery.get(target.tabId);
|
||||||
|
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||||
|
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||||
|
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
const state = this.frameDiscovery.get(tabId);
|
||||||
|
const generation = state?.generation ?? 0;
|
||||||
|
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||||
|
|
||||||
|
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||||
|
// its command promise settles. Wait for a short quiet window, then finish
|
||||||
|
// only if neither the state object nor its generation changed.
|
||||||
|
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||||
|
if (quietTime <= 0) break;
|
||||||
|
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||||
|
const current = this.frameDiscovery.get(tabId);
|
||||||
|
if (
|
||||||
|
current === state &&
|
||||||
|
(current?.generation ?? 0) === generation &&
|
||||||
|
!current?.pending.size
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||||
|
}
|
||||||
|
|
||||||
|
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||||
|
const existing = this.frameDiscovery.get(tabId);
|
||||||
|
if (existing) return existing;
|
||||||
|
const created: FrameDiscoveryState = {
|
||||||
|
sessions: new Set(),
|
||||||
|
pending: new Set(),
|
||||||
|
generation: 0,
|
||||||
|
};
|
||||||
|
this.frameDiscovery.set(tabId, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearFrameState(tabId: number): void {
|
||||||
|
this.frameDiscovery.delete(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
private bindDialogHandler(): void {
|
private bindDialogHandler(): void {
|
||||||
if (this.dialogSubscription) return;
|
if (this.dialogSubscription) return;
|
||||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||||
if (method !== "Page.javascriptDialogOpening") return;
|
if (method !== "Page.javascriptDialogOpening") return;
|
||||||
const tabId = source.tabId;
|
const tabId = source.tabId;
|
||||||
if (typeof tabId !== "number") return;
|
if (typeof tabId !== "number") return;
|
||||||
@@ -443,13 +672,23 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
|
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
|
||||||
|
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||||
const parsed = parseDialogOpeningParams(params);
|
const parsed = parseDialogOpeningParams(params);
|
||||||
try {
|
try {
|
||||||
|
if (
|
||||||
|
this.options.shouldAutoAcceptDialog &&
|
||||||
|
!(await this.options.shouldAutoAcceptDialog(tabId))
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// The tab may have been returned while its current scope was checked.
|
||||||
|
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||||
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
|
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
|
||||||
if (parsed.type === "prompt") {
|
if (parsed.type === "prompt") {
|
||||||
handleParams.promptText = parsed.defaultPrompt ?? "";
|
handleParams.promptText = parsed.defaultPrompt ?? "";
|
||||||
}
|
}
|
||||||
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
|
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
|
||||||
|
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||||
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
|
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
|
||||||
this.dialogSequences.set(tabId, sequence);
|
this.dialogSequences.set(tabId, sequence);
|
||||||
this.appendDialog(tabId, {
|
this.appendDialog(tabId, {
|
||||||
@@ -614,6 +853,7 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(source.tabId);
|
this.clearDialogState(source.tabId);
|
||||||
this.clearConsoleState(source.tabId);
|
this.clearConsoleState(source.tabId);
|
||||||
this.clearNetworkState(source.tabId);
|
this.clearNetworkState(source.tabId);
|
||||||
|
this.clearFrameState(source.tabId);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
this.api.onDetach.addListener(listener);
|
this.api.onDetach.addListener(listener);
|
||||||
@@ -632,19 +872,15 @@ export class ChromiumCdp {
|
|||||||
this.consoleSubscription = null;
|
this.consoleSubscription = null;
|
||||||
this.networkSubscription?.dispose();
|
this.networkSubscription?.dispose();
|
||||||
this.networkSubscription = null;
|
this.networkSubscription = null;
|
||||||
|
this.frameTargetSubscription?.dispose();
|
||||||
|
this.frameTargetSubscription = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Detach tabs only when no other live session has claimed them. */
|
/** Detach tabs only when no other live session has claimed them. */
|
||||||
async detachSession(sessionId: string): Promise<void> {
|
async detachSession(sessionId: string): Promise<void> {
|
||||||
const tabsToDetach: number[] = [];
|
await Promise.all(
|
||||||
for (const [tabId, owners] of this.tabOwners) {
|
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
|
||||||
owners.delete(sessionId);
|
);
|
||||||
if (owners.size === 0) {
|
|
||||||
this.tabOwners.delete(tabId);
|
|
||||||
tabsToDetach.push(tabId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
export interface CdpTarget {
|
||||||
|
tabId: number;
|
||||||
|
sessionId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrame {
|
||||||
|
frameId: string;
|
||||||
|
parentFrameId?: string;
|
||||||
|
ownerBackendNodeId?: number;
|
||||||
|
url?: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameGraph {
|
||||||
|
rootFrameId: string;
|
||||||
|
frames: CdpFrame[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameTreeNode {
|
||||||
|
frame: {
|
||||||
|
id: string;
|
||||||
|
parentId?: string;
|
||||||
|
url?: string;
|
||||||
|
};
|
||||||
|
childFrames?: CdpFrameTreeNode[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameTreeSource {
|
||||||
|
target: CdpTarget;
|
||||||
|
tree: CdpFrameTreeNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cdpTargetKey(target: CdpTarget): string {
|
||||||
|
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeFrame(
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
order: string[],
|
||||||
|
node: CdpFrameTreeNode,
|
||||||
|
target: CdpTarget,
|
||||||
|
): void {
|
||||||
|
const existing = frames.get(node.frame.id);
|
||||||
|
if (!existing) order.push(node.frame.id);
|
||||||
|
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||||
|
const url = node.frame.url || existing?.url;
|
||||||
|
frames.set(node.frame.id, {
|
||||||
|
frameId: node.frame.id,
|
||||||
|
...(parentFrameId ? { parentFrameId } : {}),
|
||||||
|
...(url ? { url } : {}),
|
||||||
|
target,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function walkFrameTree(
|
||||||
|
source: CdpFrameTreeSource,
|
||||||
|
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
order: string[],
|
||||||
|
): void {
|
||||||
|
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||||
|
{ node: source.tree, inheritedTarget: source.target },
|
||||||
|
];
|
||||||
|
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||||
|
|
||||||
|
while (stack.length > 0) {
|
||||||
|
const current = stack.pop();
|
||||||
|
if (!current || expanded.has(current.node)) continue;
|
||||||
|
expanded.add(current.node);
|
||||||
|
|
||||||
|
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||||
|
mergeFrame(frames, order, current.node, target);
|
||||||
|
|
||||||
|
const children = current.node.childFrames ?? [];
|
||||||
|
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||||
|
stack.push({ node: children[index], inheritedTarget: target });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||||
|
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||||
|
if (!rootSource) return null;
|
||||||
|
|
||||||
|
const frames = new Map<string, CdpFrame>();
|
||||||
|
const order: string[] = [];
|
||||||
|
const targetByRootFrameId = new Map(
|
||||||
|
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||||
|
);
|
||||||
|
// The top-level source is authoritative if malformed input reports the same
|
||||||
|
// frame as both a root target and a child target.
|
||||||
|
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||||
|
|
||||||
|
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||||
|
for (const source of sources) {
|
||||||
|
if (source === rootSource) continue;
|
||||||
|
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
rootFrameId: rootSource.tree.frame.id,
|
||||||
|
frames: order.flatMap((frameId) => {
|
||||||
|
const frame = frames.get(frameId);
|
||||||
|
return frame ? [frame] : [];
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
useRef,
|
useRef,
|
||||||
useState,
|
useState,
|
||||||
} from "react";
|
} from "react";
|
||||||
|
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||||
import logoUrl from "../../assets/logo.png";
|
import logoUrl from "../../assets/logo.png";
|
||||||
|
|
||||||
export interface HelpRequestData {
|
export interface HelpRequestData {
|
||||||
@@ -19,6 +20,10 @@ export interface HelpRequestData {
|
|||||||
displayMode?: "full" | "compact";
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight. */
|
/** CSS selectors to scroll to + flash-highlight. */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||||
|
rects?: HelpHighlightRect[];
|
||||||
|
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||||
|
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||||
onContinue: (note: string) => void;
|
onContinue: (note: string) => void;
|
||||||
onCancel: () => void;
|
onCancel: () => void;
|
||||||
}
|
}
|
||||||
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
|
|||||||
const FALLBACK_PANEL_H = 180;
|
const FALLBACK_PANEL_H = 180;
|
||||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||||
const EMPTY_SELECTORS: string[] = [];
|
const EMPTY_SELECTORS: string[] = [];
|
||||||
|
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||||
|
|
||||||
function clamp(value: number, min: number, max: number): number {
|
function clamp(value: number, min: number, max: number): number {
|
||||||
return Math.max(min, Math.min(max, value));
|
return Math.max(min, Math.min(max, value));
|
||||||
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const isCompact = request?.displayMode === "compact";
|
const isCompact = request?.displayMode === "compact";
|
||||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||||
|
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||||
|
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||||
const [note, setNote] = useState("");
|
const [note, setNote] = useState("");
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||||
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
// Reset the note and collapse state whenever a new request appears.
|
// Reset the note and collapse state whenever a new request appears.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setNote("");
|
setNote("");
|
||||||
|
setLiveRects(null);
|
||||||
setCollapsed(false);
|
setCollapsed(false);
|
||||||
placementRef.current = null;
|
placementRef.current = null;
|
||||||
setDragPos(null);
|
setDragPos(null);
|
||||||
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
dragStartRef.current = null;
|
dragStartRef.current = null;
|
||||||
}, [request?.id]);
|
}, [request?.id]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!request || isCompact || !request.refreshRects) return;
|
||||||
|
let disposed = false;
|
||||||
|
let resolving = false;
|
||||||
|
const refresh = async () => {
|
||||||
|
if (resolving) return;
|
||||||
|
resolving = true;
|
||||||
|
try {
|
||||||
|
const rects = await request.refreshRects?.();
|
||||||
|
if (!disposed && rects) setLiveRects(rects);
|
||||||
|
} finally {
|
||||||
|
resolving = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const onViewportChange = () => void refresh();
|
||||||
|
window.addEventListener("scroll", onViewportChange, true);
|
||||||
|
window.addEventListener("resize", onViewportChange);
|
||||||
|
const interval = window.setInterval(onViewportChange, 500);
|
||||||
|
return () => {
|
||||||
|
disposed = true;
|
||||||
|
window.removeEventListener("scroll", onViewportChange, true);
|
||||||
|
window.removeEventListener("resize", onViewportChange);
|
||||||
|
window.clearInterval(interval);
|
||||||
|
};
|
||||||
|
}, [request, isCompact]);
|
||||||
|
|
||||||
// Scroll the first matched target into view once per distinct request id.
|
// Scroll the first matched target into view once per distinct request id.
|
||||||
useLayoutEffect(() => {
|
useLayoutEffect(() => {
|
||||||
if (!request) return;
|
if (!request) return;
|
||||||
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const update = () => {
|
const update = () => {
|
||||||
const nextBoxes = measure(effectiveSelectors);
|
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||||
};
|
};
|
||||||
update();
|
update();
|
||||||
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
window.removeEventListener("resize", update);
|
window.removeEventListener("resize", update);
|
||||||
window.clearInterval(interval);
|
window.clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [request, effectiveSelectors]);
|
}, [request, effectiveSelectors, effectiveRects]);
|
||||||
|
|
||||||
const onHeaderPointerDown = useCallback(
|
const onHeaderPointerDown = useCallback(
|
||||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||||
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (userMovedRef.current) return;
|
if (userMovedRef.current) return;
|
||||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||||
const anchor = unionBox(measured);
|
const anchor = unionBox(measured);
|
||||||
const banner = bannerRef.current;
|
const banner = bannerRef.current;
|
||||||
if (!anchor || !banner) {
|
if (!anchor || !banner) {
|
||||||
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||||
|
|
||||||
if (!request) return null;
|
if (!request) return null;
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
|
|||||||
id: "req-1",
|
id: "req-1",
|
||||||
isActiveTab: true,
|
isActiveTab: true,
|
||||||
tabTitle: "Example",
|
tabTitle: "Example",
|
||||||
timeoutMs: 5000,
|
timeoutMs: 60_000,
|
||||||
onAllow,
|
onAllow,
|
||||||
onDeny,
|
onDeny,
|
||||||
},
|
},
|
||||||
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(screen.getByText("允许借用标签页?")).toBeTruthy();
|
expect(screen.getByText("允许借用标签页?")).toBeTruthy();
|
||||||
for (let i = 0; i < 5; i++) {
|
for (let i = 0; i < 60; i++) {
|
||||||
await act(async () => {
|
await act(async () => {
|
||||||
await vi.advanceTimersByTimeAsync(1000);
|
await vi.advanceTimersByTimeAsync(1000);
|
||||||
});
|
});
|
||||||
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
|
|||||||
id: "req-fallback",
|
id: "req-fallback",
|
||||||
isActiveTab: true,
|
isActiveTab: true,
|
||||||
tabTitle: "Example",
|
tabTitle: "Example",
|
||||||
timeoutMs: 5000,
|
timeoutMs: 60_000,
|
||||||
onAllow,
|
onAllow,
|
||||||
onDeny,
|
onDeny,
|
||||||
},
|
},
|
||||||
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
|
|||||||
/>,
|
/>,
|
||||||
);
|
);
|
||||||
|
|
||||||
for (let i = 0; i < 5; i++) {
|
for (let i = 0; i < 60; i++) {
|
||||||
await act(async () => {
|
await act(async () => {
|
||||||
await vi.advanceTimersByTimeAsync(1000);
|
await vi.advanceTimersByTimeAsync(1000);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||||
import { createElement } from "react";
|
import { createElement } from "react";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||||
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||||
|
);
|
||||||
|
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||||
|
expect(highlight?.style.top).toBe("30px");
|
||||||
|
expect(highlight?.style.left).toBe("40px");
|
||||||
|
expect(highlight?.style.width).toBe("120px");
|
||||||
|
expect(highlight?.style.height).toBe("50px");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||||
|
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({
|
||||||
|
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||||
|
refreshRects,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
window.dispatchEvent(new Event("scroll"));
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||||
|
expect(highlight?.style.top).toBe("80px");
|
||||||
|
expect(highlight?.style.left).toBe("90px");
|
||||||
|
});
|
||||||
|
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the inactive render stable", () => {
|
it("keeps the inactive render stable", () => {
|
||||||
const { container, rerender } = renderOverlay(null);
|
const { container, rerender } = renderOverlay(null);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { createHelpRequestData } from "../help-request";
|
||||||
|
|
||||||
|
describe("createHelpRequestData", () => {
|
||||||
|
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||||
|
const finish = vi.fn();
|
||||||
|
const query = vi.fn(async () => ({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
const request = createHelpRequestData(
|
||||||
|
{
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
{ finish, query },
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||||
|
{ top: 80, left: 90, width: 140, height: 60 },
|
||||||
|
]);
|
||||||
|
request.onContinue(" done ");
|
||||||
|
request.onCancel();
|
||||||
|
|
||||||
|
expect(query).toHaveBeenCalledOnce();
|
||||||
|
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||||
|
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores rectangle replies for a replaced request", async () => {
|
||||||
|
const request = createHelpRequestData(
|
||||||
|
{
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
finish: vi.fn(),
|
||||||
|
query: vi.fn(async () => ({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: "help-2",
|
||||||
|
prompt: "new challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
import { startRecordCapture } from "../record-capture";
|
||||||
|
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
runtime: {
|
runtime: {
|
||||||
sendMessage: vi.fn(() => Promise.resolve()),
|
sendMessage: vi.fn((message: { sequence?: number }) =>
|
||||||
|
Promise.resolve({ ok: true, sequence: message.sequence }),
|
||||||
|
),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -46,58 +48,6 @@ function click(el: Element): void {
|
|||||||
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleRecordContentMessage stop/cancel", () => {
|
|
||||||
it("ignores STOP when no recording is active", () => {
|
|
||||||
const dispose = vi.fn();
|
|
||||||
const onStop = vi.fn();
|
|
||||||
const setActiveRequestId = vi.fn();
|
|
||||||
const setCapture = vi.fn();
|
|
||||||
const sendResponse = vi.fn();
|
|
||||||
|
|
||||||
const needsAsync = handleRecordContentMessage(
|
|
||||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
|
||||||
{
|
|
||||||
activeRequestId: null,
|
|
||||||
capture: { dispose },
|
|
||||||
setActiveRequestId,
|
|
||||||
setCapture,
|
|
||||||
onStart: vi.fn(),
|
|
||||||
onStop,
|
|
||||||
},
|
|
||||||
sendResponse,
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(needsAsync).toBe(false);
|
|
||||||
expect(dispose).not.toHaveBeenCalled();
|
|
||||||
expect(onStop).not.toHaveBeenCalled();
|
|
||||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
|
||||||
expect(setCapture).not.toHaveBeenCalled();
|
|
||||||
expect(sendResponse).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("ignores STOP for a mismatched requestId", () => {
|
|
||||||
const dispose = vi.fn();
|
|
||||||
const onStop = vi.fn();
|
|
||||||
|
|
||||||
const needsAsync = handleRecordContentMessage(
|
|
||||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
|
||||||
{
|
|
||||||
activeRequestId: "rec-1",
|
|
||||||
capture: { dispose },
|
|
||||||
setActiveRequestId: vi.fn(),
|
|
||||||
setCapture: vi.fn(),
|
|
||||||
onStart: vi.fn(),
|
|
||||||
onStop,
|
|
||||||
},
|
|
||||||
vi.fn(),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(needsAsync).toBe(false);
|
|
||||||
expect(dispose).not.toHaveBeenCalled();
|
|
||||||
expect(onStop).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("record-capture semantic", () => {
|
describe("record-capture semantic", () => {
|
||||||
let steps: RecordStepPayload[];
|
let steps: RecordStepPayload[];
|
||||||
|
|
||||||
@@ -174,6 +124,19 @@ describe("record-capture semantic", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not emit page navigation from a child Document capture", () => {
|
||||||
|
const originalUrl = location.href;
|
||||||
|
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
|
||||||
|
captureNavigation: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
history.pushState({}, "", "#inside-frame");
|
||||||
|
expect(steps).toEqual([]);
|
||||||
|
|
||||||
|
capture.dispose();
|
||||||
|
history.replaceState({}, "", originalUrl);
|
||||||
|
});
|
||||||
|
|
||||||
it("does not record clicks on anonymous layout divs", () => {
|
it("does not record clicks on anonymous layout divs", () => {
|
||||||
document.body.innerHTML = `
|
document.body.innerHTML = `
|
||||||
<div id="chrome">page chrome</div>
|
<div id="chrome">page chrome</div>
|
||||||
@@ -228,6 +191,7 @@ describe("record-capture semantic", () => {
|
|||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { role: "button", name: "Open user navigation menu" },
|
target: { role: "button", name: "Open user navigation menu" },
|
||||||
|
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||||
});
|
});
|
||||||
expect(steps[1]).toMatchObject({
|
expect(steps[1]).toMatchObject({
|
||||||
op: "click",
|
op: "click",
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
|
||||||
|
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
|
||||||
|
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
|
||||||
|
import { RecordFrameAgent } from "../recording/frame-agent";
|
||||||
|
|
||||||
|
class PortListeners<T extends (...args: never[]) => unknown> {
|
||||||
|
readonly values = new Set<T>();
|
||||||
|
addListener = (listener: T) => this.values.add(listener);
|
||||||
|
removeListener = (listener: T) => this.values.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
function portHarness() {
|
||||||
|
const onMessage = new PortListeners<(message: unknown) => void>();
|
||||||
|
const onDisconnect = new PortListeners<() => void>();
|
||||||
|
const outbound: RecordFramePortMessage[] = [];
|
||||||
|
const port = {
|
||||||
|
name: "bsk-record-frame",
|
||||||
|
onMessage,
|
||||||
|
onDisconnect,
|
||||||
|
postMessage(message: RecordFramePortMessage) {
|
||||||
|
outbound.push(message);
|
||||||
|
if (message.type === "ready") {
|
||||||
|
queueMicrotask(() => {
|
||||||
|
for (const listener of onMessage.values) {
|
||||||
|
listener({
|
||||||
|
type: "ready_ack",
|
||||||
|
requestId: message.requestId,
|
||||||
|
producerId: message.producerId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
disconnect: vi.fn(),
|
||||||
|
} as unknown as chrome.runtime.Port;
|
||||||
|
return {
|
||||||
|
port,
|
||||||
|
outbound,
|
||||||
|
receive(message: RecordFramePortMessage) {
|
||||||
|
for (const listener of onMessage.values) listener(message);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("RecordFrameAgent", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
document.body.replaceChildren();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts when crypto.randomUUID is unavailable", async () => {
|
||||||
|
const harness = portHarness();
|
||||||
|
const getRandomValues = vi.fn((bytes: Uint8Array) => {
|
||||||
|
bytes.fill(0x11);
|
||||||
|
return bytes;
|
||||||
|
});
|
||||||
|
vi.stubGlobal("crypto", { getRandomValues });
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: {
|
||||||
|
connect: vi.fn(() => harness.port),
|
||||||
|
sendMessage: vi.fn(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const agent = new RecordFrameAgent();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
|
||||||
|
).resolves.toEqual({ ok: true });
|
||||||
|
expect(getRandomValues).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||||
|
const harness = portHarness();
|
||||||
|
const sendMessage = vi
|
||||||
|
.fn<(message: { sequence: number }) => Promise<unknown>>()
|
||||||
|
.mockRejectedValueOnce(new Error("offline"))
|
||||||
|
.mockRejectedValueOnce(new Error("still offline"))
|
||||||
|
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: {
|
||||||
|
connect: vi.fn(() => harness.port),
|
||||||
|
sendMessage,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
|
||||||
|
const agent = new RecordFrameAgent();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
|
||||||
|
).resolves.toEqual({ ok: true });
|
||||||
|
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||||
|
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "final value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
|
||||||
|
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(harness.outbound).toContainEqual({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: "rec-1",
|
||||||
|
commandId: "stop-1",
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||||
|
|
||||||
|
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(harness.outbound).toContainEqual({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: "rec-1",
|
||||||
|
commandId: "stop-2",
|
||||||
|
ok: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(sendMessage).toHaveBeenCalledTimes(3);
|
||||||
|
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { RecordStepMessage } from "@/lib/record-bridge";
|
||||||
|
import { RecordStepDelivery } from "../record-step-delivery";
|
||||||
|
|
||||||
|
describe("RecordStepDelivery", () => {
|
||||||
|
it("retries from the first unacknowledged sequence before sending later steps", async () => {
|
||||||
|
const sent: RecordStepMessage[] = [];
|
||||||
|
const send = vi.fn(async (message: RecordStepMessage) => {
|
||||||
|
sent.push(message);
|
||||||
|
if (sent.length === 1) throw new Error("service worker unavailable");
|
||||||
|
return { ok: true, sequence: message.sequence };
|
||||||
|
});
|
||||||
|
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
|
||||||
|
|
||||||
|
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
|
||||||
|
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
|
||||||
|
|
||||||
|
await expect(delivery.flush()).resolves.toBe(true);
|
||||||
|
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
|
||||||
|
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps unacknowledged steps pending for a later flush", async () => {
|
||||||
|
const send = vi
|
||||||
|
.fn<(message: RecordStepMessage) => Promise<unknown>>()
|
||||||
|
.mockRejectedValueOnce(new Error("offline"))
|
||||||
|
.mockRejectedValueOnce(new Error("still offline"))
|
||||||
|
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||||
|
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
|
||||||
|
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
|
||||||
|
|
||||||
|
await expect(delivery.flush()).resolves.toBe(false);
|
||||||
|
await expect(delivery.flush()).resolves.toBe(true);
|
||||||
|
expect(send).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||||
|
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||||
|
|
||||||
|
export interface HelpRequestDataDeps {
|
||||||
|
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||||
|
query(): Promise<HelpQueryResponse | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build the content-side representation shared by live delivery and recovery. */
|
||||||
|
export function createHelpRequestData(
|
||||||
|
message: Omit<HelpRequestMessage, "type">,
|
||||||
|
deps: HelpRequestDataDeps,
|
||||||
|
): HelpRequestData {
|
||||||
|
return {
|
||||||
|
id: message.requestId,
|
||||||
|
prompt: message.prompt,
|
||||||
|
...(message.title ? { title: message.title } : {}),
|
||||||
|
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||||
|
selectors: message.selectors,
|
||||||
|
rects: message.rects,
|
||||||
|
refreshRects: async () => {
|
||||||
|
const response = await deps.query();
|
||||||
|
return response?.active && response.request?.requestId === message.requestId
|
||||||
|
? response.request.rects
|
||||||
|
: undefined;
|
||||||
|
},
|
||||||
|
onContinue: (note: string) =>
|
||||||
|
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import {
|
import {
|
||||||
|
type CaptureTargetDescriptor,
|
||||||
describeEventTarget,
|
describeEventTarget,
|
||||||
describeTarget,
|
describeTarget,
|
||||||
resolveClickableElement,
|
resolveClickableElement,
|
||||||
resolveHoverElement,
|
resolveHoverElement,
|
||||||
type TargetDescriptor,
|
|
||||||
} from "@/lib/describe-target";
|
} from "@/lib/describe-target";
|
||||||
import {
|
import {
|
||||||
evaluateHoverTrigger,
|
evaluateHoverTrigger,
|
||||||
@@ -12,22 +12,8 @@ import {
|
|||||||
hasDirectHoverInteractiveSignal,
|
hasDirectHoverInteractiveSignal,
|
||||||
hasStrongHoverExpansionSignal,
|
hasStrongHoverExpansionSignal,
|
||||||
} from "@/lib/hover-trigger-policy";
|
} from "@/lib/hover-trigger-policy";
|
||||||
import {
|
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||||
isRecordCancelMessage,
|
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||||
isRecordStartMessage,
|
|
||||||
isRecordStopMessage,
|
|
||||||
RECORD_CANCEL,
|
|
||||||
RECORD_START,
|
|
||||||
RECORD_STEP,
|
|
||||||
RECORD_STOP,
|
|
||||||
type RecordCancelMessage,
|
|
||||||
type RecordStartAck,
|
|
||||||
type RecordStartMessage,
|
|
||||||
type RecordStepPayload,
|
|
||||||
type RecordStopAck,
|
|
||||||
type RecordStopMessage,
|
|
||||||
} from "@/lib/record-bridge";
|
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
|
||||||
import {
|
import {
|
||||||
closestHoverSurfaceCandidate,
|
closestHoverSurfaceCandidate,
|
||||||
collectHoverSurfaceStates,
|
collectHoverSurfaceStates,
|
||||||
@@ -37,42 +23,21 @@ import {
|
|||||||
isLikelyHoverSurfaceOwner,
|
isLikelyHoverSurfaceOwner,
|
||||||
} from "./record-hover-surface";
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
|
||||||
const failedStepDeliveries = new Set<string>();
|
|
||||||
const knownRecordRequests = new Set<string>();
|
|
||||||
|
|
||||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
|
||||||
const payload = { type: RECORD_STEP, requestId, step };
|
|
||||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
|
||||||
() => true,
|
|
||||||
() => {
|
|
||||||
failedStepDeliveries.add(requestId);
|
|
||||||
return false;
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
|
||||||
sends.add(pending);
|
|
||||||
pendingStepSends.set(requestId, sends);
|
|
||||||
void pending.then(() => {
|
|
||||||
sends.delete(pending);
|
|
||||||
if (sends.size === 0) pendingStepSends.delete(requestId);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RecordCaptureController {
|
export interface RecordCaptureController {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface FillSession {
|
interface FillSession {
|
||||||
element: FillableElement;
|
element: FillableElement;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
baselineValue: string;
|
baselineValue: string;
|
||||||
lastValue: string;
|
lastValue: string;
|
||||||
|
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
element: Element;
|
element: Element;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
recordedAt: number;
|
recordedAt: number;
|
||||||
score: number;
|
score: number;
|
||||||
eligible: boolean;
|
eligible: boolean;
|
||||||
@@ -157,6 +122,24 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return {
|
||||||
|
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function geometryForEventTarget(
|
||||||
|
target: EventTarget | null,
|
||||||
|
): RecordStepPayload["geometry"] | undefined {
|
||||||
|
if (!(target instanceof Element)) return undefined;
|
||||||
|
const clickable = target.closest(
|
||||||
|
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||||
|
);
|
||||||
|
return captureGeometry(clickable ?? target);
|
||||||
|
}
|
||||||
|
|
||||||
function fillableValue(el: FillableElement): string {
|
function fillableValue(el: FillableElement): string {
|
||||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||||
return el.value;
|
return el.value;
|
||||||
@@ -232,7 +215,10 @@ function collectHoverTriggerLabelText(root: Element): string {
|
|||||||
return normalizeLabelText(text);
|
return normalizeLabelText(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
function compactHoverTargetName(
|
||||||
|
el: Element,
|
||||||
|
desc: CaptureTargetDescriptor,
|
||||||
|
): CaptureTargetDescriptor {
|
||||||
if (!desc.name) return desc;
|
if (!desc.name) return desc;
|
||||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
const compactName = collectHoverTriggerLabelText(el);
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
@@ -250,7 +236,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
return !target.role && !target.name && target.tag === "div";
|
return !target.role && !target.name && target.tag === "div";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,9 +247,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
|||||||
|
|
||||||
function normalizeHoverTarget(
|
function normalizeHoverTarget(
|
||||||
el: Element,
|
el: Element,
|
||||||
desc: TargetDescriptor,
|
desc: CaptureTargetDescriptor,
|
||||||
decision: HoverTriggerDecision,
|
decision: HoverTriggerDecision,
|
||||||
): TargetDescriptor {
|
): CaptureTargetDescriptor {
|
||||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
return { ...desc, name: "image" };
|
return { ...desc, name: "image" };
|
||||||
}
|
}
|
||||||
@@ -277,7 +263,7 @@ function normalizeHoverTarget(
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||||
if (!(el instanceof HTMLElement)) return null;
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
const style = hoverTriggerStyle(el);
|
const style = hoverTriggerStyle(el);
|
||||||
return {
|
return {
|
||||||
@@ -373,6 +359,7 @@ function scheduleInputCompletionCommit(
|
|||||||
export function startRecordCapture(
|
export function startRecordCapture(
|
||||||
_requestId: string,
|
_requestId: string,
|
||||||
sendStep: (step: RecordStepPayload) => void,
|
sendStep: (step: RecordStepPayload) => void,
|
||||||
|
options: { captureNavigation?: boolean } = {},
|
||||||
): RecordCaptureController {
|
): RecordCaptureController {
|
||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
@@ -411,14 +398,17 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: session.target,
|
target: session.target,
|
||||||
|
geometry: captureGeometry(session.element),
|
||||||
value,
|
value,
|
||||||
|
commit: session.pendingCommit ?? "blur",
|
||||||
...(isPassword ? { redacted: true } : {}),
|
...(isPassword ? { redacted: true } : {}),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const commitFillSession = () => {
|
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||||
if (!fillSession || composing) return;
|
if (!fillSession || composing) return;
|
||||||
const session = fillSession;
|
const session = fillSession;
|
||||||
|
session.pendingCommit = commit;
|
||||||
fillSession = null;
|
fillSession = null;
|
||||||
emitFill(session);
|
emitFill(session);
|
||||||
committedValues.set(session.element, session.lastValue);
|
committedValues.set(session.element, session.lastValue);
|
||||||
@@ -624,6 +614,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: hover.target,
|
target: hover.target,
|
||||||
|
geometry: captureGeometry(hover.element),
|
||||||
});
|
});
|
||||||
emittedHoverElements.add(hover.element);
|
emittedHoverElements.add(hover.element);
|
||||||
};
|
};
|
||||||
@@ -678,6 +669,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "click",
|
op: "click",
|
||||||
target,
|
target,
|
||||||
|
geometry: geometryForEventTarget(eventTarget(event)),
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
@@ -779,7 +771,7 @@ export function startRecordCapture(
|
|||||||
scheduleInputCompletionCommit(
|
scheduleInputCompletionCommit(
|
||||||
sessionElement,
|
sessionElement,
|
||||||
syncFillSessionValue,
|
syncFillSessionValue,
|
||||||
commitFillSession,
|
() => commitFillSession("suggestion"),
|
||||||
(el) => fillSession?.element === el,
|
(el) => fillSession?.element === el,
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
@@ -839,6 +831,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "select",
|
op: "select",
|
||||||
target: desc,
|
target: desc,
|
||||||
|
geometry: captureGeometry(target),
|
||||||
values,
|
values,
|
||||||
labels,
|
labels,
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
@@ -879,7 +872,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
commitFillSession();
|
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
@@ -887,7 +880,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
...(desc ? { target: desc } : {}),
|
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||||
...(modifiers.length ? { modifiers } : {}),
|
...(modifiers.length ? { modifiers } : {}),
|
||||||
expects_navigation: event.key === "Enter",
|
expects_navigation: event.key === "Enter",
|
||||||
});
|
});
|
||||||
@@ -903,23 +896,33 @@ export function startRecordCapture(
|
|||||||
document.addEventListener("change", onChange, true);
|
document.addEventListener("change", onChange, true);
|
||||||
document.addEventListener("keydown", onKeyDown, true);
|
document.addEventListener("keydown", onKeyDown, true);
|
||||||
|
|
||||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
const captureNavigation = options.captureNavigation ?? true;
|
||||||
urlObserver.observe(document, { subtree: true, childList: true });
|
const urlObserver = captureNavigation
|
||||||
|
? new MutationObserver(() => emitNavigateIfChanged())
|
||||||
|
: null;
|
||||||
|
urlObserver?.observe(document, { subtree: true, childList: true });
|
||||||
const onUrlEvent = () => emitNavigateIfChanged();
|
const onUrlEvent = () => emitNavigateIfChanged();
|
||||||
window.addEventListener("hashchange", onUrlEvent);
|
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||||
window.addEventListener("popstate", onUrlEvent);
|
// the `commit` argument and stamp it onto the recorded fill step.
|
||||||
window.addEventListener("pagehide", commitFillSession);
|
const onPageHide = () => commitFillSession();
|
||||||
|
if (captureNavigation) {
|
||||||
|
window.addEventListener("hashchange", onUrlEvent);
|
||||||
|
window.addEventListener("popstate", onUrlEvent);
|
||||||
|
}
|
||||||
|
window.addEventListener("pagehide", onPageHide);
|
||||||
|
|
||||||
const originalPushState = history.pushState;
|
const originalPushState = history.pushState;
|
||||||
const originalReplaceState = history.replaceState;
|
const originalReplaceState = history.replaceState;
|
||||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
if (captureNavigation) {
|
||||||
originalPushState.apply(this, args);
|
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
originalPushState.apply(this, args);
|
||||||
};
|
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
};
|
||||||
originalReplaceState.apply(this, args);
|
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
originalReplaceState.apply(this, args);
|
||||||
};
|
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
dispose() {
|
dispose() {
|
||||||
@@ -933,93 +936,16 @@ export function startRecordCapture(
|
|||||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||||
document.removeEventListener("change", onChange, true);
|
document.removeEventListener("change", onChange, true);
|
||||||
document.removeEventListener("keydown", onKeyDown, true);
|
document.removeEventListener("keydown", onKeyDown, true);
|
||||||
urlObserver.disconnect();
|
urlObserver?.disconnect();
|
||||||
window.removeEventListener("hashchange", onUrlEvent);
|
if (captureNavigation) {
|
||||||
window.removeEventListener("popstate", onUrlEvent);
|
window.removeEventListener("hashchange", onUrlEvent);
|
||||||
window.removeEventListener("pagehide", commitFillSession);
|
window.removeEventListener("popstate", onUrlEvent);
|
||||||
history.pushState = originalPushState;
|
}
|
||||||
history.replaceState = originalReplaceState;
|
window.removeEventListener("pagehide", onPageHide);
|
||||||
|
if (captureNavigation) {
|
||||||
|
history.pushState = originalPushState;
|
||||||
|
history.replaceState = originalReplaceState;
|
||||||
|
}
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
|
||||||
|
|
||||||
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
|
||||||
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function handleRecordContentMessage(
|
|
||||||
message: RecordContentMessage,
|
|
||||||
state: {
|
|
||||||
activeRequestId: string | null;
|
|
||||||
capture: RecordCaptureController | null;
|
|
||||||
setActiveRequestId(id: string | null): void;
|
|
||||||
setCapture(capture: RecordCaptureController | null): void;
|
|
||||||
onStart(requestId: string, startedAtMs?: number): void;
|
|
||||||
onStop(): void;
|
|
||||||
},
|
|
||||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
|
||||||
): boolean {
|
|
||||||
if (isRecordStartMessage(message)) {
|
|
||||||
if (!knownRecordRequests.has(message.requestId)) {
|
|
||||||
knownRecordRequests.add(message.requestId);
|
|
||||||
failedStepDeliveries.delete(message.requestId);
|
|
||||||
}
|
|
||||||
state.capture?.dispose();
|
|
||||||
state.setCapture(
|
|
||||||
startRecordCapture(message.requestId, (step) => {
|
|
||||||
sendRecordStep(message.requestId, step);
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
state.setActiveRequestId(message.requestId);
|
|
||||||
state.onStart(message.requestId, message.startedAtMs);
|
|
||||||
sendResponse?.({ ok: true });
|
|
||||||
return sendResponse !== undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
|
||||||
// Require an active recording that matches this requestId — otherwise a
|
|
||||||
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
|
||||||
// clear overlay state even though nothing was capturing.
|
|
||||||
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
state.capture?.dispose();
|
|
||||||
state.setCapture(null);
|
|
||||||
const finishStop = () => {
|
|
||||||
state.onStop();
|
|
||||||
state.setActiveRequestId(null);
|
|
||||||
};
|
|
||||||
if (isRecordStopMessage(message) && sendResponse) {
|
|
||||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
|
||||||
void Promise.all(pending).then((delivered) => {
|
|
||||||
finishStop();
|
|
||||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
|
||||||
if (succeeded) {
|
|
||||||
failedStepDeliveries.delete(message.requestId);
|
|
||||||
knownRecordRequests.delete(message.requestId);
|
|
||||||
}
|
|
||||||
sendResponse(
|
|
||||||
succeeded
|
|
||||||
? { ok: true }
|
|
||||||
: {
|
|
||||||
ok: false,
|
|
||||||
error: "failed to deliver one or more recorded steps",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (isRecordCancelMessage(message)) {
|
|
||||||
failedStepDeliveries.delete(message.requestId);
|
|
||||||
knownRecordRequests.delete(message.requestId);
|
|
||||||
}
|
|
||||||
finishStop();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import { createRandomUuid } from "@/lib/random-uuid";
|
||||||
|
import {
|
||||||
|
isAcceptedRecordStepAck,
|
||||||
|
RECORD_STEP,
|
||||||
|
type RecordStepMessage,
|
||||||
|
type RecordStepPayload,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
|
||||||
|
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
|
||||||
|
|
||||||
|
export class RecordStepDelivery {
|
||||||
|
readonly #requestId: string;
|
||||||
|
readonly #producerId: string;
|
||||||
|
readonly #send: SendRecordStepMessage;
|
||||||
|
readonly #pending = new Map<number, RecordStepPayload>();
|
||||||
|
#nextSequence = 1;
|
||||||
|
#tail = Promise.resolve();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
requestId: string,
|
||||||
|
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||||
|
producerId: string = createRandomUuid(),
|
||||||
|
) {
|
||||||
|
this.#requestId = requestId;
|
||||||
|
this.#producerId = producerId;
|
||||||
|
this.#send = send;
|
||||||
|
}
|
||||||
|
|
||||||
|
enqueue(step: RecordStepPayload): void {
|
||||||
|
const sequence = this.#nextSequence;
|
||||||
|
this.#nextSequence += 1;
|
||||||
|
this.#pending.set(sequence, step);
|
||||||
|
void this.#schedule();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<boolean> {
|
||||||
|
await this.#tail;
|
||||||
|
await this.#schedule();
|
||||||
|
return this.#pending.size === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#schedule(): Promise<void> {
|
||||||
|
const drain = () => this.#drain();
|
||||||
|
this.#tail = this.#tail.then(drain, drain);
|
||||||
|
return this.#tail;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #drain(): Promise<void> {
|
||||||
|
for (const [sequence, step] of this.#pending) {
|
||||||
|
const message: RecordStepMessage = {
|
||||||
|
type: RECORD_STEP,
|
||||||
|
requestId: this.#requestId,
|
||||||
|
producerId: this.#producerId,
|
||||||
|
sequence,
|
||||||
|
step,
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const response = await this.#send(message);
|
||||||
|
if (!isAcceptedRecordStepAck(response, sequence)) return;
|
||||||
|
this.#pending.delete(sequence);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
import { createRandomUuid } from "@/lib/random-uuid";
|
||||||
|
import {
|
||||||
|
markRecordingDocument,
|
||||||
|
type RecordingDocumentMarker,
|
||||||
|
waitForDocumentElement,
|
||||||
|
} from "@/lib/recording/document-marker";
|
||||||
|
import {
|
||||||
|
isRecordFrameStartMessage,
|
||||||
|
RECORD_FRAME_PORT,
|
||||||
|
RECORD_FRAME_QUERY,
|
||||||
|
RECORD_FRAME_START,
|
||||||
|
type RecordFramePortMessage,
|
||||||
|
type RecordFrameQueryResponse,
|
||||||
|
type RecordFrameStartMessage,
|
||||||
|
} from "@/lib/recording/frame-bridge";
|
||||||
|
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
|
||||||
|
import { RecordStepDelivery } from "../record-step-delivery";
|
||||||
|
|
||||||
|
interface ActiveFrameRecording {
|
||||||
|
requestId: string;
|
||||||
|
producerId: string;
|
||||||
|
port: chrome.runtime.Port;
|
||||||
|
delivery: RecordStepDelivery;
|
||||||
|
capture: RecordCaptureController;
|
||||||
|
marker: RecordingDocumentMarker;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
|
||||||
|
|
||||||
|
export class RecordFrameAgent {
|
||||||
|
#active: ActiveFrameRecording | null = null;
|
||||||
|
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
|
||||||
|
#disposed = false;
|
||||||
|
|
||||||
|
async start(
|
||||||
|
message: RecordFrameStartMessage,
|
||||||
|
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||||
|
if (this.#active?.requestId === message.requestId) return { ok: true };
|
||||||
|
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||||
|
if (this.#startPromise) {
|
||||||
|
await this.#startPromise;
|
||||||
|
return this.start(message);
|
||||||
|
}
|
||||||
|
if (this.#active) this.cancel(this.#active.requestId);
|
||||||
|
const starting = this.#start(message);
|
||||||
|
this.#startPromise = starting;
|
||||||
|
try {
|
||||||
|
return await starting;
|
||||||
|
} finally {
|
||||||
|
if (this.#startPromise === starting) this.#startPromise = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(requestId: string): void {
|
||||||
|
const active = this.#active;
|
||||||
|
if (!active || active.requestId !== requestId) return;
|
||||||
|
active.capture.dispose();
|
||||||
|
active.marker.restore();
|
||||||
|
active.port.disconnect();
|
||||||
|
this.#active = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
dispose(): void {
|
||||||
|
this.#disposed = true;
|
||||||
|
if (this.#active) this.cancel(this.#active.requestId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async #start(
|
||||||
|
message: RecordFrameStartMessage,
|
||||||
|
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||||
|
const producerId = createRandomUuid();
|
||||||
|
const root = await waitForDocumentElement();
|
||||||
|
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||||
|
const marker = markRecordingDocument(producerId, root);
|
||||||
|
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
|
||||||
|
try {
|
||||||
|
await registerPort(port, message.requestId, producerId);
|
||||||
|
} catch {
|
||||||
|
marker.restore();
|
||||||
|
port.disconnect();
|
||||||
|
return { ok: false, error: "failed to register recording document" };
|
||||||
|
}
|
||||||
|
if (this.#disposed) {
|
||||||
|
marker.restore();
|
||||||
|
port.disconnect();
|
||||||
|
return { ok: false, error: "recording document is disposed" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
|
||||||
|
const capture = startRecordCapture(
|
||||||
|
message.requestId,
|
||||||
|
(step) => {
|
||||||
|
marker.ensure();
|
||||||
|
delivery.enqueue(step);
|
||||||
|
},
|
||||||
|
{ captureNavigation: window.top === window },
|
||||||
|
);
|
||||||
|
const active: ActiveFrameRecording = {
|
||||||
|
requestId: message.requestId,
|
||||||
|
producerId,
|
||||||
|
port,
|
||||||
|
delivery,
|
||||||
|
capture,
|
||||||
|
marker,
|
||||||
|
};
|
||||||
|
this.#active = active;
|
||||||
|
port.onMessage.addListener((raw: unknown) => {
|
||||||
|
const command = raw as Partial<RecordFramePortMessage>;
|
||||||
|
if (command.type === "stop" && command.requestId === active.requestId) {
|
||||||
|
void this.#stop(active, command.commandId);
|
||||||
|
} else if (command.type === "cancel" && command.requestId === active.requestId) {
|
||||||
|
this.cancel(active.requestId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
port.onDisconnect.addListener(() => {
|
||||||
|
if (this.#active !== active) return;
|
||||||
|
active.capture.dispose();
|
||||||
|
active.marker.restore();
|
||||||
|
this.#active = null;
|
||||||
|
});
|
||||||
|
return { ok: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
|
||||||
|
if (!commandId || this.#active !== active) return;
|
||||||
|
active.capture.dispose();
|
||||||
|
const succeeded = await active.delivery.flush();
|
||||||
|
try {
|
||||||
|
active.port.postMessage({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: active.requestId,
|
||||||
|
commandId,
|
||||||
|
ok: succeeded,
|
||||||
|
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!succeeded || this.#active !== active) return;
|
||||||
|
active.marker.restore();
|
||||||
|
this.#active = null;
|
||||||
|
active.port.disconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function registerPort(
|
||||||
|
port: chrome.runtime.Port,
|
||||||
|
requestId: string,
|
||||||
|
producerId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
reject(new Error("recording document registration timed out"));
|
||||||
|
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
|
||||||
|
const cleanup = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
port.onMessage.removeListener(onMessage);
|
||||||
|
port.onDisconnect.removeListener(onDisconnect);
|
||||||
|
};
|
||||||
|
const onMessage = (raw: unknown) => {
|
||||||
|
const message = raw as Partial<RecordFramePortMessage>;
|
||||||
|
if (
|
||||||
|
message.type !== "ready_ack" ||
|
||||||
|
message.requestId !== requestId ||
|
||||||
|
message.producerId !== producerId
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const onDisconnect = () => {
|
||||||
|
cleanup();
|
||||||
|
reject(new Error("recording document port disconnected"));
|
||||||
|
};
|
||||||
|
port.onMessage.addListener(onMessage);
|
||||||
|
port.onDisconnect.addListener(onDisconnect);
|
||||||
|
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function attachRecordFrameAgent(): () => void {
|
||||||
|
const agent = new RecordFrameAgent();
|
||||||
|
const onMessage = (
|
||||||
|
message: unknown,
|
||||||
|
_sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
|
||||||
|
) => {
|
||||||
|
if (!isRecordFrameStartMessage(message)) return false;
|
||||||
|
void agent.start(message).then(sendResponse);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
chrome.runtime.onMessage.addListener(onMessage);
|
||||||
|
void chrome.runtime
|
||||||
|
.sendMessage({ type: RECORD_FRAME_QUERY })
|
||||||
|
.then((response: RecordFrameQueryResponse | undefined) => {
|
||||||
|
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
|
||||||
|
return agent.start({
|
||||||
|
type: RECORD_FRAME_START,
|
||||||
|
requestId: response.requestId,
|
||||||
|
startedAtMs: response.startedAtMs,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch(() => {});
|
||||||
|
return () => {
|
||||||
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
|
agent.dispose();
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -21,10 +21,11 @@ import {
|
|||||||
type OverlayMode,
|
type OverlayMode,
|
||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||||
|
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
|
||||||
import {
|
import {
|
||||||
attachBorrowNotificationButtonHandler,
|
attachBorrowNotificationButtonHandler,
|
||||||
attachBorrowNotificationClickHandler,
|
attachBorrowNotificationClickHandler,
|
||||||
@@ -38,7 +39,10 @@ import {
|
|||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
attachRecordQueryListener,
|
attachRecordQueryListener,
|
||||||
attachRecordStepListener,
|
attachRecordStepListener,
|
||||||
|
type RecordRuntimeDeps,
|
||||||
} from "@/tools/record";
|
} from "@/tools/record";
|
||||||
|
import { chromeTabsApi } from "@/tools/shared";
|
||||||
|
import { chromeTabMutationApi } from "@/tools/tabs";
|
||||||
import { detectBrowserMeta } from "@/transport/handshake";
|
import { detectBrowserMeta } from "@/transport/handshake";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
import { WSTransport } from "@/transport/ws-transport";
|
import { WSTransport } from "@/transport/ws-transport";
|
||||||
@@ -47,7 +51,12 @@ export default defineBackground(() => {
|
|||||||
const controller = new ConnectionController();
|
const controller = new ConnectionController();
|
||||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||||
const sessions = new SessionManager();
|
const sessions = new SessionManager();
|
||||||
const cdp = new ChromiumCdp();
|
const cdp = new ChromiumCdp(undefined, {
|
||||||
|
shouldAutoAcceptDialog: async (tabId) => {
|
||||||
|
const tab = await chrome.tabs.get(tabId);
|
||||||
|
return sessions.findByWindowId(tab.windowId) !== null;
|
||||||
|
},
|
||||||
|
});
|
||||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||||
let overlayGeneration = 0;
|
let overlayGeneration = 0;
|
||||||
const controlModes = new Map<string, OverlayMode>();
|
const controlModes = new Map<string, OverlayMode>();
|
||||||
@@ -78,6 +87,24 @@ export default defineBackground(() => {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Authoritative overlay state for a specific tab. Agent Window tabs are
|
||||||
|
* free by default; only tabs explicitly claimed through session startup,
|
||||||
|
* `tab_create`, or `tab_borrow` receive the control overlay.
|
||||||
|
*/
|
||||||
|
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
|
||||||
|
if (typeof tabId === "number" && typeof windowId === "number") {
|
||||||
|
const ctx = sessions.findByWindowId(windowId);
|
||||||
|
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
type: OVERLAY_AGENT_STATE,
|
||||||
|
sessionId: null,
|
||||||
|
mode: "hidden",
|
||||||
|
generation: overlayGeneration,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function pushOverlayStateToTab(
|
async function pushOverlayStateToTab(
|
||||||
tabId: number,
|
tabId: number,
|
||||||
state: OverlayAgentStateMessage,
|
state: OverlayAgentStateMessage,
|
||||||
@@ -89,13 +116,18 @@ export default defineBackground(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
|
||||||
|
const state = overlayStateForTab(tabId, windowId);
|
||||||
|
await pushOverlayStateToTab(tabId, state);
|
||||||
|
}
|
||||||
|
|
||||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||||
const state = overlayStateForWindow(windowId);
|
|
||||||
const tabs = await chrome.tabs.query({ windowId });
|
const tabs = await chrome.tabs.query({ windowId });
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
tabs.map((tab) =>
|
tabs.map((tab) => {
|
||||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
if (typeof tab.id !== "number") return Promise.resolve();
|
||||||
),
|
return pushOverlayStateForTab(tab.id, windowId);
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -134,23 +166,66 @@ export default defineBackground(() => {
|
|||||||
if (typeof tab.windowId !== "number") return;
|
if (typeof tab.windowId !== "number") return;
|
||||||
pushOverlayStateForAgentWindow(tab.windowId);
|
pushOverlayStateForAgentWindow(tab.windowId);
|
||||||
});
|
});
|
||||||
|
// Newly observed tabs are free unless the agent's own creation path has
|
||||||
|
// already claimed their concrete id. This listener only renders that
|
||||||
|
// state; it never infers or mutates ownership from event ordering.
|
||||||
|
chrome.tabs.onCreated.addListener((tab) => {
|
||||||
|
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
|
||||||
|
if (!sessions.findByWindowId(tab.windowId)) return;
|
||||||
|
void pushOverlayStateForTab(tab.id, tab.windowId);
|
||||||
|
});
|
||||||
|
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
|
||||||
|
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
|
||||||
|
});
|
||||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||||
// stale `true` does not keep waking us on every page load until the
|
// stale `true` does not keep waking us on every page load until the
|
||||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||||
void sessionsLive.refresh();
|
void sessionsLive.refresh();
|
||||||
const cleanupAfterDisconnect = createDisconnectCleanup({
|
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||||
manager: sessions,
|
manager: sessions,
|
||||||
sessionStopDeps: { cdp },
|
// Same contract as the dispatcher's `tool.session_stop`: without these
|
||||||
|
// deps the agent-tab cleanup and the window-release path are dead code.
|
||||||
|
sessionStopDeps: {
|
||||||
|
cdp,
|
||||||
|
tabManagement: { tabs: chromeTabMutationApi },
|
||||||
|
tabsQuery: chromeTabsApi,
|
||||||
|
},
|
||||||
onSessionsChanged: () => {
|
onSessionsChanged: () => {
|
||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const recordDeps = {
|
||||||
|
tabsApi: chrome.tabs,
|
||||||
|
cdp,
|
||||||
|
frameCoordinator: recordFrameCoordinator,
|
||||||
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
},
|
||||||
|
} satisfies RecordRuntimeDeps;
|
||||||
|
recordFrameCoordinator.attach();
|
||||||
|
attachRecordStepListener(recordDeps);
|
||||||
|
attachRecordFinishListener(recordDeps);
|
||||||
|
attachRecordQueryListener(recordDeps);
|
||||||
|
|
||||||
const dispatcher = new ToolDispatcher({
|
const dispatcher = new ToolDispatcher({
|
||||||
transport,
|
transport,
|
||||||
sessions,
|
sessions,
|
||||||
cdp,
|
cdp,
|
||||||
|
recording: recordDeps,
|
||||||
onSessionsChanged: onOverlaySessionStateChanged,
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
onBrowserControlResumed,
|
onBrowserControlResumed,
|
||||||
|
onAgentTabClaimed: (tabId, windowId) => {
|
||||||
|
void pushOverlayStateForTab(tabId, windowId);
|
||||||
|
},
|
||||||
approveBorrow: (ctx) =>
|
approveBorrow: (ctx) =>
|
||||||
requestBorrowConfirmation(ctx.tabId, {
|
requestBorrowConfirmation(ctx.tabId, {
|
||||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||||
@@ -170,27 +245,6 @@ export default defineBackground(() => {
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
const recordDeps = {
|
|
||||||
tabsApi: chrome.tabs,
|
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
|
||||||
try {
|
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
// Message listeners stay up (cheap; fire only for record message types).
|
|
||||||
// Tab / webNavigation observation attaches lazily while a recording is
|
|
||||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
|
||||||
attachRecordStepListener();
|
|
||||||
attachRecordFinishListener(recordDeps);
|
|
||||||
attachRecordQueryListener(recordDeps);
|
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
attachBorrowNotificationClickHandler({
|
attachBorrowNotificationClickHandler({
|
||||||
onClicked: chrome.notifications.onClicked,
|
onClicked: chrome.notifications.onClicked,
|
||||||
@@ -305,7 +359,11 @@ export default defineBackground(() => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (msg.kind === OVERLAY_MSG_READY) {
|
if (msg.kind === OVERLAY_MSG_READY) {
|
||||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
// Resolve from explicit tab ownership before sending so free tabs never
|
||||||
|
// flash the control mask.
|
||||||
|
if (typeof sender.tab?.id === "number") {
|
||||||
|
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,14 +7,10 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
|||||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||||
|
import { createHelpRequestData } from "@/content/help-request";
|
||||||
import overlayCss from "@/content/overlay.css?inline";
|
import overlayCss from "@/content/overlay.css?inline";
|
||||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||||
import {
|
|
||||||
handleRecordContentMessage,
|
|
||||||
isRecordContentMessage,
|
|
||||||
type RecordCaptureController,
|
|
||||||
} from "@/content/record-capture";
|
|
||||||
import {
|
import {
|
||||||
type CaptureSuppressAck,
|
type CaptureSuppressAck,
|
||||||
type CaptureSuppressMessage,
|
type CaptureSuppressMessage,
|
||||||
@@ -44,11 +40,17 @@ import {
|
|||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||||
import {
|
import {
|
||||||
|
isRecordCancelMessage,
|
||||||
|
isRecordStartMessage,
|
||||||
|
isRecordStopMessage,
|
||||||
RECORD_FINISH,
|
RECORD_FINISH,
|
||||||
RECORD_QUERY,
|
RECORD_QUERY,
|
||||||
|
type RecordCancelMessage,
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
|
type RecordStartMessage,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import type {
|
import type {
|
||||||
BorrowCancelMessage,
|
BorrowCancelMessage,
|
||||||
@@ -68,7 +70,6 @@ export default defineContentScript({
|
|||||||
if (window.top !== window) return;
|
if (window.top !== window) return;
|
||||||
|
|
||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
let overlayHost: HTMLElement | null = null;
|
let overlayHost: HTMLElement | null = null;
|
||||||
@@ -227,8 +228,6 @@ export default defineContentScript({
|
|||||||
if (previousHelp) {
|
if (previousHelp) {
|
||||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
recordCapture?.dispose();
|
|
||||||
recordCapture = null;
|
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
renderAll();
|
renderAll();
|
||||||
}
|
}
|
||||||
@@ -258,6 +257,9 @@ export default defineContentScript({
|
|||||||
| HelpRequestMessage
|
| HelpRequestMessage
|
||||||
| HelpCancelMessage
|
| HelpCancelMessage
|
||||||
| CaptureSuppressMessage
|
| CaptureSuppressMessage
|
||||||
|
| RecordStartMessage
|
||||||
|
| RecordStopMessage
|
||||||
|
| RecordCancelMessage
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAgentStateMessage
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
@@ -268,41 +270,32 @@ export default defineContentScript({
|
|||||||
return captureSuppress.handleMessage(message, sendResponse);
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordStartMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
activeRecordRequestId = message.requestId;
|
||||||
message,
|
overlays.setAgentRecordRequest({
|
||||||
{
|
id: message.requestId,
|
||||||
activeRequestId: activeRecordRequestId,
|
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
|
||||||
capture: recordCapture,
|
onFinish: () => {
|
||||||
setActiveRequestId: (id) => {
|
void chrome.runtime.sendMessage({
|
||||||
activeRecordRequestId = id;
|
type: RECORD_FINISH,
|
||||||
},
|
requestId: message.requestId,
|
||||||
setCapture: (capture) => {
|
});
|
||||||
recordCapture = capture;
|
|
||||||
},
|
|
||||||
onStart: (requestId, startedAtMs) => {
|
|
||||||
overlays.setAgentRecordRequest({
|
|
||||||
id: requestId,
|
|
||||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
|
||||||
onFinish: () => {
|
|
||||||
void chrome.runtime.sendMessage({
|
|
||||||
type: RECORD_FINISH,
|
|
||||||
requestId,
|
|
||||||
});
|
|
||||||
},
|
|
||||||
});
|
|
||||||
renderAll();
|
|
||||||
},
|
|
||||||
onStop: () => {
|
|
||||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
|
||||||
renderAll();
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
sendResponse as unknown as
|
});
|
||||||
| ((response: RecordStartAck | RecordStopAck) => void)
|
renderAll();
|
||||||
| undefined,
|
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
|
||||||
);
|
return false;
|
||||||
return needsAsync;
|
}
|
||||||
|
|
||||||
|
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||||
|
if (activeRecordRequestId !== message.requestId) return false;
|
||||||
|
overlays.clearAgentRecordRequest(message.requestId);
|
||||||
|
activeRecordRequestId = null;
|
||||||
|
renderAll();
|
||||||
|
if (isRecordStopMessage(message)) {
|
||||||
|
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -344,16 +337,7 @@ export default defineContentScript({
|
|||||||
|
|
||||||
if (isHelpRequestMessage(message)) {
|
if (isHelpRequestMessage(message)) {
|
||||||
const helpMsg = message as HelpRequestMessage;
|
const helpMsg = message as HelpRequestMessage;
|
||||||
const previousHelp = overlays.setAgentHelpRequest({
|
const previousHelp = mountHelpRequest(helpMsg);
|
||||||
id: helpMsg.requestId,
|
|
||||||
prompt: helpMsg.prompt,
|
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
|
||||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
|
||||||
selectors: helpMsg.selectors,
|
|
||||||
onContinue: (note: string) =>
|
|
||||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
|
||||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
|
||||||
});
|
|
||||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
@@ -406,17 +390,18 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||||
overlays.setAgentHelpRequest({
|
return overlays.setAgentHelpRequest(
|
||||||
id: helpMsg.requestId,
|
createHelpRequestData(helpMsg, {
|
||||||
prompt: helpMsg.prompt,
|
finish: (requestId, outcome, note) => {
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
void sendHelpFinish(requestId, outcome, note);
|
||||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
},
|
||||||
selectors: helpMsg.selectors,
|
query: () =>
|
||||||
onContinue: (note: string) =>
|
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
HelpQueryResponse | undefined
|
||||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
>,
|
||||||
});
|
}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||||
@@ -533,9 +518,6 @@ export default defineContentScript({
|
|||||||
chrome.runtime.onMessage.removeListener(onMessage);
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||||
window.removeEventListener("pageshow", onPageShow);
|
window.removeEventListener("pageshow", onPageShow);
|
||||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
|
||||||
recordCapture?.dispose();
|
|
||||||
recordCapture = null;
|
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -122,8 +122,8 @@ describe("App", () => {
|
|||||||
|
|
||||||
expect(screen.queryByText(/^扩展 v/)).toBeNull();
|
expect(screen.queryByText(/^扩展 v/)).toBeNull();
|
||||||
expect(screen.queryByText(/^daemon v/)).toBeNull();
|
expect(screen.queryByText(/^daemon v/)).toBeNull();
|
||||||
expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION);
|
expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`);
|
||||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion);
|
expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`);
|
||||||
expect(screen.getByText("03c3e47f")).toBeTruthy();
|
expect(screen.getByText("03c3e47f")).toBeTruthy();
|
||||||
|
|
||||||
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
|
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
|
||||||
@@ -200,6 +200,9 @@ describe("App", () => {
|
|||||||
expect(copied).toContain("--browser 03c3e47f");
|
expect(copied).toContain("--browser 03c3e47f");
|
||||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||||
|
expect(copied).toContain("./trace");
|
||||||
|
expect(copied).toContain("trace.json");
|
||||||
|
expect(copied).toContain("states/");
|
||||||
// Button label stays static; a transient toast confirms the copy.
|
// Button label stays static; a transient toast confirms the copy.
|
||||||
expect(copyButton.textContent).toContain("复制录制指令");
|
expect(copyButton.textContent).toContain("复制录制指令");
|
||||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||||
@@ -244,6 +247,41 @@ describe("App", () => {
|
|||||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||||
|
mockUseConnectionState.mockReturnValue({
|
||||||
|
snapshot: {
|
||||||
|
...baseSnapshot,
|
||||||
|
state: "version_skew",
|
||||||
|
instanceId: "03c3e47f",
|
||||||
|
handshake: {
|
||||||
|
server: "bh",
|
||||||
|
version: mockDaemonVersion,
|
||||||
|
protocol_version: "1.0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
statusState: "version_skew",
|
||||||
|
setLabel,
|
||||||
|
setConnectionEnabled,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
expect(screen.getByText("已连接")).toBeTruthy();
|
||||||
|
expect(screen.getByText("可升级")).toBeTruthy();
|
||||||
|
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||||
|
expect(screen.queryByText("Action needed")).toBeNull();
|
||||||
|
expect(screen.queryByText("兼容")).toBeNull();
|
||||||
|
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||||
|
expect(warning.textContent).toContain("CLI 协议");
|
||||||
|
expect(warning.textContent).toContain("扩展协议");
|
||||||
|
|
||||||
|
openRecordView();
|
||||||
|
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||||
|
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||||
|
fireEvent.click(copyButton);
|
||||||
|
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("control hints toggle", () => {
|
describe("control hints toggle", () => {
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ export function App() {
|
|||||||
}, [copiedTick]);
|
}, [copiedTick]);
|
||||||
|
|
||||||
const isSkewed = statusState === "version_skew";
|
const isSkewed = statusState === "version_skew";
|
||||||
|
const connectionLive = statusState === "connected" || isSkewed;
|
||||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||||
const extensionVersion = snapshot.extensionVersion || "—";
|
const extensionVersion = snapshot.extensionVersion || "—";
|
||||||
@@ -90,7 +91,7 @@ export function App() {
|
|||||||
setCopiedInstanceId(true);
|
setCopiedInstanceId(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||||
const recordPurpose = purposeDraft.trim();
|
const recordPurpose = purposeDraft.trim();
|
||||||
const recordStartUrl = startUrlDraft.trim();
|
const recordStartUrl = startUrlDraft.trim();
|
||||||
const recordCommand = snapshot.instanceId
|
const recordCommand = snapshot.instanceId
|
||||||
@@ -206,12 +207,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
<p
|
<p
|
||||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||||
data-slot="popup-version-skew-warning"
|
data-slot="popup-version-skew-warning"
|
||||||
>
|
>
|
||||||
{t("popup.versionSkewWarning", {
|
{t("popup.versionSkewWarning", {
|
||||||
extensionProtocol: PROTOCOL_VERSION,
|
extensionProtocol: PROTOCOL_VERSION,
|
||||||
daemonProtocol,
|
cliProtocol: daemonProtocol,
|
||||||
})}
|
})}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
@@ -266,9 +267,9 @@ export function App() {
|
|||||||
data-slot="popup-meta"
|
data-slot="popup-meta"
|
||||||
>
|
>
|
||||||
<div className="flex shrink-0 items-center gap-1">
|
<div className="flex shrink-0 items-center gap-1">
|
||||||
<span title={t("popup.extensionVersionHint")}>{extensionVersion}</span>
|
<span title={t("popup.daemonVersionHint")}>CLI {daemonVersion}</span>
|
||||||
<span aria-hidden>/</span>
|
<span aria-hidden>/</span>
|
||||||
<span title={t("popup.daemonVersionHint")}>{daemonVersion}</span>
|
<span title={t("popup.extensionVersionHint")}>Ext {extensionVersion}</span>
|
||||||
</div>
|
</div>
|
||||||
<div className="flex min-w-0 items-center justify-end gap-1">
|
<div className="flex min-w-0 items-center justify-end gap-1">
|
||||||
<span className="shrink-0">{t("popup.instanceTitle")}</span>
|
<span className="shrink-0">{t("popup.instanceTitle")}</span>
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
|
||||||
|
|
||||||
|
export default defineContentScript({
|
||||||
|
matches: ["<all_urls>"],
|
||||||
|
runAt: "document_start",
|
||||||
|
allFrames: true,
|
||||||
|
matchAboutBlank: true,
|
||||||
|
matchOriginAsFallback: true,
|
||||||
|
|
||||||
|
main(ctx) {
|
||||||
|
const dispose = attachRecordFrameAgent();
|
||||||
|
ctx.onInvalidated(dispose);
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
|||||||
import {
|
import {
|
||||||
CAPTURE_SUPPRESS,
|
CAPTURE_SUPPRESS,
|
||||||
type CaptureSuppressMessage,
|
type CaptureSuppressMessage,
|
||||||
withOverlaysHiddenForCapture,
|
withExtensionOverlayHidden,
|
||||||
} from "../capture-suppress-bridge";
|
} from "../capture-suppress-bridge";
|
||||||
|
|
||||||
function recordingSendToTab(events: string[]) {
|
function recordingSendToTab(events: string[]) {
|
||||||
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("withOverlaysHiddenForCapture", () => {
|
describe("withExtensionOverlayHidden", () => {
|
||||||
it("sends begin → fn → end and returns fn's result", async () => {
|
it("sends begin → fn → end and returns fn's result", async () => {
|
||||||
const events: string[] = [];
|
const events: string[] = [];
|
||||||
const sendToTab = recordingSendToTab(events);
|
const sendToTab = recordingSendToTab(events);
|
||||||
|
|
||||||
const result = await withOverlaysHiddenForCapture(
|
const result = await withExtensionOverlayHidden(
|
||||||
7,
|
7,
|
||||||
async () => {
|
async () => {
|
||||||
events.push("capture");
|
events.push("capture");
|
||||||
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
|||||||
const sendToTab = recordingSendToTab(events);
|
const sendToTab = recordingSendToTab(events);
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
withOverlaysHiddenForCapture(
|
withExtensionOverlayHidden(
|
||||||
7,
|
7,
|
||||||
async () => {
|
async () => {
|
||||||
events.push("capture");
|
events.push("capture");
|
||||||
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
|||||||
});
|
});
|
||||||
const fn = vi.fn(async () => "shot");
|
const fn = vi.fn(async () => "shot");
|
||||||
|
|
||||||
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
|
||||||
|
|
||||||
expect(result).toBe("shot");
|
expect(result).toBe("shot");
|
||||||
expect(fn).toHaveBeenCalledTimes(1);
|
expect(fn).toHaveBeenCalledTimes(1);
|
||||||
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
|||||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
|
||||||
|
|
||||||
expect(result).toBe("shot");
|
expect(result).toBe("shot");
|
||||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||||
|
|||||||
@@ -26,20 +26,20 @@ function handshake(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("computeConnectedState (protocol-based compat)", () => {
|
describe("computeConnectedState (protocol-based compat)", () => {
|
||||||
it("returns connected when protocol strings match", () => {
|
it("returns connected when daemon protocol equals extension protocol", () => {
|
||||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||||
kind: "connected",
|
kind: "connected",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("min_compatible_protocol");
|
expect(result.reason).toContain("min_compatible_protocol");
|
||||||
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState({
|
const result = computeConnectedState({
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.1.0",
|
min_compatible_peer: "0.1.0",
|
||||||
});
|
});
|
||||||
expect(result).toEqual({ kind: "connected" });
|
expect(result).toEqual({ kind: "connected" });
|
||||||
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
expect(result.reason).toContain("daemon min_compatible_protocol");
|
||||||
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||||
expect(second.id).not.toBe(first.id);
|
expect(second.id).not.toBe(first.id);
|
||||||
|
|
||||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
expect(controller.snapshot().state).not.toBe("connected");
|
expect(controller.snapshot().state).not.toBe("connected");
|
||||||
|
|
||||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
|
||||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { waitForDocumentSettled } from "../recording/document-settle";
|
||||||
|
|
||||||
|
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||||
|
const send = vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||||
|
send,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("waitForDocumentSettled", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels before probing a superseded observation", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const settled = waitForDocumentSettled(
|
||||||
|
cdp,
|
||||||
|
{ target: { tabId: 7 } },
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
controller.abort();
|
||||||
|
await vi.advanceTimersByTimeAsync(60);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("cancelled");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes a same-process iframe through its isolated execution context", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, {
|
||||||
|
target: { tabId: 7 },
|
||||||
|
frameId: "child-frame",
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
expect.objectContaining({ frameId: "child-frame" }),
|
||||||
|
);
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 91 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes an OOPIF probe through its CDP target session", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 27 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
|
|||||||
prompt: "log in",
|
prompt: "log in",
|
||||||
displayMode: "compact",
|
displayMode: "compact",
|
||||||
selectors: ["#login"],
|
selectors: ["#login"],
|
||||||
|
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||||
timeoutMs: 1000,
|
timeoutMs: 1000,
|
||||||
}),
|
}),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { createRandomUuid } from "../random-uuid";
|
||||||
|
|
||||||
|
describe("createRandomUuid", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses crypto.randomUUID when it is available", () => {
|
||||||
|
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
|
||||||
|
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
|
||||||
|
|
||||||
|
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
|
||||||
|
expect(randomUUID).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to getRandomValues and sets UUID v4 bits", () => {
|
||||||
|
const bytes = new Uint8Array(16).fill(0xff);
|
||||||
|
const getRandomValues = vi.fn((target: Uint8Array) => {
|
||||||
|
target.set(bytes);
|
||||||
|
return target;
|
||||||
|
});
|
||||||
|
vi.stubGlobal("crypto", { getRandomValues });
|
||||||
|
|
||||||
|
const uuid = createRandomUuid();
|
||||||
|
|
||||||
|
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
||||||
|
expect(getRandomValues).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
isAcceptedRecordStepAck,
|
||||||
|
isRecordStepMessage,
|
||||||
|
RECORD_STEP,
|
||||||
|
type RecordStepMessage,
|
||||||
|
} from "../record-bridge";
|
||||||
|
|
||||||
|
const message: RecordStepMessage = {
|
||||||
|
type: RECORD_STEP,
|
||||||
|
requestId: "rec-1",
|
||||||
|
producerId: "document-1",
|
||||||
|
sequence: 1,
|
||||||
|
step: { op: "click", target: { tag: "button", name: "Submit" } },
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("record step delivery protocol", () => {
|
||||||
|
it("requires a producer and a positive integer sequence", () => {
|
||||||
|
expect(isRecordStepMessage(message)).toBe(true);
|
||||||
|
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
|
||||||
|
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
|
||||||
|
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts only the acknowledgement for the sent sequence", () => {
|
||||||
|
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
|
||||||
|
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
|
||||||
|
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
|
||||||
|
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
|
||||||
|
|
||||||
|
class ListenerSet<T extends (...args: never[]) => unknown> {
|
||||||
|
readonly listeners = new Set<T>();
|
||||||
|
addListener = (listener: T) => this.listeners.add(listener);
|
||||||
|
removeListener = (listener: T) => this.listeners.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakePort(sender: chrome.runtime.MessageSender) {
|
||||||
|
const messages: unknown[] = [];
|
||||||
|
const onMessage = new ListenerSet<(message: unknown) => void>();
|
||||||
|
const onDisconnect = new ListenerSet<() => void>();
|
||||||
|
return {
|
||||||
|
port: {
|
||||||
|
name: RECORD_FRAME_PORT,
|
||||||
|
sender,
|
||||||
|
onMessage,
|
||||||
|
onDisconnect,
|
||||||
|
postMessage: vi.fn((message: unknown) => messages.push(message)),
|
||||||
|
disconnect: vi.fn(),
|
||||||
|
} as unknown as chrome.runtime.Port,
|
||||||
|
messages,
|
||||||
|
receive(message: unknown) {
|
||||||
|
for (const listener of onMessage.listeners) listener(message);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("RecordFrameCoordinator", () => {
|
||||||
|
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
|
||||||
|
const onMessage = new ListenerSet<
|
||||||
|
(
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: unknown) => void,
|
||||||
|
) => boolean
|
||||||
|
>();
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
onConnect.listeners.clear();
|
||||||
|
onMessage.listeners.clear();
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: { onConnect, onMessage },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("answers child-document queries for the initial tab before it is armed", () => {
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [],
|
||||||
|
sendToDocument: async () => ({ ok: true }),
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
coordinator.begin("rec-1", 10, 3);
|
||||||
|
|
||||||
|
const query = (tabId: number) => {
|
||||||
|
let response: unknown;
|
||||||
|
for (const listener of onMessage.listeners) {
|
||||||
|
listener(
|
||||||
|
{ type: RECORD_FRAME_QUERY },
|
||||||
|
{
|
||||||
|
tab: { id: tabId },
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "child-document",
|
||||||
|
} as chrome.runtime.MessageSender,
|
||||||
|
(value) => {
|
||||||
|
response = value;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
|
||||||
|
expect(query(4)).toEqual({ active: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts a child Document that appears after the initial frame snapshot", async () => {
|
||||||
|
let notifyFrameNavigation:
|
||||||
|
| ((frame: {
|
||||||
|
tabId: number;
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
lifecycle: "committed" | "completed";
|
||||||
|
}) => void)
|
||||||
|
| undefined;
|
||||||
|
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||||
|
sendToDocument,
|
||||||
|
subscribeFrameNavigation(listener) {
|
||||||
|
notifyFrameNavigation = listener;
|
||||||
|
return () => {};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
const onDocumentReady = vi.fn();
|
||||||
|
coordinator.begin("rec-1", 10, 3, onDocumentReady);
|
||||||
|
await coordinator.armTab("rec-1", 3);
|
||||||
|
sendToDocument.mockClear();
|
||||||
|
|
||||||
|
notifyFrameNavigation?.({
|
||||||
|
tabId: 3,
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "late-child",
|
||||||
|
lifecycle: "completed",
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(sendToDocument).toHaveBeenCalledWith(
|
||||||
|
3,
|
||||||
|
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
|
||||||
|
{ documentId: "late-child" },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
expect(onDocumentReady).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const child = fakePort({
|
||||||
|
tab: { id: 3 },
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "late-child",
|
||||||
|
} as chrome.runtime.MessageSender);
|
||||||
|
for (const listener of onConnect.listeners) listener(child.port);
|
||||||
|
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||||
|
notifyFrameNavigation?.({
|
||||||
|
tabId: 3,
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "late-child",
|
||||||
|
lifecycle: "completed",
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(onDocumentReady).toHaveBeenCalledWith({
|
||||||
|
tabId: 3,
|
||||||
|
documentId: "late-child",
|
||||||
|
browserFrameId: 7,
|
||||||
|
producerId: "producer-1",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
|
||||||
|
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [
|
||||||
|
{ frameId: 0, documentId: "top-document" },
|
||||||
|
{ frameId: 7, documentId: "child-document" },
|
||||||
|
],
|
||||||
|
sendToDocument,
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
coordinator.begin("rec-1", 10, 3);
|
||||||
|
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
|
||||||
|
|
||||||
|
const sender = {
|
||||||
|
tab: { id: 3 },
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "child-document",
|
||||||
|
} as chrome.runtime.MessageSender;
|
||||||
|
const frame = fakePort(sender);
|
||||||
|
for (const listener of onConnect.listeners) listener(frame.port);
|
||||||
|
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||||
|
|
||||||
|
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
|
||||||
|
tabId: 3,
|
||||||
|
documentId: "child-document",
|
||||||
|
browserFrameId: 7,
|
||||||
|
producerId: "producer-1",
|
||||||
|
});
|
||||||
|
|
||||||
|
const stopping = coordinator.stop("rec-1");
|
||||||
|
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
|
||||||
|
const stop = frame.messages.find(
|
||||||
|
(message): message is { type: "stop"; commandId: string } =>
|
||||||
|
typeof message === "object" &&
|
||||||
|
message !== null &&
|
||||||
|
(message as { type?: string }).type === "stop",
|
||||||
|
);
|
||||||
|
expect(stop).toBeDefined();
|
||||||
|
frame.receive({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: "rec-1",
|
||||||
|
commandId: stop!.commandId,
|
||||||
|
ok: true,
|
||||||
|
});
|
||||||
|
await expect(stopping).resolves.toBe(true);
|
||||||
|
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a producer used from another Document", async () => {
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||||
|
sendToDocument: async () => ({ ok: true }),
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
coordinator.begin("rec-1", 10, 3);
|
||||||
|
await coordinator.armTab("rec-1", 3);
|
||||||
|
const sender = {
|
||||||
|
tab: { id: 3 },
|
||||||
|
frameId: 0,
|
||||||
|
documentId: "top-document",
|
||||||
|
} as chrome.runtime.MessageSender;
|
||||||
|
const frame = fakePort(sender);
|
||||||
|
for (const listener of onConnect.listeners) listener(frame.port);
|
||||||
|
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
coordinator.sourceFor("rec-1", "producer-1", {
|
||||||
|
...sender,
|
||||||
|
documentId: "other-document",
|
||||||
|
}),
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
RECORD_DOCUMENT_ATTRIBUTE,
|
||||||
|
recordingDocumentMarkerValue,
|
||||||
|
} from "@/shared/recording-document-identity";
|
||||||
|
import { markRecordingDocument } from "../recording/document-marker";
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
|
||||||
|
describe("recording document marker", () => {
|
||||||
|
it("rejects a non-random Document identity", () => {
|
||||||
|
expect(() => markRecordingDocument("user@example.com")).toThrow(
|
||||||
|
"recording Document identity must be a random UUID",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restores the Document attribute exactly", () => {
|
||||||
|
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
|
||||||
|
const producerId = "123e4567-e89b-42d3-a456-426614174000";
|
||||||
|
const marker = markRecordingDocument(producerId);
|
||||||
|
|
||||||
|
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
|
||||||
|
recordingDocumentMarkerValue(producerId),
|
||||||
|
);
|
||||||
|
marker.restore();
|
||||||
|
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
|
||||||
|
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("indexes the CDP scope from the safe frame identity", () => {
|
||||||
|
const index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 3, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId: "producer-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [],
|
||||||
|
refs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(index.documentScope("producer-1")).toEqual({
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 3, sessionId: "oopif-session" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when a recording identity maps to multiple Documents", () => {
|
||||||
|
const index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "left",
|
||||||
|
target: { tabId: 3, sessionId: "left-session" },
|
||||||
|
recordingDocumentId: "duplicate",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "right",
|
||||||
|
target: { tabId: 3, sessionId: "right-session" },
|
||||||
|
recordingDocumentId: "duplicate",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [],
|
||||||
|
refs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(index.documentScope("duplicate")).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||||
|
const session = new RecordingObservationSession({ redactValues });
|
||||||
|
const state = session.registry.register({
|
||||||
|
url: URL,
|
||||||
|
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||||
|
});
|
||||||
|
session.cursor.lastSettled = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "input",
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
}),
|
||||||
|
url: URL,
|
||||||
|
};
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillTrace(value: string, redactValues: boolean) {
|
||||||
|
const session = sessionWithInput(redactValues);
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
||||||
|
value,
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
draft.postStateId = draft.preStateId;
|
||||||
|
return buildTraceV3({
|
||||||
|
registry: session.registry,
|
||||||
|
drafts: [draft],
|
||||||
|
annotations: session.annotations,
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
redactValues,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
return finalizedFillTrace(value, redactValues).states[0]!.body;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
it("omits a fill literal when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
|
||||||
|
expect(dumped).toContain("step 1: fill");
|
||||||
|
expect(dumped).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps ordinary fill details", () => {
|
||||||
|
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({
|
||||||
|
url: "https://example.com/a\nb",
|
||||||
|
title: "hello\nworld",
|
||||||
|
vomText: "@vom 1",
|
||||||
|
});
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
||||||
|
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("recording state ownership", () => {
|
||||||
|
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||||
|
const first = new RecordingStateRegistry();
|
||||||
|
const second = new RecordingStateRegistry();
|
||||||
|
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||||
|
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||||
|
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
registry.register({ url: URL, vomText: "same" });
|
||||||
|
const state = registry.register({
|
||||||
|
url: URL,
|
||||||
|
title: "Login",
|
||||||
|
vomText: "same",
|
||||||
|
truncated: true,
|
||||||
|
});
|
||||||
|
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("draft binding", () => {
|
||||||
|
it("keeps capture semantics when no observation exists", () => {
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not bind an unmatched new action to a stale observation", () => {
|
||||||
|
const session = sessionWithInput();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 2, true);
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "Confirm",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
|
||||||
|
const captureRecordingObservation = vi.hoisted(() => vi.fn());
|
||||||
|
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
|
||||||
|
|
||||||
|
vi.mock("../recording/observation-capture", async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
|
||||||
|
return { ...actual, captureRecordingObservation };
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock("../recording/document-settle", async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import("../recording/document-settle")>();
|
||||||
|
return { ...actual, waitForDocumentSettled };
|
||||||
|
});
|
||||||
|
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
function observation(url = "https://example.com/") {
|
||||||
|
return {
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
|
url,
|
||||||
|
title: "Example",
|
||||||
|
vomText: '@vom 1\nRootWebArea "Example"',
|
||||||
|
truncated: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function runtime(): RecordingObservationRuntime {
|
||||||
|
return new RecordingObservationRuntime({
|
||||||
|
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
|
||||||
|
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("RecordingObservationRuntime", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
captureRecordingObservation.mockReset();
|
||||||
|
waitForDocumentSettled.mockClear();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shares one initial capture and includes it in flush", async () => {
|
||||||
|
let release!: (value: ReturnType<typeof observation>) => void;
|
||||||
|
captureRecordingObservation.mockReturnValueOnce(
|
||||||
|
new Promise((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
const first = recording.captureInitial(7);
|
||||||
|
const second = recording.captureInitial(7);
|
||||||
|
const flushed = recording.flush();
|
||||||
|
let flushCompleted = false;
|
||||||
|
void flushed.then(() => {
|
||||||
|
flushCompleted = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
|
||||||
|
expect(flushCompleted).toBe(false);
|
||||||
|
|
||||||
|
release(observation());
|
||||||
|
await Promise.all([first, second, flushed]);
|
||||||
|
expect(flushCompleted).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels an in-flight initial capture", async () => {
|
||||||
|
captureRecordingObservation.mockImplementationOnce(
|
||||||
|
(input: { signal?: AbortSignal }) =>
|
||||||
|
new Promise((_, reject) => {
|
||||||
|
input.signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
const pending = recording.captureInitial(7);
|
||||||
|
recording.cancel();
|
||||||
|
|
||||||
|
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries a failed initial capture when the recording settles at stop", async () => {
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockRejectedValueOnce(new Error("document swapped"))
|
||||||
|
.mockResolvedValueOnce(observation());
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
|
||||||
|
await recording.settleTrailing(7, []);
|
||||||
|
const trace = recording.buildTrace({
|
||||||
|
drafts: [],
|
||||||
|
startedAt: "2026-08-19T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "0.1.6",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
||||||
|
expect(trace.states).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps runtime matching data out of the serialized trace", async () => {
|
||||||
|
const captured = observation();
|
||||||
|
captured.index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
|
||||||
|
});
|
||||||
|
captureRecordingObservation.mockResolvedValueOnce(captured);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
const dumped = JSON.stringify(
|
||||||
|
recording.buildTrace({
|
||||||
|
drafts: [],
|
||||||
|
startedAt: "2026-08-19T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "0.1.6",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(dumped).toContain("RootWebArea");
|
||||||
|
expect(dumped).not.toContain("matchNodes");
|
||||||
|
expect(dumped).not.toContain("localRect");
|
||||||
|
expect(dumped).not.toContain("backendNodeId");
|
||||||
|
expect(dumped).not.toContain("ObservationNodeIndex");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("captures a tab transition without sharing observation cursors", async () => {
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockResolvedValueOnce(observation("https://example.com/first"))
|
||||||
|
.mockResolvedValueOnce(observation("https://example.com/second"));
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(4);
|
||||||
|
const transition = await recording.captureTabTransition(4, 5);
|
||||||
|
await recording.captureInitial(4);
|
||||||
|
await recording.captureInitial(5);
|
||||||
|
|
||||||
|
expect(transition).toEqual({
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s2",
|
||||||
|
targetUrl: "https://example.com/second",
|
||||||
|
});
|
||||||
|
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("binds an iframe draft to its marked CDP Document scope", async () => {
|
||||||
|
captureRecordingObservation.mockResolvedValueOnce({
|
||||||
|
...observation(),
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 7, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId: "producer-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const recording = runtime();
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "click" as const,
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||||
|
recording.cancel();
|
||||||
|
|
||||||
|
const draft = drafts[0];
|
||||||
|
expect(draft?.op).toBe("click");
|
||||||
|
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||||
|
expect(draft.targetHint).toMatchObject({
|
||||||
|
frameId: "child",
|
||||||
|
geometrySpace: "local",
|
||||||
|
});
|
||||||
|
expect(draft.matchedTarget?.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refreshes a late iframe before matching its first action", async () => {
|
||||||
|
const iframeObservation = {
|
||||||
|
...observation(),
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 7, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId: "producer-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [
|
||||||
|
{
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
role: "button",
|
||||||
|
name: "表格视图",
|
||||||
|
line: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockResolvedValueOnce(observation())
|
||||||
|
.mockResolvedValueOnce(iframeObservation)
|
||||||
|
.mockResolvedValueOnce(iframeObservation);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
await recording.refreshDocument(7, "producer-1");
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "表格视图" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||||
|
recording.cancel();
|
||||||
|
|
||||||
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
|
||||||
|
expect(waitForDocumentSettled).toHaveBeenCalledWith(
|
||||||
|
expect.anything(),
|
||||||
|
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
|
||||||
|
{ signal: expect.any(AbortSignal) },
|
||||||
|
);
|
||||||
|
const draft = drafts[0];
|
||||||
|
expect(draft?.op).toBe("click");
|
||||||
|
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||||
|
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not poison final flush when a frame readiness refresh fails", async () => {
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockResolvedValueOnce(observation())
|
||||||
|
.mockRejectedValueOnce(new Error("child Document replaced"));
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
|
||||||
|
"child Document replaced",
|
||||||
|
);
|
||||||
|
await expect(recording.flush()).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
|
||||||
|
|
||||||
describe("recording-step-buffer", () => {
|
|
||||||
it("stores semantic click without summary", () => {
|
|
||||||
const buffer = { steps: [], pendingNavigation: false };
|
|
||||||
appendRecordedPayload(buffer, {
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
expects_navigation: true,
|
|
||||||
});
|
|
||||||
expect(buffer.steps).toEqual([
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
expect(buffer.pendingNavigation).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [
|
|
||||||
{
|
|
||||||
op: "click" as const,
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
currentUrl: "https://example.com/a",
|
|
||||||
pendingNavigation: true,
|
|
||||||
pendingNavigationDeadline: Date.now() + 5_000,
|
|
||||||
};
|
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
|
||||||
expect(buffer.steps).toEqual([
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
navigated_to: "https://example.com/b",
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [
|
|
||||||
{
|
|
||||||
op: "select" as const,
|
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
|
||||||
values: ["tech"],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
currentUrl: "https://example.com/list",
|
|
||||||
pendingNavigation: true,
|
|
||||||
pendingNavigationDeadline: Date.now() + 5_000,
|
|
||||||
};
|
|
||||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
|
||||||
expect(buffer.steps).toEqual([
|
|
||||||
{
|
|
||||||
op: "select",
|
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
|
||||||
values: ["tech"],
|
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("emits navigate for uncaused URL changes", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [],
|
|
||||||
currentUrl: "https://example.com/a",
|
|
||||||
pendingNavigation: false,
|
|
||||||
};
|
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
|
||||||
expect(buffer.steps[0]).toMatchObject({
|
|
||||||
op: "navigate",
|
|
||||||
url: "https://example.com/b",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { SettleController } from "../recording/settle-controller";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const OBSERVATION: RegisteredObservation = {
|
||||||
|
stateId: "s-next",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
|
url: "https://example.com/next",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("SettleController", () => {
|
||||||
|
afterEach(() => vi.useRealTimers());
|
||||||
|
|
||||||
|
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
let firstSignal: AbortSignal | undefined;
|
||||||
|
vi.spyOn(session, "capture")
|
||||||
|
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
||||||
|
firstSignal = signal;
|
||||||
|
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce(OBSERVATION);
|
||||||
|
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
controller.schedule(drafts, 0);
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
drafts.push({
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", name: "Second" },
|
||||||
|
preStateId: "s-next",
|
||||||
|
});
|
||||||
|
controller.schedule(drafts, 1);
|
||||||
|
expect(firstSignal?.aborted).toBe(true);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flush();
|
||||||
|
expect(drafts[1]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||||
|
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||||
|
resolveFirstTab = resolve;
|
||||||
|
});
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi
|
||||||
|
.fn()
|
||||||
|
.mockImplementationOnce(() => firstTab)
|
||||||
|
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
session.cursor.lastSettled = OBSERVATION;
|
||||||
|
const drafts: RecordingDraftStep[] = [];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
||||||
|
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flushRedirects();
|
||||||
|
|
||||||
|
expect(drafts).toEqual([]);
|
||||||
|
expect(controller.hasPending).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("settles an action in its own OOPIF document scope", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
|
||||||
|
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
await controller.flush();
|
||||||
|
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 39 }),
|
||||||
|
);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
||||||
|
|
||||||
|
describe("recording-step-buffer", () => {
|
||||||
|
it("stores semantic click without summary", () => {
|
||||||
|
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||||
|
appendRecordedPayload(buffer, {
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "button", role: "button", name: "发布" },
|
||||||
|
expects_navigation: true,
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(buffer.navigation.pendingNavigation).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the hovered element description as a capture fallback", () => {
|
||||||
|
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||||
|
appendRecordedPayload(
|
||||||
|
buffer,
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
op: "click" as const,
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://example.com/a",
|
||||||
|
pendingNavigation: true,
|
||||||
|
pendingNavigationDeadline: Date.now() + 5_000,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
|
navigatedTo: "https://example.com/b",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
op: "select" as const,
|
||||||
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
|
values: ["tech"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://example.com/list",
|
||||||
|
pendingNavigation: true,
|
||||||
|
pendingNavigationDeadline: Date.now() + 5_000,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "select",
|
||||||
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
|
values: ["tech"],
|
||||||
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits navigate for uncaused URL changes", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
|
||||||
|
};
|
||||||
|
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||||
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
|
expect(buffer.steps[0]).toMatchObject({
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/b",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://passport.example/login",
|
||||||
|
pendingNavigation: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const hop1 = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
"https://passport.example/callback",
|
||||||
|
false,
|
||||||
|
"link",
|
||||||
|
["server_redirect"],
|
||||||
|
);
|
||||||
|
expect(hop1).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://passport.example/callback",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
|
||||||
|
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||||
|
"client_redirect",
|
||||||
|
]);
|
||||||
|
expect(hop2).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://app.example/dashboard",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reuse redirect metadata for a later content-observed URL change", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://example.com/redirect",
|
||||||
|
pendingNavigation: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
|
||||||
|
|
||||||
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
|
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
|
||||||
|
|
||||||
|
describe("RecordingTabCoordinator", () => {
|
||||||
|
it("keeps navigation state isolated per tab", () => {
|
||||||
|
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
|
||||||
|
const first = tabs.navigation(4);
|
||||||
|
first.pendingNavigation = true;
|
||||||
|
const second = tabs.navigation(5, "https://example.com/second");
|
||||||
|
|
||||||
|
expect(second).toEqual({
|
||||||
|
currentUrl: "https://example.com/second",
|
||||||
|
pendingNavigation: false,
|
||||||
|
});
|
||||||
|
expect(tabs.navigation(4)).toBe(first);
|
||||||
|
expect(tabs.navigation(4).pendingNavigation).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("invalidates an earlier activation when a newer tab becomes active", () => {
|
||||||
|
const tabs = new RecordingTabCoordinator(4);
|
||||||
|
const second = tabs.noteActivation(5);
|
||||||
|
const third = tabs.noteActivation(6);
|
||||||
|
|
||||||
|
expect(tabs.isLatest(second)).toBe(false);
|
||||||
|
expect(tabs.isLatest(third)).toBe(true);
|
||||||
|
tabs.commit(6);
|
||||||
|
expect(tabs.currentTabId).toBe(6);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("commits the freshly observed URL when returning to an existing tab", () => {
|
||||||
|
const tabs = new RecordingTabCoordinator(4);
|
||||||
|
tabs.navigation(5, "https://example.com/old");
|
||||||
|
|
||||||
|
tabs.commit(5, "https://example.com/new");
|
||||||
|
|
||||||
|
expect(tabs.currentTabId).toBe(5);
|
||||||
|
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { matchObservationTarget } from "../recording/target-matcher";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
frameId,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function observation(
|
||||||
|
matchNodes: CaptureVomMatchNode[],
|
||||||
|
refs: RenderedRef[],
|
||||||
|
): RegisteredObservation {
|
||||||
|
return {
|
||||||
|
stateId: "s1",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
||||||
|
url: "https://example.com",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchObservationTarget", () => {
|
||||||
|
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
||||||
|
const geometry = node(42, "child");
|
||||||
|
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
||||||
|
const ref: RenderedRef = {
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
line: 1,
|
||||||
|
};
|
||||||
|
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
||||||
|
|
||||||
|
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a unique node using canonical top-level viewport geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "left"), node(42, "right")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: "right",
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches iframe capture geometry in the child viewport coordinate space", () => {
|
||||||
|
const child = node(42, "child", 410);
|
||||||
|
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[child],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: "child",
|
||||||
|
geometrySpace: "local",
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restricts a missing frame hint to the root frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "child")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when the source Document has no VOM frame mapping", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: null,
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
fallback: { tag: "button", role: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched for ambiguous geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(41, "root"), node(42, "child")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { frameId: "child" },
|
||||||
|
fallback: { tag: "button", role: "button", name: "保存" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", role: "button", name: "取消" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
});
|
||||||
+50
-34
@@ -1,6 +1,15 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
import { shouldRecordPress } from "../recording/draft-policy";
|
||||||
|
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
||||||
|
|
||||||
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
||||||
|
return buildTraceV2({
|
||||||
|
steps,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
...(startUrl ? { startUrl } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
describe("shouldRecordPress", () => {
|
||||||
it("keeps Enter and Escape", () => {
|
it("keeps Enter and Escape", () => {
|
||||||
@@ -18,36 +27,47 @@ describe("shouldRecordPress", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
describe("reduceTraceSteps", () => {
|
||||||
|
it("does not expose internal tab transitions in trace v2", () => {
|
||||||
|
const trace = reduceTraceSteps([
|
||||||
|
{
|
||||||
|
op: "switch_tab",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s2",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(trace.steps).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
it("builds steps with pages dictionary and page id references", () => {
|
||||||
const drafts: DraftTraceStep[] = [
|
const drafts: RecordingDraftStep[] = [
|
||||||
{
|
{
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
url: "https://example.com/search?q=hello&utm_source=x",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
value: "browser skill",
|
value: "browser skill",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "Enter",
|
key: "Enter",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
navigated_to: "https://example.com/results/42",
|
navigatedTo: "https://example.com/results/42",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
navigated_to: "https://example.com/p/99",
|
navigatedTo: "https://example.com/p/99",
|
||||||
page_url: "https://example.com/results/42",
|
pageUrl: "https://example.com/results/42",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "a",
|
key: "a",
|
||||||
page_url: "https://example.com/p/99",
|
pageUrl: "https://example.com/p/99",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -109,9 +129,9 @@ describe("reduceTraceSteps", () => {
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "Search query" },
|
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
||||||
value: "",
|
value: "",
|
||||||
page_url: "https://example.com/search",
|
pageUrl: "https://example.com/search",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/search",
|
"https://example.com/search",
|
||||||
@@ -125,16 +145,16 @@ describe("reduceTraceSteps", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
const { pages, steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
labels: ["技术"],
|
labels: ["技术"],
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
page_url: "https://example.com/list",
|
pageUrl: "https://example.com/list",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/list",
|
"https://example.com/list",
|
||||||
@@ -151,36 +171,32 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps hover steps before menu clicks", () => {
|
it("drops hover steps, which only exist in trace v3", () => {
|
||||||
const { steps } = reduceTraceSteps(
|
const { steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { tag: "span", role: "button", name: "Account" },
|
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||||
page_url: "https://example.com/app",
|
pageUrl: "https://example.com/app",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "a", role: "link", name: "Profile" },
|
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||||
page_url: "https://example.com/app",
|
pageUrl: "https://example.com/app",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/app",
|
"https://example.com/app",
|
||||||
);
|
);
|
||||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
op: "hover",
|
|
||||||
page: "p1",
|
|
||||||
target: { name: "Account" },
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
buildTraceV2({
|
||||||
[{ op: "navigate", url: "https://example.com/other" }],
|
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
||||||
"https://example.com/start",
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
),
|
startUrl: "https://example.com/start",
|
||||||
|
}).entry.start_url,
|
||||||
).toBe("https://example.com/start");
|
).toBe("https://example.com/start");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
describe("trace reducer v3", () => {
|
||||||
|
it("removes form literals when value redaction is requested", () => {
|
||||||
|
const reduced = reduceTraceStepsV3(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
value: "private@example.com",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "select",
|
||||||
|
values: ["private-account-id"],
|
||||||
|
labels: ["Private account"],
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{ redactValues: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
|
||||||
|
expect(reduced.steps[1]).toMatchObject({ op: "select" });
|
||||||
|
expect(reduced.steps[1]).not.toHaveProperty("selection");
|
||||||
|
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
|
||||||
|
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits tab transitions only for callers that advertised support", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "switch_tab",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s2",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
|
||||||
|
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
|
||||||
|
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not export a tab transition without both observation endpoints", () => {
|
||||||
|
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
|
||||||
|
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
|
||||||
|
|
||||||
|
expect(
|
||||||
|
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
|
||||||
|
).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/final",
|
||||||
|
transitionQualifiers: ["server_redirect"],
|
||||||
|
preStateId: "s2",
|
||||||
|
postStateId: "s3",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const before = structuredClone(drafts);
|
||||||
|
const reduced = reduceTraceStepsV3(drafts);
|
||||||
|
|
||||||
|
expect(drafts).toEqual(before);
|
||||||
|
expect(reduced.steps).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
id: 1,
|
||||||
|
op: "navigate",
|
||||||
|
state: "s1",
|
||||||
|
to: "https://example.com/final",
|
||||||
|
result: { state: "s3" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds the wire model from protocol constants", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||||
|
preStateId: state.id,
|
||||||
|
postStateId: state.id,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(trace.version).toBe(TRACE_VERSION_V3);
|
||||||
|
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
||||||
|
expect(trace.steps[0]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { role: "button", name: "Save", unmatched: true },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
/**
|
/**
|
||||||
* Wire protocol and background-side helper for hiding the extension's
|
* Wire protocol and background-side helper for temporarily excluding the
|
||||||
* in-page overlay while a screenshot is captured, so captured frames only
|
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
|
||||||
* contain the page itself.
|
* coordinate-driven native input uses it so Chrome hit-tests the page.
|
||||||
*
|
*
|
||||||
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
|
||||||
* which sends `begin` (the content script hides the overlay host and only
|
* which sends `begin` (the content script hides the overlay host and only
|
||||||
* acks once the compositor has produced an overlay-free frame) and always
|
* acks once the compositor has produced an overlay-free frame) and always
|
||||||
* follows up with `end`, even when the capture throws. Tabs without the
|
* follows up with `end`, even when the capture throws. Tabs without the
|
||||||
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
|||||||
* (or any other `begin` failure) simply skips suppression — there is no
|
* (or any other `begin` failure) simply skips suppression — there is no
|
||||||
* overlay to hide in that tab.
|
* overlay to hide in that tab.
|
||||||
*/
|
*/
|
||||||
export async function withOverlaysHiddenForCapture<T>(
|
export async function withExtensionOverlayHidden<T>(
|
||||||
tabId: number,
|
tabId: number,
|
||||||
fn: () => Promise<T>,
|
fn: () => Promise<T>,
|
||||||
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
/**
|
/**
|
||||||
* Build a semantic TargetDescriptor for an interacted element.
|
* Build a semantic capture descriptor for an interacted element.
|
||||||
*
|
*
|
||||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||||
* on screen (usually a short visible name). Tag-only noise like
|
* on screen (usually a short visible name). Tag-only noise like
|
||||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
/** Content-script capture descriptor before VOM geometric matching. */
|
||||||
|
export interface CaptureTargetDescriptor {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
|||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
* Recording “点击div” with no name fails this bar.
|
* Recording “点击div” with no name fails this bar.
|
||||||
*/
|
*/
|
||||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
const name = target.name?.trim();
|
const name = target.name?.trim();
|
||||||
if (name && isActionableLabel(name)) return true;
|
if (name && isActionableLabel(name)) return true;
|
||||||
if (
|
if (
|
||||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeTarget(el: Element): TargetDescriptor {
|
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||||
const tag = el.tagName.toLowerCase();
|
const tag = el.tagName.toLowerCase();
|
||||||
const role = inferRole(el);
|
const role = inferRole(el);
|
||||||
const name = accessibleName(el);
|
const name = accessibleName(el);
|
||||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const clickable = resolveClickableElement(target);
|
const clickable = resolveClickableElement(target);
|
||||||
if (!clickable) return null;
|
if (!clickable) return null;
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
|
|||||||
export const HELP_QUERY = "bsk-help-query";
|
export const HELP_QUERY = "bsk-help-query";
|
||||||
export const HELP_FINISH = "bsk-help-finish";
|
export const HELP_FINISH = "bsk-help-finish";
|
||||||
|
|
||||||
|
export interface HelpHighlightRect {
|
||||||
|
top: number;
|
||||||
|
left: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface HelpRequestMessage {
|
export interface HelpRequestMessage {
|
||||||
type: typeof HELP_REQUEST;
|
type: typeof HELP_REQUEST;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
|
|||||||
displayMode?: "full" | "compact";
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||||
|
rects?: HelpHighlightRect[];
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
|||||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||||
Array.isArray(m.selectors) &&
|
Array.isArray(m.selectors) &&
|
||||||
m.selectors.every((selector) => typeof selector === "string") &&
|
m.selectors.every((selector) => typeof selector === "string") &&
|
||||||
|
(m.rects === undefined ||
|
||||||
|
(Array.isArray(m.rects) &&
|
||||||
|
m.rects.every(
|
||||||
|
(rect) =>
|
||||||
|
typeof rect === "object" &&
|
||||||
|
rect !== null &&
|
||||||
|
["top", "left", "width", "height"].every((key) =>
|
||||||
|
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||||
|
),
|
||||||
|
))) &&
|
||||||
typeof m.timeoutMs === "number"
|
typeof m.timeoutMs === "number"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,10 @@
|
|||||||
* between the content-script control overlay and the background SW.
|
* between the content-script control overlay and the background SW.
|
||||||
*
|
*
|
||||||
* Content script → background:
|
* Content script → background:
|
||||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
|
||||||
* overlay state for the sender's window.
|
* (see overlayStateForTab) and pushes the authoritative overlay state for
|
||||||
|
* that tab. It also proactively pushes on tab create and control-mode
|
||||||
|
* change, so a tab usually receives state before it ever sends ready.
|
||||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||||
* every inflight + queued tool call for that session with
|
* every inflight + queued tool call for that session with
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
/**
|
||||||
|
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
|
||||||
|
*
|
||||||
|
* Crypto.randomUUID() is restricted to secure contexts, while
|
||||||
|
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
|
||||||
|
* origins. BrowserSkill records many intranet pages served over HTTP, so the
|
||||||
|
* fallback is required for recording to work on those pages.
|
||||||
|
*/
|
||||||
|
export function createRandomUuid(): string {
|
||||||
|
const cryptoApi = globalThis.crypto;
|
||||||
|
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
|
||||||
|
|
||||||
|
const bytes = new Uint8Array(16);
|
||||||
|
cryptoApi.getRandomValues(bytes);
|
||||||
|
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
|
||||||
|
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
||||||
|
|
||||||
|
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||||
|
return [
|
||||||
|
hex.slice(0, 8),
|
||||||
|
hex.slice(8, 12),
|
||||||
|
hex.slice(12, 16),
|
||||||
|
hex.slice(16, 20),
|
||||||
|
hex.slice(20),
|
||||||
|
].join("-");
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
* service worker and a tab's content script.
|
* service worker and a tab's content script.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -16,6 +16,10 @@ export interface RecordStartAck {
|
|||||||
ok: true;
|
ok: true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordStepAck =
|
||||||
|
| { ok: true; sequence: number }
|
||||||
|
| { ok: false; expectedSequence: number; error: string };
|
||||||
|
|
||||||
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
||||||
|
|
||||||
export interface RecordQueryMessage {
|
export interface RecordQueryMessage {
|
||||||
@@ -42,7 +46,7 @@ export interface RecordStartMessage {
|
|||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||||
target?: TargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||||
@@ -50,17 +54,30 @@ export interface RecordStepPayload {
|
|||||||
labels?: string[];
|
labels?: string[];
|
||||||
url?: string;
|
url?: string;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
|
commit?: "enter" | "suggestion" | "blur";
|
||||||
/** Page URL when the step was captured. */
|
/** Page URL when the step was captured. */
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
/** Event-target bounds in the source frame's viewport coordinate space. */
|
||||||
|
geometry?: {
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
tag: string;
|
||||||
|
};
|
||||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||||
expects_navigation?: boolean;
|
expects_navigation?: boolean;
|
||||||
/** Whether an observed URL change was synchronously caused by the action. */
|
/** Whether an observed URL change was synchronously caused by the action. */
|
||||||
navigation_caused_by_action?: boolean;
|
navigation_caused_by_action?: boolean;
|
||||||
|
/** Raw webNavigation transition metadata for cause mapping. */
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepMessage {
|
export interface RecordStepMessage {
|
||||||
type: typeof RECORD_STEP;
|
type: typeof RECORD_STEP;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
|
/** Stable for the lifetime of one content-script document. */
|
||||||
|
producerId: string;
|
||||||
|
/** Monotonic within one content-script producer and recording request. */
|
||||||
|
sequence: number;
|
||||||
step: RecordStepPayload;
|
step: RecordStepPayload;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,8 +129,27 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
|||||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
|
if (
|
||||||
|
m.type !== RECORD_STEP ||
|
||||||
|
typeof m.requestId !== "string" ||
|
||||||
|
typeof m.producerId !== "string" ||
|
||||||
|
m.producerId.length === 0 ||
|
||||||
|
typeof m.sequence !== "number" ||
|
||||||
|
!Number.isSafeInteger(m.sequence) ||
|
||||||
|
m.sequence < 1
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
const step = m.step;
|
const step = m.step;
|
||||||
if (typeof step !== "object" || step === null) return false;
|
if (typeof step !== "object" || step === null) return false;
|
||||||
return typeof (step as RecordStepPayload).op === "string";
|
return typeof (step as RecordStepPayload).op === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isAcceptedRecordStepAck(
|
||||||
|
value: unknown,
|
||||||
|
sequence: number,
|
||||||
|
): value is Extract<RecordStepAck, { ok: true }> {
|
||||||
|
if (typeof value !== "object" || value === null) return false;
|
||||||
|
const ack = value as Partial<RecordStepAck>;
|
||||||
|
return ack.ok === true && ack.sequence === sequence;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,136 +0,0 @@
|
|||||||
import type { DraftTraceStep } from "@/transport/types";
|
|
||||||
import type { RecordStepPayload } from "./record-bridge";
|
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
|
||||||
steps: DraftTraceStep[];
|
|
||||||
currentUrl?: string;
|
|
||||||
pendingNavigation: boolean;
|
|
||||||
pendingNavigationDeadline?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
|
||||||
|
|
||||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|
||||||
const pageUrl = payload.page_url;
|
|
||||||
switch (payload.op) {
|
|
||||||
case "click":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "click",
|
|
||||||
target: payload.target,
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "hover":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "hover",
|
|
||||||
target: payload.target,
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "fill":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "fill",
|
|
||||||
target: payload.target,
|
|
||||||
value: payload.value ?? "",
|
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "press":
|
|
||||||
return payload.key
|
|
||||||
? {
|
|
||||||
op: "press",
|
|
||||||
key: payload.key,
|
|
||||||
...(payload.target ? { target: payload.target } : {}),
|
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "select":
|
|
||||||
return payload.target && payload.values
|
|
||||||
? {
|
|
||||||
op: "select",
|
|
||||||
target: payload.target,
|
|
||||||
values: payload.values,
|
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "navigate":
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
|
||||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
|
||||||
const step = buffer.steps[i];
|
|
||||||
if (!step) continue;
|
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
|
||||||
buffer.steps[i] = { ...step, navigated_to: url };
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function observeRecordedNavigation(
|
|
||||||
buffer: RecordingStepBuffer,
|
|
||||||
url: string,
|
|
||||||
causedByAction?: boolean,
|
|
||||||
): void {
|
|
||||||
if (!url || url === buffer.currentUrl) return;
|
|
||||||
buffer.currentUrl = url;
|
|
||||||
const pendingIsCurrent =
|
|
||||||
buffer.pendingNavigation &&
|
|
||||||
(buffer.pendingNavigationDeadline === undefined ||
|
|
||||||
buffer.pendingNavigationDeadline >= Date.now());
|
|
||||||
|
|
||||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
|
||||||
buffer.pendingNavigation = false;
|
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
|
||||||
if (!annotateLastStepNavigation(buffer, url)) {
|
|
||||||
buffer.steps.push({
|
|
||||||
op: "navigate",
|
|
||||||
url,
|
|
||||||
page_url: url,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
buffer.pendingNavigation = false;
|
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
|
||||||
buffer.steps.push({
|
|
||||||
op: "navigate",
|
|
||||||
url,
|
|
||||||
page_url: url,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function appendRecordedPayload(
|
|
||||||
buffer: RecordingStepBuffer,
|
|
||||||
payload: RecordStepPayload,
|
|
||||||
): void {
|
|
||||||
if (payload.op === "navigate") {
|
|
||||||
if (payload.url) {
|
|
||||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const step = toDraftStep({
|
|
||||||
...payload,
|
|
||||||
page_url: payload.page_url ?? buffer.currentUrl,
|
|
||||||
});
|
|
||||||
if (!step) return;
|
|
||||||
buffer.steps.push(step);
|
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
|
||||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
|
||||||
buffer.pendingNavigationDeadline = buffer.pendingNavigation
|
|
||||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
|
||||||
: undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import {
|
||||||
|
RECORD_DOCUMENT_ATTRIBUTE,
|
||||||
|
recordingDocumentMarkerValue,
|
||||||
|
} from "@/shared/recording-document-identity";
|
||||||
|
|
||||||
|
export interface RecordingDocumentMarker {
|
||||||
|
restore(): void;
|
||||||
|
ensure(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function waitForDocumentElement(): Promise<HTMLElement> {
|
||||||
|
if (document.documentElement) return Promise.resolve(document.documentElement);
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
if (!document.documentElement) return;
|
||||||
|
observer.disconnect();
|
||||||
|
resolve(document.documentElement);
|
||||||
|
});
|
||||||
|
observer.observe(document, { childList: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markRecordingDocument(
|
||||||
|
producerId: string,
|
||||||
|
root: HTMLElement = document.documentElement,
|
||||||
|
): RecordingDocumentMarker {
|
||||||
|
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
const markerValue = recordingDocumentMarkerValue(producerId);
|
||||||
|
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
|
||||||
|
ensure();
|
||||||
|
return {
|
||||||
|
ensure,
|
||||||
|
restore() {
|
||||||
|
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
|
||||||
|
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
||||||
|
|
||||||
|
const SETTLE_MIN_MS = 150;
|
||||||
|
const SETTLE_QUIET_MS = 250;
|
||||||
|
const SETTLE_MAX_MS = 2_000;
|
||||||
|
const SETTLE_POLL_MS = 60;
|
||||||
|
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
||||||
|
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export interface DocumentSettleScope {
|
||||||
|
target: CdpTarget;
|
||||||
|
/** Omit only for the target's root document. */
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProbeContext {
|
||||||
|
executionContextId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executionContextForScope(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
): Promise<number | undefined> {
|
||||||
|
if (!scope.frameId) return undefined;
|
||||||
|
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
{
|
||||||
|
frameId: scope.frameId,
|
||||||
|
worldName: SETTLE_WORLD_NAME,
|
||||||
|
grantUniveralAccess: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return result.executionContextId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
context: ProbeContext,
|
||||||
|
): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
if (scope.frameId && context.executionContextId === undefined) {
|
||||||
|
context.executionContextId = await executionContextForScope(cdp, scope);
|
||||||
|
if (context.executionContextId === undefined) return null;
|
||||||
|
}
|
||||||
|
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
...(context.executionContextId !== undefined
|
||||||
|
? { contextId: context.executionContextId }
|
||||||
|
: {}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
context.executionContextId = undefined;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForDocumentSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
options: { signal?: AbortSignal } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
const context: ProbeContext = {};
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS, options.signal);
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, scope, context);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe || probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
|
|
||||||
|
export function shouldRecordPress(
|
||||||
|
key: string,
|
||||||
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||||
|
): boolean {
|
||||||
|
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||||
|
const mods = modifiers ?? [];
|
||||||
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
||||||
|
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveDraftStartUrl(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
explicitStartUrl?: string,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): string {
|
||||||
|
if (explicitStartUrl) return explicitStartUrl;
|
||||||
|
const navigation = drafts.find(
|
||||||
|
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
||||||
|
);
|
||||||
|
if (navigation) return navigation.url;
|
||||||
|
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
export const RECORD_FRAME_PORT = "bsk-record-frame";
|
||||||
|
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
|
||||||
|
export const RECORD_FRAME_START = "bsk-record-frame-start";
|
||||||
|
|
||||||
|
export interface RecordFrameQueryMessage {
|
||||||
|
type: typeof RECORD_FRAME_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordFrameQueryResponse {
|
||||||
|
active: boolean;
|
||||||
|
requestId?: string;
|
||||||
|
startedAtMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordFrameStartMessage {
|
||||||
|
type: typeof RECORD_FRAME_START;
|
||||||
|
requestId: string;
|
||||||
|
startedAtMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordFramePortMessage =
|
||||||
|
| {
|
||||||
|
type: "ready";
|
||||||
|
requestId: string;
|
||||||
|
producerId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "ready_ack";
|
||||||
|
requestId: string;
|
||||||
|
producerId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "stop";
|
||||||
|
requestId: string;
|
||||||
|
commandId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "cancel";
|
||||||
|
requestId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "stopped";
|
||||||
|
requestId: string;
|
||||||
|
commandId: string;
|
||||||
|
ok: boolean;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
|
||||||
|
return (
|
||||||
|
typeof value === "object" &&
|
||||||
|
value !== null &&
|
||||||
|
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
|
||||||
|
if (typeof value !== "object" || value === null) return false;
|
||||||
|
const message = value as Partial<RecordFrameStartMessage>;
|
||||||
|
return (
|
||||||
|
message.type === RECORD_FRAME_START &&
|
||||||
|
typeof message.requestId === "string" &&
|
||||||
|
typeof message.startedAtMs === "number"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
import { createRandomUuid } from "../random-uuid";
|
||||||
|
import {
|
||||||
|
isRecordFrameQueryMessage,
|
||||||
|
RECORD_FRAME_PORT,
|
||||||
|
RECORD_FRAME_START,
|
||||||
|
type RecordFramePortMessage,
|
||||||
|
type RecordFrameQueryResponse,
|
||||||
|
type RecordFrameStartMessage,
|
||||||
|
} from "./frame-bridge";
|
||||||
|
|
||||||
|
export interface RecordingCaptureScope {
|
||||||
|
tabId: number;
|
||||||
|
documentId: string;
|
||||||
|
browserFrameId: number;
|
||||||
|
producerId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ArmedRecording {
|
||||||
|
requestId: string;
|
||||||
|
startedAtMs: number;
|
||||||
|
tabIds: Set<number>;
|
||||||
|
agents: Map<string, FrameAgent>;
|
||||||
|
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
|
||||||
|
finishing: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FrameAgent extends RecordingCaptureScope {
|
||||||
|
requestId: string;
|
||||||
|
port: chrome.runtime.Port;
|
||||||
|
stopWaiters: Map<string, (ok: boolean) => void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BrowserFrame {
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BrowserFrameNavigation extends BrowserFrame {
|
||||||
|
tabId: number;
|
||||||
|
lifecycle: "committed" | "completed";
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
|
||||||
|
|
||||||
|
export interface RecordFrameCoordinatorDeps {
|
||||||
|
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
|
||||||
|
sendToDocument(
|
||||||
|
tabId: number,
|
||||||
|
message: RecordFrameStartMessage,
|
||||||
|
target: { documentId?: string; frameId?: number },
|
||||||
|
): Promise<unknown>;
|
||||||
|
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function documentKey(tabId: number, documentId: string): string {
|
||||||
|
return `${tabId}:${documentId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function senderAddress(
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
): { tabId: number; documentId: string; browserFrameId: number } | null {
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
const documentId = sender.documentId;
|
||||||
|
const browserFrameId = sender.frameId;
|
||||||
|
if (
|
||||||
|
typeof tabId !== "number" ||
|
||||||
|
typeof documentId !== "string" ||
|
||||||
|
typeof browserFrameId !== "number"
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { tabId, documentId, browserFrameId };
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultDeps(): RecordFrameCoordinatorDeps {
|
||||||
|
return {
|
||||||
|
async getAllFrames(tabId) {
|
||||||
|
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
|
||||||
|
},
|
||||||
|
sendToDocument(tabId, message, target) {
|
||||||
|
return chrome.tabs.sendMessage(tabId, message, target);
|
||||||
|
},
|
||||||
|
subscribeFrameNavigation(listener) {
|
||||||
|
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
|
||||||
|
listener({ ...details, lifecycle: "committed" });
|
||||||
|
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
|
||||||
|
listener({ ...details, lifecycle: "completed" });
|
||||||
|
chrome.webNavigation.onCommitted.addListener(onCommitted);
|
||||||
|
chrome.webNavigation.onCompleted.addListener(onCompleted);
|
||||||
|
return () => {
|
||||||
|
chrome.webNavigation.onCommitted.removeListener(onCommitted);
|
||||||
|
chrome.webNavigation.onCompleted.removeListener(onCompleted);
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordFrameCoordinator {
|
||||||
|
readonly #deps: RecordFrameCoordinatorDeps;
|
||||||
|
readonly #recordings = new Map<string, ArmedRecording>();
|
||||||
|
#attached = false;
|
||||||
|
#detachFrameNavigation: (() => void) | null = null;
|
||||||
|
|
||||||
|
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
|
||||||
|
this.#deps = deps;
|
||||||
|
}
|
||||||
|
|
||||||
|
attach(): () => void {
|
||||||
|
if (this.#attached) return () => {};
|
||||||
|
this.#attached = true;
|
||||||
|
chrome.runtime.onConnect.addListener(this.#onConnect);
|
||||||
|
chrome.runtime.onMessage.addListener(this.#onMessage);
|
||||||
|
this.#detachFrameNavigation =
|
||||||
|
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
|
||||||
|
return () => {
|
||||||
|
if (!this.#attached) return;
|
||||||
|
this.#attached = false;
|
||||||
|
chrome.runtime.onConnect.removeListener(this.#onConnect);
|
||||||
|
chrome.runtime.onMessage.removeListener(this.#onMessage);
|
||||||
|
this.#detachFrameNavigation?.();
|
||||||
|
this.#detachFrameNavigation = null;
|
||||||
|
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
|
||||||
|
this.#recordings.clear();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
begin(
|
||||||
|
requestId: string,
|
||||||
|
startedAtMs: number,
|
||||||
|
initialTabId: number,
|
||||||
|
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
|
||||||
|
): void {
|
||||||
|
const previous = this.#recordings.get(requestId);
|
||||||
|
if (previous) this.#cancelAgents(previous);
|
||||||
|
this.#recordings.set(requestId, {
|
||||||
|
requestId,
|
||||||
|
startedAtMs,
|
||||||
|
// Register the initial tab before navigation begins. A newly-created
|
||||||
|
// child document can query at document_start before armTab's frame
|
||||||
|
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
|
||||||
|
tabIds: new Set([initialTabId]),
|
||||||
|
agents: new Map(),
|
||||||
|
...(onDocumentReady ? { onDocumentReady } : {}),
|
||||||
|
finishing: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async armTab(requestId: string, tabId: number): Promise<boolean> {
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!recording || recording.finishing) return false;
|
||||||
|
recording.tabIds.add(tabId);
|
||||||
|
|
||||||
|
let frames: BrowserFrame[];
|
||||||
|
try {
|
||||||
|
frames = await this.#deps.getAllFrames(tabId);
|
||||||
|
} catch {
|
||||||
|
frames = [{ frameId: 0 }];
|
||||||
|
}
|
||||||
|
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
|
||||||
|
|
||||||
|
const results = await Promise.all(
|
||||||
|
frames.map(async (frame) => {
|
||||||
|
const started = await this.#startFrame(recording, tabId, frame);
|
||||||
|
return { frameId: frame.frameId, started };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return results.some((result) => result.frameId === 0 && result.started);
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceFor(
|
||||||
|
requestId: string,
|
||||||
|
producerId: string,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
): RecordingCaptureScope | null {
|
||||||
|
const address = senderAddress(sender);
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!address || !recording) return null;
|
||||||
|
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
|
||||||
|
if (!agent || agent.producerId !== producerId) return null;
|
||||||
|
return {
|
||||||
|
tabId: agent.tabId,
|
||||||
|
documentId: agent.documentId,
|
||||||
|
browserFrameId: agent.browserFrameId,
|
||||||
|
producerId: agent.producerId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async stop(requestId: string): Promise<boolean> {
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!recording) return true;
|
||||||
|
recording.finishing = true;
|
||||||
|
const commandId = createRandomUuid();
|
||||||
|
const results = await Promise.all(
|
||||||
|
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||||
|
);
|
||||||
|
const succeeded = results.every(Boolean);
|
||||||
|
if (succeeded) this.#recordings.delete(requestId);
|
||||||
|
else recording.finishing = false;
|
||||||
|
return succeeded;
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(requestId: string): void {
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!recording) return;
|
||||||
|
this.#cancelAgents(recording);
|
||||||
|
this.#recordings.delete(requestId);
|
||||||
|
}
|
||||||
|
|
||||||
|
readonly #onMessage = (
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: RecordFrameQueryResponse) => void,
|
||||||
|
): boolean => {
|
||||||
|
if (!isRecordFrameQueryMessage(message)) return false;
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
const recording =
|
||||||
|
typeof tabId === "number"
|
||||||
|
? [...this.#recordings.values()].find(
|
||||||
|
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
|
||||||
|
)
|
||||||
|
: undefined;
|
||||||
|
sendResponse(
|
||||||
|
recording
|
||||||
|
? {
|
||||||
|
active: true,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
}
|
||||||
|
: { active: false },
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
|
||||||
|
if (frame.frameId === 0) return;
|
||||||
|
for (const recording of this.#recordings.values()) {
|
||||||
|
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
|
||||||
|
// onCommitted can race document_start, while onCompleted runs after the
|
||||||
|
// content script is available. Starting twice is safe and closes both
|
||||||
|
// the early-query and late-SPA-frame gaps.
|
||||||
|
void (async () => {
|
||||||
|
const started = await this.#startFrame(recording, frame.tabId, frame);
|
||||||
|
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
|
||||||
|
const agent = frame.documentId
|
||||||
|
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
|
||||||
|
: [...recording.agents.values()].find(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
|
||||||
|
);
|
||||||
|
if (!agent) return;
|
||||||
|
await recording.onDocumentReady?.({
|
||||||
|
tabId: agent.tabId,
|
||||||
|
documentId: agent.documentId,
|
||||||
|
browserFrameId: agent.browserFrameId,
|
||||||
|
producerId: agent.producerId,
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
readonly #onConnect = (port: chrome.runtime.Port): void => {
|
||||||
|
if (port.name !== RECORD_FRAME_PORT) return;
|
||||||
|
const address = senderAddress(port.sender ?? {});
|
||||||
|
if (!address) {
|
||||||
|
port.disconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let registered: FrameAgent | null = null;
|
||||||
|
port.onMessage.addListener((raw: unknown) => {
|
||||||
|
const message = raw as Partial<RecordFramePortMessage>;
|
||||||
|
if (message.type === "ready") {
|
||||||
|
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
|
||||||
|
port.disconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const requestId = message.requestId;
|
||||||
|
const producerId = message.producerId;
|
||||||
|
if (registered) {
|
||||||
|
if (registered.requestId === requestId && registered.producerId === producerId) {
|
||||||
|
port.postMessage({
|
||||||
|
type: "ready_ack",
|
||||||
|
requestId,
|
||||||
|
producerId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} else {
|
||||||
|
port.disconnect();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (
|
||||||
|
!recording ||
|
||||||
|
recording.finishing ||
|
||||||
|
!recording.tabIds.has(address.tabId) ||
|
||||||
|
producerId.length === 0
|
||||||
|
) {
|
||||||
|
port.disconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const agent: FrameAgent = {
|
||||||
|
...address,
|
||||||
|
requestId,
|
||||||
|
producerId,
|
||||||
|
port,
|
||||||
|
stopWaiters: new Map(),
|
||||||
|
};
|
||||||
|
registered = agent;
|
||||||
|
const key = documentKey(address.tabId, address.documentId);
|
||||||
|
const previous = recording.agents.get(key);
|
||||||
|
if (previous && previous !== agent) previous.port.disconnect();
|
||||||
|
recording.agents.set(key, agent);
|
||||||
|
port.postMessage({
|
||||||
|
type: "ready_ack",
|
||||||
|
requestId,
|
||||||
|
producerId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
message.type === "stopped" &&
|
||||||
|
registered &&
|
||||||
|
message.requestId === registered.requestId &&
|
||||||
|
typeof message.commandId === "string"
|
||||||
|
) {
|
||||||
|
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
|
||||||
|
registered.stopWaiters.delete(message.commandId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
port.onDisconnect.addListener(() => {
|
||||||
|
if (!registered) return;
|
||||||
|
for (const resolve of registered.stopWaiters.values()) resolve(true);
|
||||||
|
const recording = this.#recordings.get(registered.requestId);
|
||||||
|
const key = documentKey(registered.tabId, registered.documentId);
|
||||||
|
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
agent.stopWaiters.delete(commandId);
|
||||||
|
resolve(false);
|
||||||
|
}, RECORD_FRAME_STOP_TIMEOUT_MS);
|
||||||
|
agent.stopWaiters.set(commandId, (ok) => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(ok);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
agent.port.postMessage({
|
||||||
|
type: "stop",
|
||||||
|
requestId,
|
||||||
|
commandId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} catch {
|
||||||
|
clearTimeout(timer);
|
||||||
|
agent.stopWaiters.delete(commandId);
|
||||||
|
resolve(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #startFrame(
|
||||||
|
recording: ArmedRecording,
|
||||||
|
tabId: number,
|
||||||
|
frame: BrowserFrame,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const message: RecordFrameStartMessage = {
|
||||||
|
type: RECORD_FRAME_START,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const response = await this.#deps.sendToDocument(tabId, message, {
|
||||||
|
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
|
||||||
|
});
|
||||||
|
return isStarted(response);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#cancelAgents(recording: ArmedRecording): void {
|
||||||
|
for (const agent of recording.agents.values()) {
|
||||||
|
try {
|
||||||
|
agent.port.postMessage({
|
||||||
|
type: "cancel",
|
||||||
|
requestId: recording.requestId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} catch {
|
||||||
|
// A destroyed Document has no remaining capture state to cancel.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isStarted(value: unknown): boolean {
|
||||||
|
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const recordFrameCoordinator = new RecordFrameCoordinator();
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
|
||||||
|
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
|
||||||
|
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import {
|
||||||
|
type CaptureVomMatchNode,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
|
||||||
|
export interface IndexedObservationNode {
|
||||||
|
frameId: string;
|
||||||
|
geometry: CaptureVomMatchNode;
|
||||||
|
ref?: RenderedRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CapturedRecordingObservation {
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RegisteredObservation {
|
||||||
|
stateId: string;
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingDocumentScope {
|
||||||
|
frameId: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||||
|
return `${frameId}:${backendNodeId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameTagKey(frameId: string, tag: string): string {
|
||||||
|
return `${frameId}:${tag.toLowerCase()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ObservationNodeIndex {
|
||||||
|
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||||
|
readonly #refById = new Map<string, RenderedRef>();
|
||||||
|
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||||
|
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
|
||||||
|
Partial<Pick<CaptureVomObservationResult, "frames">>,
|
||||||
|
) {
|
||||||
|
const refByNode = new Map<string, RenderedRef>();
|
||||||
|
for (const ref of input.refs) {
|
||||||
|
const frameId = ref.frameId ?? input.rootFrameId;
|
||||||
|
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
||||||
|
this.#refById.set(ref.ref, ref);
|
||||||
|
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
frameRefs.push(ref);
|
||||||
|
this.#refsByFrame.set(frameId, frameRefs);
|
||||||
|
}
|
||||||
|
for (const frame of input.frames ?? []) {
|
||||||
|
const documentId = frame.recordingDocumentId;
|
||||||
|
if (!documentId) continue;
|
||||||
|
this.#scopeByProducer.set(
|
||||||
|
documentId,
|
||||||
|
this.#scopeByProducer.has(documentId)
|
||||||
|
? null
|
||||||
|
: { frameId: frame.frameId, target: frame.target },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const geometry of input.matchNodes) {
|
||||||
|
const { frameId } = geometry;
|
||||||
|
const entry = {
|
||||||
|
frameId,
|
||||||
|
geometry,
|
||||||
|
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
||||||
|
};
|
||||||
|
const key = frameTagKey(frameId, geometry.tag);
|
||||||
|
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
||||||
|
bucket.push(entry);
|
||||||
|
this.#nodesByFrameTag.set(key, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
||||||
|
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
ref(refId: string): RenderedRef | undefined {
|
||||||
|
return this.#refById.get(refId);
|
||||||
|
}
|
||||||
|
|
||||||
|
refs(frameId: string): readonly RenderedRef[] {
|
||||||
|
return this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
documentScope(producerId: string): RecordingDocumentScope | undefined {
|
||||||
|
return this.#scopeByProducer.get(producerId) ?? undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureRecordingObservation(input: {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<CapturedRecordingObservation> {
|
||||||
|
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
||||||
|
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
||||||
|
maxTokens: input.maxTokens,
|
||||||
|
redactValues: input.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
signal: input.signal,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: new ObservationNodeIndex(captured),
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
vomText: captured.text,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
||||||
|
import { RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
||||||
|
|
||||||
|
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
||||||
|
const MIN_CAPTURE_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
export interface TabObservationCursor {
|
||||||
|
lastSettled: RegisteredObservation | null;
|
||||||
|
lastCaptureAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||||
|
return (
|
||||||
|
draft.op === "click" ||
|
||||||
|
draft.op === "hover" ||
|
||||||
|
draft.op === "fill" ||
|
||||||
|
draft.op === "press" ||
|
||||||
|
draft.op === "select"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingObservationSession {
|
||||||
|
readonly registry: RecordingStateRegistry;
|
||||||
|
readonly cursor: TabObservationCursor;
|
||||||
|
readonly annotations: StepAnnotation[];
|
||||||
|
readonly #maxTokens: number;
|
||||||
|
readonly #redactValues: boolean;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
options: {
|
||||||
|
registry?: RecordingStateRegistry;
|
||||||
|
cursor?: TabObservationCursor;
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
maxTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
this.registry = options.registry ?? new RecordingStateRegistry();
|
||||||
|
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
||||||
|
this.annotations = options.annotations ?? [];
|
||||||
|
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
||||||
|
this.#redactValues = options.redactValues ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async capture(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RegisteredObservation> {
|
||||||
|
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
||||||
|
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
||||||
|
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
|
const captured = await captureRecordingObservation({
|
||||||
|
cdp,
|
||||||
|
tabsApi,
|
||||||
|
tabId,
|
||||||
|
maxTokens: this.#maxTokens,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const state = this.registry.register({
|
||||||
|
url: captured.url,
|
||||||
|
title: captured.title,
|
||||||
|
vomText: captured.vomText,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
});
|
||||||
|
const observation: RegisteredObservation = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: captured.index,
|
||||||
|
url: captured.url,
|
||||||
|
};
|
||||||
|
this.cursor.lastSettled = observation;
|
||||||
|
this.cursor.lastCaptureAt = Date.now();
|
||||||
|
return observation;
|
||||||
|
}
|
||||||
|
|
||||||
|
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
||||||
|
const observation = this.cursor.lastSettled;
|
||||||
|
if (isTargeted(draft)) {
|
||||||
|
draft.matchedTarget = observation
|
||||||
|
? matchObservationTarget({
|
||||||
|
observation,
|
||||||
|
hint: draft.targetHint,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: unmatchedTarget(draft.captureTarget);
|
||||||
|
}
|
||||||
|
if (!observation) return;
|
||||||
|
|
||||||
|
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
||||||
|
if (previousActionPending && unmatched) return;
|
||||||
|
draft.preStateId = observation.stateId;
|
||||||
|
this.registry.markStep(observation.stateId, draftId);
|
||||||
|
|
||||||
|
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
||||||
|
const ref = observation.index.ref(draft.matchedTarget.ref);
|
||||||
|
if (!ref) return;
|
||||||
|
this.annotations.push({
|
||||||
|
draftId,
|
||||||
|
op: draft.op,
|
||||||
|
line: ref.line,
|
||||||
|
stateId: observation.stateId,
|
||||||
|
...(draft.op === "fill" && !this.#redactValues
|
||||||
|
? { detail: JSON.stringify(draft.value) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import type { StopReason, TraceV3 } from "@/transport/types";
|
||||||
|
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||||
|
import type { RegisteredObservation } from "./observation-capture";
|
||||||
|
import { RecordingObservationSession } from "./observation-session";
|
||||||
|
import { inferMissingPostStates, SettleController } from "./settle-controller";
|
||||||
|
import { RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { buildTraceV3 } from "./trace-builder-v3";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
interface TabRecordingContext {
|
||||||
|
session: RecordingObservationSession;
|
||||||
|
settle: SettleController;
|
||||||
|
pendingCapture: PendingCapture | null;
|
||||||
|
frameRefresh: Promise<void>;
|
||||||
|
frameRefreshAbort: AbortController | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingCapture {
|
||||||
|
promise: Promise<RegisteredObservation>;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingObservationRuntime {
|
||||||
|
readonly #cdp: CdpRunner;
|
||||||
|
readonly #tabsApi: ChromeTabsApi;
|
||||||
|
readonly #registry = new RecordingStateRegistry();
|
||||||
|
readonly #annotations: StepAnnotation[] = [];
|
||||||
|
readonly #contexts = new Map<number, TabRecordingContext>();
|
||||||
|
readonly #maxTokens?: number;
|
||||||
|
readonly #redactValues: boolean;
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
maxTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
}) {
|
||||||
|
this.#cdp = input.cdp;
|
||||||
|
this.#tabsApi = input.tabsApi;
|
||||||
|
this.#maxTokens = input.maxTokens;
|
||||||
|
this.#redactValues = input.redactValues ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
#context(tabId: number): TabRecordingContext {
|
||||||
|
const existing = this.#contexts.get(tabId);
|
||||||
|
if (existing) return existing;
|
||||||
|
const session = new RecordingObservationSession({
|
||||||
|
registry: this.#registry,
|
||||||
|
annotations: this.#annotations,
|
||||||
|
maxTokens: this.#maxTokens,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
});
|
||||||
|
const context: TabRecordingContext = {
|
||||||
|
session,
|
||||||
|
settle: new SettleController({
|
||||||
|
session,
|
||||||
|
cdp: this.#cdp,
|
||||||
|
tabsApi: this.#tabsApi,
|
||||||
|
tabId,
|
||||||
|
}),
|
||||||
|
pendingCapture: null,
|
||||||
|
frameRefresh: Promise.resolve(),
|
||||||
|
frameRefreshAbort: null,
|
||||||
|
};
|
||||||
|
this.#contexts.set(tabId, context);
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #capture(tabId: number): Promise<RegisteredObservation> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (context.pendingCapture) return context.pendingCapture.promise;
|
||||||
|
|
||||||
|
const abort = new AbortController();
|
||||||
|
let pending!: PendingCapture;
|
||||||
|
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
|
||||||
|
pending = {
|
||||||
|
abort,
|
||||||
|
promise: promise.finally(() => {
|
||||||
|
if (context.pendingCapture === pending) context.pendingCapture = null;
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
context.pendingCapture = pending;
|
||||||
|
return pending.promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async captureInitial(tabId: number): Promise<void> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (context.session.cursor.lastSettled) return;
|
||||||
|
await this.#capture(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Refresh the safe observation after a late child Document becomes usable. */
|
||||||
|
async refreshDocument(tabId: number, producerId: string): Promise<void> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
context.frameRefreshAbort?.abort();
|
||||||
|
const abort = new AbortController();
|
||||||
|
context.frameRefreshAbort = abort;
|
||||||
|
const previous = context.frameRefresh;
|
||||||
|
const refresh = (async () => {
|
||||||
|
await previous;
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
|
||||||
|
// The first capture discovers the safe producer -> CDP Document scope.
|
||||||
|
await this.#capture(tabId);
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
|
||||||
|
if (scope) {
|
||||||
|
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
|
||||||
|
}
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
|
||||||
|
// Capture again after the child SPA is quiet so the next action can be
|
||||||
|
// matched against refs that actually exist in its pre-action state.
|
||||||
|
await this.#capture(tabId);
|
||||||
|
})();
|
||||||
|
const tracked = refresh.catch(() => {});
|
||||||
|
context.frameRefresh = tracked;
|
||||||
|
try {
|
||||||
|
await refresh;
|
||||||
|
} finally {
|
||||||
|
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async captureTabTransition(
|
||||||
|
fromTabId: number,
|
||||||
|
toTabId: number,
|
||||||
|
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
|
||||||
|
const from = this.#context(fromTabId);
|
||||||
|
await this.#flushContext(from);
|
||||||
|
if (!from.session.cursor.lastSettled) {
|
||||||
|
try {
|
||||||
|
await this.captureInitial(fromTabId);
|
||||||
|
} catch {
|
||||||
|
// A target-tab observation can still preserve the forward transition.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const to = this.#context(toTabId);
|
||||||
|
let target = to.session.cursor.lastSettled;
|
||||||
|
try {
|
||||||
|
target = await this.#capture(toTabId);
|
||||||
|
} catch {
|
||||||
|
// Reuse the last settled target state when a refresh races navigation.
|
||||||
|
}
|
||||||
|
|
||||||
|
const source = from.session.cursor.lastSettled;
|
||||||
|
return {
|
||||||
|
...(source ? { preStateId: source.stateId } : {}),
|
||||||
|
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
bindTabTransition(
|
||||||
|
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
|
||||||
|
draftId: number,
|
||||||
|
): void {
|
||||||
|
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async processDraft(
|
||||||
|
tabId: number,
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
producerId?: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const draft = drafts[draftIndex];
|
||||||
|
if (!draft) return;
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (!context.session.cursor.lastSettled) {
|
||||||
|
try {
|
||||||
|
await this.captureInitial(tabId);
|
||||||
|
} catch {
|
||||||
|
// Post-action settle can still provide a usable state.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const scope = producerId
|
||||||
|
? context.session.cursor.lastSettled?.index.documentScope(producerId)
|
||||||
|
: undefined;
|
||||||
|
if (
|
||||||
|
producerId &&
|
||||||
|
(draft.op === "click" ||
|
||||||
|
draft.op === "hover" ||
|
||||||
|
draft.op === "fill" ||
|
||||||
|
draft.op === "press" ||
|
||||||
|
draft.op === "select")
|
||||||
|
) {
|
||||||
|
draft.targetHint = {
|
||||||
|
...(draft.targetHint ?? {}),
|
||||||
|
frameId: scope?.frameId ?? null,
|
||||||
|
geometrySpace: "local",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
|
||||||
|
context.settle.schedule(drafts, draftIndex, scope);
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleSettle(
|
||||||
|
tabId: number,
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
scope?: DocumentSettleScope,
|
||||||
|
): void {
|
||||||
|
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
|
||||||
|
}
|
||||||
|
|
||||||
|
clearRedirect(tabId: number): void {
|
||||||
|
this.#contexts.get(tabId)?.settle.clearRedirect();
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
|
||||||
|
this.#context(tabId).settle.scheduleRedirect(drafts, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
async #flushContext(context: TabRecordingContext): Promise<void> {
|
||||||
|
await context.frameRefresh;
|
||||||
|
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||||
|
await context.settle.flushRedirects();
|
||||||
|
await context.settle.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flushRedirects(tabId: number): Promise<void> {
|
||||||
|
await this.#contexts.get(tabId)?.settle.flushRedirects();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
for (const context of this.#contexts.values()) await this.#flushContext(context);
|
||||||
|
}
|
||||||
|
|
||||||
|
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||||
|
if (!context.session.cursor.lastSettled) {
|
||||||
|
try {
|
||||||
|
await this.captureInitial(tabId);
|
||||||
|
} catch {
|
||||||
|
// The trace may remain observation-free when CDP is unavailable.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await context.settle.settleTrailing(drafts);
|
||||||
|
inferMissingPostStates(drafts);
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
for (const context of this.#contexts.values()) {
|
||||||
|
context.frameRefreshAbort?.abort();
|
||||||
|
context.pendingCapture?.abort.abort();
|
||||||
|
context.settle.cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
buildTrace(input: {
|
||||||
|
drafts: RecordingDraftStep[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
includeTabSwitches?: boolean;
|
||||||
|
}): TraceV3 {
|
||||||
|
return buildTraceV3({
|
||||||
|
registry: this.#registry,
|
||||||
|
drafts: input.drafts,
|
||||||
|
annotations: this.#annotations,
|
||||||
|
startedAt: input.startedAt,
|
||||||
|
purpose: input.purpose,
|
||||||
|
startUrl: input.startUrl,
|
||||||
|
stoppedBy: input.stoppedBy,
|
||||||
|
bskVersion: input.bskVersion,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
includeTabSwitches: input.includeTabSwitches,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||||
|
import { RecordingObservationSession } from "./observation-session";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface PendingSettle {
|
||||||
|
abort: AbortController;
|
||||||
|
scope: DocumentSettleScope;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingRedirect {
|
||||||
|
url: string;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
(error as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
||||||
|
for (let index = 0; index < drafts.length - 1; index += 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
const next = drafts[index + 1];
|
||||||
|
if (draft && next && !draft.postStateId && next.preStateId) {
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SettleController {
|
||||||
|
readonly #session: RecordingObservationSession;
|
||||||
|
readonly #cdp: CdpRunner;
|
||||||
|
readonly #tabsApi: ChromeTabsApi;
|
||||||
|
readonly #tabId: number;
|
||||||
|
readonly #rootScope: DocumentSettleScope;
|
||||||
|
readonly #pending = new Map<number, PendingSettle>();
|
||||||
|
#queue = Promise.resolve();
|
||||||
|
#pendingRedirect: PendingRedirect | null = null;
|
||||||
|
#redirectQueue = Promise.resolve();
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
session: RecordingObservationSession;
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
}) {
|
||||||
|
this.#session = input.session;
|
||||||
|
this.#cdp = input.cdp;
|
||||||
|
this.#tabsApi = input.tabsApi;
|
||||||
|
this.#tabId = input.tabId;
|
||||||
|
this.#rootScope = { target: { tabId: input.tabId } };
|
||||||
|
}
|
||||||
|
|
||||||
|
get hasPending(): boolean {
|
||||||
|
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #captureWithRetry(signal?: AbortSignal) {
|
||||||
|
try {
|
||||||
|
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
||||||
|
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
schedule(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
scope: DocumentSettleScope = this.#rootScope,
|
||||||
|
): void {
|
||||||
|
if (scope.target.tabId !== this.#tabId) {
|
||||||
|
throw new Error(
|
||||||
|
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const landing = drafts[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of this.#pending) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.abort.abort();
|
||||||
|
this.#pending.delete(index);
|
||||||
|
if (landing && drafts[index] && !drafts[index].postStateId) {
|
||||||
|
drafts[index].postStateId = landing;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.#pending.get(draftIndex)?.abort.abort();
|
||||||
|
const pending = { abort: new AbortController(), scope };
|
||||||
|
this.#pending.set(draftIndex, pending);
|
||||||
|
const task = async () => {
|
||||||
|
if (pending.abort.signal.aborted) return;
|
||||||
|
try {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
const observation = await this.#captureWithRetry(pending.abort.signal);
|
||||||
|
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
||||||
|
drafts[draftIndex].postStateId = observation.stateId;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (!isAbortError(error)) {
|
||||||
|
console.warn(
|
||||||
|
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.#queue = this.#queue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
for (const pending of this.#pending.values()) pending.abort.abort();
|
||||||
|
this.#pending.clear();
|
||||||
|
this.clearRedirect();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#queue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#queue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
||||||
|
}
|
||||||
|
|
||||||
|
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
||||||
|
const trailing: RecordingDraftStep[] = [];
|
||||||
|
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
try {
|
||||||
|
const observation = await this.#captureWithRetry();
|
||||||
|
for (const draft of trailing) draft.postStateId = observation.stateId;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
clearRedirect(): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
const pending: PendingRedirect = { url, abort: new AbortController() };
|
||||||
|
this.#pendingRedirect = pending;
|
||||||
|
const task = () => this.#settleRedirect(drafts, pending);
|
||||||
|
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
||||||
|
|
||||||
|
let finalUrl = pending.url;
|
||||||
|
try {
|
||||||
|
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
||||||
|
} catch {
|
||||||
|
// The navigation event URL remains the best available destination.
|
||||||
|
}
|
||||||
|
if (this.#pendingRedirect !== pending) return;
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
if (
|
||||||
|
!finalUrl ||
|
||||||
|
finalUrl === "about:blank" ||
|
||||||
|
this.#session.cursor.lastSettled?.url === finalUrl
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const last = drafts[drafts.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
||||||
|
await this.flush();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
pageUrl: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
transitionQualifiers: ["server_redirect"],
|
||||||
|
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
drafts.push(draft);
|
||||||
|
const draftIndex = drafts.length - 1;
|
||||||
|
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
||||||
|
this.schedule(drafts, draftIndex);
|
||||||
|
await this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flushRedirects(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#redirectQueue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#redirectQueue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
export interface RecordedStateEntry {
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
|
||||||
|
vomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
stepsHere: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function stateIdentity(url: string, body: string): string {
|
||||||
|
return `${url}\0${body}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingStateRegistry {
|
||||||
|
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
||||||
|
readonly #idByIdentity = new Map<string, string>();
|
||||||
|
#nextId = 1;
|
||||||
|
|
||||||
|
register(input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}): RecordedStateEntry {
|
||||||
|
const identity = stateIdentity(input.url, input.vomText);
|
||||||
|
const existingId = this.#idByIdentity.get(identity);
|
||||||
|
if (existingId) {
|
||||||
|
const existing = this.#entriesById.get(existingId)!;
|
||||||
|
if (!existing.title && input.title) existing.title = input.title;
|
||||||
|
if (input.truncated) existing.truncated = true;
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry: RecordedStateEntry = {
|
||||||
|
id: `s${this.#nextId}`,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
vomText: input.vomText,
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
stepsHere: [],
|
||||||
|
};
|
||||||
|
this.#nextId += 1;
|
||||||
|
this.#entriesById.set(entry.id, entry);
|
||||||
|
this.#idByIdentity.set(identity, entry.id);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
values(): RecordedStateEntry[] {
|
||||||
|
return [...this.#entriesById.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
markStep(stateId: string, draftId: number): void {
|
||||||
|
const entry = this.#entriesById.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
import type { RecordStepPayload } from "../record-bridge";
|
||||||
|
import { hasRedirectQualifier } from "./navigation-policy";
|
||||||
|
import type { RecordingDraftStep, TargetMatchHint } from "./types";
|
||||||
|
|
||||||
|
export interface RecordingStepBuffer {
|
||||||
|
steps: RecordingDraftStep[];
|
||||||
|
navigation: RecordingNavigationCursor;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingNavigationCursor {
|
||||||
|
currentUrl?: string;
|
||||||
|
pendingNavigation: boolean;
|
||||||
|
pendingNavigationDeadline?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
|
function toDraftStep(
|
||||||
|
payload: RecordStepPayload,
|
||||||
|
targetHint?: TargetMatchHint,
|
||||||
|
): RecordingDraftStep | null {
|
||||||
|
const pageUrl = payload.page_url;
|
||||||
|
const common = {
|
||||||
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
|
...(targetHint ? { targetHint } : {}),
|
||||||
|
};
|
||||||
|
switch (payload.op) {
|
||||||
|
case "click":
|
||||||
|
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
|
||||||
|
case "hover":
|
||||||
|
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
|
||||||
|
case "fill":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: payload.target,
|
||||||
|
value: payload.value ?? "",
|
||||||
|
...(payload.commit ? { commit: payload.commit } : {}),
|
||||||
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
|
...common,
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "press":
|
||||||
|
return payload.key
|
||||||
|
? {
|
||||||
|
op: "press",
|
||||||
|
key: payload.key,
|
||||||
|
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||||
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
|
...common,
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "select":
|
||||||
|
return payload.target && payload.values
|
||||||
|
? {
|
||||||
|
op: "select",
|
||||||
|
captureTarget: payload.target,
|
||||||
|
values: payload.values,
|
||||||
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
|
...common,
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "navigate":
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||||
|
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
|
||||||
|
const step = buffer.steps[index];
|
||||||
|
if (!step) continue;
|
||||||
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
|
step.navigatedTo = url;
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NavigationObserveResult =
|
||||||
|
| { kind: "noop" }
|
||||||
|
| { kind: "annotated"; index: number }
|
||||||
|
| { kind: "appended"; index: number }
|
||||||
|
| { kind: "coalesce_redirect"; url: string };
|
||||||
|
|
||||||
|
export function observeRecordedNavigation(
|
||||||
|
buffer: RecordingStepBuffer,
|
||||||
|
url: string,
|
||||||
|
causedByAction?: boolean,
|
||||||
|
transitionType?: string,
|
||||||
|
transitionQualifiers?: string[],
|
||||||
|
): NavigationObserveResult {
|
||||||
|
const navigation = buffer.navigation;
|
||||||
|
if (!url || url === navigation.currentUrl) return { kind: "noop" };
|
||||||
|
navigation.currentUrl = url;
|
||||||
|
|
||||||
|
const pendingIsCurrent =
|
||||||
|
navigation.pendingNavigation &&
|
||||||
|
(navigation.pendingNavigationDeadline === undefined ||
|
||||||
|
navigation.pendingNavigationDeadline >= Date.now());
|
||||||
|
|
||||||
|
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||||
|
navigation.pendingNavigation = false;
|
||||||
|
navigation.pendingNavigationDeadline = undefined;
|
||||||
|
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||||
|
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
|
||||||
|
} else {
|
||||||
|
navigation.pendingNavigation = false;
|
||||||
|
navigation.pendingNavigationDeadline = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasRedirectQualifier(transitionQualifiers)) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
|
buffer.steps.push({
|
||||||
|
op: "navigate",
|
||||||
|
url,
|
||||||
|
pageUrl: url,
|
||||||
|
transitionType,
|
||||||
|
transitionQualifiers,
|
||||||
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function appendRecordedPayload(
|
||||||
|
buffer: RecordingStepBuffer,
|
||||||
|
payload: RecordStepPayload,
|
||||||
|
targetHint?: TargetMatchHint,
|
||||||
|
): number | null {
|
||||||
|
if (payload.op === "navigate") {
|
||||||
|
if (!payload.url) return null;
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
payload.url,
|
||||||
|
payload.navigation_caused_by_action,
|
||||||
|
payload.transitionType,
|
||||||
|
payload.transitionQualifiers,
|
||||||
|
);
|
||||||
|
return result.kind === "appended" ? result.index : null;
|
||||||
|
}
|
||||||
|
const step = toDraftStep(
|
||||||
|
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
|
||||||
|
targetHint,
|
||||||
|
);
|
||||||
|
if (!step) return null;
|
||||||
|
buffer.steps.push(step);
|
||||||
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
|
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
|
||||||
|
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
|
||||||
|
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||||
|
: undefined;
|
||||||
|
}
|
||||||
|
return buffer.steps.length - 1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import type { RecordingNavigationCursor } from "./step-buffer";
|
||||||
|
|
||||||
|
export interface TabActivation {
|
||||||
|
tabId: number;
|
||||||
|
revision: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingTabCoordinator {
|
||||||
|
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
|
||||||
|
#activeTabId: number;
|
||||||
|
#currentTabId: number;
|
||||||
|
#activationRevision = 0;
|
||||||
|
|
||||||
|
constructor(initialTabId: number, initialUrl?: string) {
|
||||||
|
this.#activeTabId = initialTabId;
|
||||||
|
this.#currentTabId = initialTabId;
|
||||||
|
this.#navigationByTab.set(initialTabId, {
|
||||||
|
currentUrl: initialUrl,
|
||||||
|
pendingNavigation: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
get activeTabId(): number {
|
||||||
|
return this.#activeTabId;
|
||||||
|
}
|
||||||
|
|
||||||
|
get currentTabId(): number {
|
||||||
|
return this.#currentTabId;
|
||||||
|
}
|
||||||
|
|
||||||
|
noteActivation(tabId: number): TabActivation {
|
||||||
|
this.#activeTabId = tabId;
|
||||||
|
this.#activationRevision += 1;
|
||||||
|
return { tabId, revision: this.#activationRevision };
|
||||||
|
}
|
||||||
|
|
||||||
|
isLatest(activation: TabActivation): boolean {
|
||||||
|
return (
|
||||||
|
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
commit(tabId: number, currentUrl?: string): void {
|
||||||
|
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
|
||||||
|
this.#currentTabId = tabId;
|
||||||
|
}
|
||||||
|
|
||||||
|
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
|
||||||
|
const existing = this.#navigationByTab.get(tabId);
|
||||||
|
if (existing) {
|
||||||
|
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
const created: RecordingNavigationCursor = {
|
||||||
|
currentUrl: fallbackUrl,
|
||||||
|
pendingNavigation: false,
|
||||||
|
};
|
||||||
|
this.#navigationByTab.set(tabId, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { TargetDescriptorV3 } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "../describe-target";
|
||||||
|
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
||||||
|
import type { TargetGeometry, TargetMatchHint } from "./types";
|
||||||
|
|
||||||
|
const MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
function close(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
||||||
|
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidateMatches(
|
||||||
|
candidate: IndexedObservationNode,
|
||||||
|
geometry: TargetGeometry,
|
||||||
|
geometrySpace: "top" | "local",
|
||||||
|
): boolean {
|
||||||
|
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
|
||||||
|
return rect != null && rectMatches(geometry.rect, rect);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
...(fallback?.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback?.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalized(value: string | undefined): string {
|
||||||
|
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
||||||
|
const role = normalized(fallback?.role);
|
||||||
|
const name = normalized(fallback?.name);
|
||||||
|
if (!role && !name) return false;
|
||||||
|
if (role && normalized(ref.role) !== role) return false;
|
||||||
|
if (name && normalized(ref.name) !== name) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
ref: ref.ref,
|
||||||
|
...(ref.role ? { role: ref.role } : {}),
|
||||||
|
...(ref.name ? { name: ref.name } : {}),
|
||||||
|
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function matchObservationTarget(input: {
|
||||||
|
observation: RegisteredObservation;
|
||||||
|
hint?: TargetMatchHint;
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}): TargetDescriptorV3 {
|
||||||
|
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
|
||||||
|
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||||
|
const geometry = input.hint?.geometry;
|
||||||
|
if (geometry) {
|
||||||
|
const matches = input.observation.index
|
||||||
|
.candidates(frameId, geometry.tag)
|
||||||
|
.filter(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.ref &&
|
||||||
|
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
|
||||||
|
);
|
||||||
|
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||||
|
const semanticMatches = matches.filter(
|
||||||
|
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||||
|
);
|
||||||
|
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
||||||
|
return unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const semanticMatches = input.observation.index
|
||||||
|
.refs(frameId)
|
||||||
|
.filter((ref) => matchesSemantics(ref, input.fallback));
|
||||||
|
return semanticMatches.length === 1
|
||||||
|
? descriptor(semanticMatches[0]!)
|
||||||
|
: unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import {
|
||||||
|
type StepV3,
|
||||||
|
type StopReason,
|
||||||
|
TRACE_VERSION_V3,
|
||||||
|
type TraceStateV3,
|
||||||
|
type TraceV3,
|
||||||
|
VOM_FORMAT_VERSION,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl } from "./draft-policy";
|
||||||
|
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
||||||
|
import { formatTraceStateBody } from "./trace-state-body";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
||||||
|
const entries = registry.values();
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
return [
|
||||||
|
...new Set(
|
||||||
|
draftIds.flatMap((id) => {
|
||||||
|
const stepId = stepIdByDraftId.get(id);
|
||||||
|
return stepId === undefined ? [] : [stepId];
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
registry: RecordingStateRegistry;
|
||||||
|
drafts: RecordingDraftStep[];
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
redactValues?: boolean;
|
||||||
|
includeTabSwitches?: boolean;
|
||||||
|
}): TraceV3 {
|
||||||
|
const reduced = reduceTraceStepsV3(input.drafts, {
|
||||||
|
includeTabSwitches: input.includeTabSwitches,
|
||||||
|
redactValues: input.redactValues,
|
||||||
|
});
|
||||||
|
const entries = publishedEntries(input.registry, reduced.steps);
|
||||||
|
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||||
|
for (const annotation of input.annotations ?? []) {
|
||||||
|
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
||||||
|
bucket.push(annotation);
|
||||||
|
annotationsByState.set(annotation.stateId, bucket);
|
||||||
|
}
|
||||||
|
const steps = reduced.steps.map((step) => ({
|
||||||
|
...step,
|
||||||
|
state: publishedId.get(step.state) ?? step.state,
|
||||||
|
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
||||||
|
}));
|
||||||
|
const states: TraceStateV3[] = entries.map((entry) => {
|
||||||
|
const id = publishedId.get(entry.id) ?? entry.id;
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body: formatTraceStateBody({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||||
|
vomText: entry.vomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||||
|
}),
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
version: TRACE_VERSION_V3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
+62
-71
@@ -1,30 +1,10 @@
|
|||||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
import type { RecordingDraftStep } from "./types";
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
|
||||||
|
|
||||||
export function shouldRecordPress(
|
|
||||||
key: string,
|
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
|
||||||
): boolean {
|
|
||||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
|
||||||
const mods = modifiers ?? [];
|
|
||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
|
||||||
// Drop bare character typing — FillSession already records the value.
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
|
||||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
/** Collapse consecutive navigations to the last hop. */
|
||||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
||||||
const out: DraftTraceStep[] = [];
|
const out: RecordingDraftStep[] = [];
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
const prev = out[out.length - 1];
|
const prev = out[out.length - 1];
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||||
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
||||||
const urls: string[] = [];
|
const urls: string[] = [];
|
||||||
if (startUrl) urls.push(startUrl);
|
if (startUrl) urls.push(startUrl);
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
urls.push(step.url);
|
urls.push(step.url);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
if (step.pageUrl) urls.push(step.pageUrl);
|
||||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
||||||
}
|
}
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const unique: string[] = [];
|
const unique: string[] = [];
|
||||||
@@ -58,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildPageRegistry(
|
function buildPageRegistry(
|
||||||
steps: DraftTraceStep[],
|
steps: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
||||||
const urls = collectUrls(steps, startUrl);
|
const urls = collectUrls(steps, startUrl);
|
||||||
const urlToId = new Map<string, string>();
|
const urlToId = new Map<string, string>();
|
||||||
const pages = urls.map((url, index) => {
|
const pages = urls.map((url, index) => {
|
||||||
@@ -81,28 +61,28 @@ function pageIdFor(
|
|||||||
return urlToId.values().next().value ?? "p1";
|
return urlToId.values().next().value ?? "p1";
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
||||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
||||||
if ("page_url" in step && step.page_url) return step.page_url;
|
if (step.pageUrl) return step.pageUrl;
|
||||||
return fallbackUrl;
|
return fallbackUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
function effectForNavigation(
|
function effectForNavigation(
|
||||||
navigatedTo: string | undefined,
|
navigatedTo: string | undefined,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
): Step["effect"] {
|
): StepV2["effect"] {
|
||||||
if (!navigatedTo) return undefined;
|
if (!navigatedTo) return undefined;
|
||||||
const pageId = urlToId.get(navigatedTo);
|
const pageId = urlToId.get(navigatedTo);
|
||||||
if (!pageId) return undefined;
|
if (!pageId) return undefined;
|
||||||
return { navigated_to: pageId };
|
return { navigated_to: pageId };
|
||||||
}
|
}
|
||||||
|
|
||||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
|
||||||
if (!effect) return step;
|
if (!effect) return step;
|
||||||
return { ...step, effect };
|
return { ...step, effect };
|
||||||
}
|
}
|
||||||
|
|
||||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||||
return values.map((value, index) => ({
|
return values.map((value, index) => ({
|
||||||
value,
|
value,
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
@@ -110,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function toV2Step(
|
function toV2Step(
|
||||||
step: DraftTraceStep,
|
step: RecordingDraftStep,
|
||||||
id: number,
|
id: number,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
fallbackUrl?: string,
|
fallbackUrl?: string,
|
||||||
): Step | null {
|
): StepV2 | null {
|
||||||
if (!shouldIncludeDraft(step)) return null;
|
if (!shouldIncludeDraft(step)) return null;
|
||||||
|
|
||||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||||
@@ -129,28 +109,23 @@ function toV2Step(
|
|||||||
to: step.url,
|
to: step.url,
|
||||||
};
|
};
|
||||||
case "click":
|
case "click":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "hover":
|
|
||||||
return {
|
|
||||||
op: "hover",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.target,
|
|
||||||
};
|
|
||||||
case "fill":
|
case "fill":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
value: step.value,
|
value: step.value,
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
...(step.redacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
@@ -161,44 +136,51 @@ function toV2Step(
|
|||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
key: step.key,
|
key: step.key,
|
||||||
...(step.target ? { target: step.target } : {}),
|
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "select":
|
case "select":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
selection: toSelection(step.values, step.labels),
|
selection: toSelection(step.values, step.labels),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
|
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
|
||||||
|
// step variant and fail to decode the whole result if we emit it.
|
||||||
|
case "hover":
|
||||||
|
case "scroll":
|
||||||
|
case "switch_tab":
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReducedTrace {
|
interface ReducedTrace {
|
||||||
pages: PageRef[];
|
pages: PageRefV2[];
|
||||||
steps: Step[];
|
steps: StepV2[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compile capture drafts into record-only trace v2 steps.
|
* Compile capture drafts into record-only trace v2 steps.
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||||
*/
|
*/
|
||||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
||||||
const collapsed = collapseNavigations(steps);
|
const collapsed = collapseNavigations(steps);
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||||
const out: Step[] = [];
|
const out: StepV2[] = [];
|
||||||
let id = 1;
|
let id = 1;
|
||||||
let lastUrl = startUrl;
|
let lastUrl = startUrl;
|
||||||
for (const draft of collapsed) {
|
for (const draft of collapsed) {
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
if (draft.op === "navigate") lastUrl = draft.url;
|
||||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
||||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
out.push(step);
|
out.push(step);
|
||||||
@@ -207,18 +189,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
|||||||
return { pages, steps: out };
|
return { pages, steps: out };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTraceStartUrl(
|
function resolveTraceStartUrl(
|
||||||
drafts: DraftTraceStep[],
|
drafts: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
pages?: PageRef[],
|
pages?: PageRefV2[],
|
||||||
): string {
|
): string {
|
||||||
if (startUrl) return startUrl;
|
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
||||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
}
|
||||||
return step.op === "navigate";
|
|
||||||
});
|
export function buildTraceV2(input: {
|
||||||
if (navigate) return navigate.url;
|
steps: RecordingDraftStep[];
|
||||||
for (const draft of drafts) {
|
startedAt: string;
|
||||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
startUrl?: string;
|
||||||
}
|
purpose?: string;
|
||||||
return pages?.[0]?.url ?? "about:blank";
|
}): TraceV2 {
|
||||||
|
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
||||||
|
return {
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
||||||
|
pages,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||||
|
import { shouldIncludeDraft } from "./draft-policy";
|
||||||
|
import { hasRedirectQualifier } from "./navigation-policy";
|
||||||
|
import { unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface CollapsedDraft {
|
||||||
|
draft: RecordingDraftStep;
|
||||||
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||||
|
return hasRedirectQualifier(step.transitionQualifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||||
|
const output: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const previous = output[output.length - 1];
|
||||||
|
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
||||||
|
previous.draft = {
|
||||||
|
...previous.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? previous.draft.postStateId,
|
||||||
|
};
|
||||||
|
previous.draftIds.push(index + 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
||||||
|
});
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
||||||
|
if (step.cause) return step.cause;
|
||||||
|
const qualifiers = step.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
||||||
|
return values.map((value, index) => ({
|
||||||
|
value,
|
||||||
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ReduceDraftOptions {
|
||||||
|
includeTabSwitches: boolean;
|
||||||
|
redactValues: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function reduceDraft(
|
||||||
|
draft: RecordingDraftStep,
|
||||||
|
id: number,
|
||||||
|
options: ReduceDraftOptions,
|
||||||
|
): StepV3 | null {
|
||||||
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
if (draft.op === "switch_tab") {
|
||||||
|
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
|
||||||
|
return {
|
||||||
|
op: "switch_tab",
|
||||||
|
id,
|
||||||
|
state: draft.preStateId,
|
||||||
|
result: { state: draft.postStateId },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const state = draft.preStateId ?? draft.postStateId;
|
||||||
|
const resultState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!state || !resultState) return null;
|
||||||
|
const common = { id, state, result: { state: resultState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
|
case "navigate":
|
||||||
|
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
||||||
|
case "click":
|
||||||
|
return {
|
||||||
|
op: "click",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "hover":
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "fill":
|
||||||
|
const fillIsRedacted = options.redactValues || draft.redacted === true;
|
||||||
|
return {
|
||||||
|
op: "fill",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
value: fillIsRedacted ? "***" : draft.value,
|
||||||
|
commit: draft.commit ?? "blur",
|
||||||
|
...(fillIsRedacted ? { redacted: true } : {}),
|
||||||
|
};
|
||||||
|
case "press":
|
||||||
|
return {
|
||||||
|
op: "press",
|
||||||
|
...common,
|
||||||
|
key: draft.key,
|
||||||
|
...(draft.captureTarget || draft.matchedTarget
|
||||||
|
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
||||||
|
: {}),
|
||||||
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
|
};
|
||||||
|
case "select":
|
||||||
|
return {
|
||||||
|
op: "select",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
|
||||||
|
};
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...common };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReducedTraceV3 {
|
||||||
|
steps: StepV3[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reduceTraceStepsV3(
|
||||||
|
steps: RecordingDraftStep[],
|
||||||
|
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
|
||||||
|
): ReducedTraceV3 {
|
||||||
|
const output: StepV3[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
|
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||||
|
const step = reduceDraft(draft, output.length + 1, {
|
||||||
|
includeTabSwitches: options.includeTabSwitches === true,
|
||||||
|
redactValues: options.redactValues === true,
|
||||||
|
});
|
||||||
|
if (!step) continue;
|
||||||
|
output.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||||
|
}
|
||||||
|
return { steps: output, stepIdByDraftId };
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import type { StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
||||||
|
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatTraceStateBody(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepIds: number[];
|
||||||
|
vomText: string;
|
||||||
|
annotations: StepAnnotation[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}): string {
|
||||||
|
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
||||||
|
lines.push(`url: ${JSON.stringify(input.url)}`);
|
||||||
|
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
||||||
|
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
||||||
|
for (const annotation of input.annotations) {
|
||||||
|
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byLine.get(annotation.line) ?? [];
|
||||||
|
bucket.push({ annotation, stepId });
|
||||||
|
byLine.set(annotation.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
||||||
|
let line = bodyLine;
|
||||||
|
for (const item of byLine.get(lineIndex) ?? []) {
|
||||||
|
line += annotationText(item.annotation, item.stepId);
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
});
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
import type {
|
||||||
|
FillCommit,
|
||||||
|
KeyModifier,
|
||||||
|
NavigationCause,
|
||||||
|
StepV3,
|
||||||
|
TargetDescriptorV3,
|
||||||
|
} from "@/transport/types";
|
||||||
|
|
||||||
|
export interface TargetGeometry {
|
||||||
|
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
tag: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TargetMatchHint {
|
||||||
|
geometry?: TargetGeometry;
|
||||||
|
/** Missing means top frame; null means the source Document could not be resolved. */
|
||||||
|
frameId?: string | null;
|
||||||
|
geometrySpace?: "top" | "local";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepAnnotation {
|
||||||
|
draftId: number;
|
||||||
|
op: StepV3["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftStateLink {
|
||||||
|
pageUrl?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftTarget {
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
targetHint?: TargetMatchHint;
|
||||||
|
matchedTarget?: TargetDescriptorV3;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftNavigationEffect {
|
||||||
|
navigatedTo?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordingDraftStep =
|
||||||
|
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
||||||
|
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
||||||
|
| ({
|
||||||
|
op: "fill";
|
||||||
|
value: string;
|
||||||
|
commit?: FillCommit;
|
||||||
|
redacted?: boolean;
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "press";
|
||||||
|
key: string;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "select";
|
||||||
|
values: string[];
|
||||||
|
labels?: string[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({ op: "scroll" } & DraftStateLink)
|
||||||
|
| ({ op: "switch_tab" } & DraftStateLink)
|
||||||
|
| ({
|
||||||
|
op: "navigate";
|
||||||
|
url: string;
|
||||||
|
cause?: NavigationCause;
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
} & DraftStateLink);
|
||||||
|
|
||||||
|
export type TargetedRecordingDraft = Extract<
|
||||||
|
RecordingDraftStep,
|
||||||
|
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
||||||
|
>;
|
||||||
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
|
|||||||
const manager = new SessionManager({
|
const manager = new SessionManager({
|
||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => nextWindowId++),
|
create: vi.fn(async () => nextWindowId++),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
remove,
|
remove,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
|
|||||||
const manager = new SessionManager({
|
const manager = new SessionManager({
|
||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 100),
|
create: vi.fn(async () => 100),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
remove: vi.fn(() => removeGate),
|
remove: vi.fn(() => removeGate),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
await manager.start("aa11");
|
await manager.start("aa11");
|
||||||
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const ctx = await manager.start("aa11");
|
const ctx = await manager.start("aa11");
|
||||||
@@ -87,6 +87,7 @@ describe("attachSessionEventHandler", () => {
|
|||||||
windowEvents: events.api,
|
windowEvents: events.api,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
manager.forgetClosedTab(7, { isWindowClosing: true });
|
||||||
events.emit(4242);
|
events.emit(4242);
|
||||||
for (let i = 0; i < 4; i += 1) await Promise.resolve();
|
for (let i = 0; i < 4; i += 1) await Promise.resolve();
|
||||||
|
|
||||||
@@ -113,7 +114,7 @@ describe("attachSessionEventHandler", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const transport = fakeTransport();
|
const transport = fakeTransport();
|
||||||
@@ -129,7 +130,7 @@ describe("attachSessionEventHandler", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const transport = fakeTransport();
|
const transport = fakeTransport();
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||||
import { SessionManager } from "../manager";
|
import { isAgentControlledTab, SessionManager } from "../manager";
|
||||||
|
|
||||||
function fakeAgentWindow(): AgentWindowApi & {
|
function fakeAgentWindow(): AgentWindowApi & {
|
||||||
createMock: ReturnType<typeof vi.fn>;
|
createMock: ReturnType<typeof vi.fn>;
|
||||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
|||||||
return id;
|
return id;
|
||||||
});
|
});
|
||||||
const removeMock = vi.fn(async (_id: number) => {});
|
const removeMock = vi.fn(async (_id: number) => {});
|
||||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
|
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
|
||||||
return {
|
return {
|
||||||
create: createMock,
|
create: createMock,
|
||||||
remove: removeMock,
|
remove: removeMock,
|
||||||
@@ -151,6 +151,76 @@ describe("SessionManager", () => {
|
|||||||
expect(sm.list()).toEqual([]);
|
expect(sm.list()).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("explicit tab claims", () => {
|
||||||
|
it("claims the exact home tab returned by AgentWindowApi", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
|
||||||
|
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps every other tab free without classification state", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
expect(isAgentControlledTab(ctx, 99)).toBe(false);
|
||||||
|
expect(ctx).not.toHaveProperty("userTabs");
|
||||||
|
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats a committed borrow as an explicit claim", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||||
|
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||||
|
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
|
||||||
|
expect(isAgentControlledTab(ctx, 42)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forgets an agent-created tab after Chrome removes it", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.agentCreatedTabs.add(42);
|
||||||
|
sm.forgetClosedTab(42);
|
||||||
|
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
|
||||||
|
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const a = await sm.start("aa11");
|
||||||
|
const b = await sm.start("bb22");
|
||||||
|
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
|
||||||
|
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
|
||||||
|
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
|
||||||
|
|
||||||
|
sm.forgetClosedTab(42);
|
||||||
|
sm.forgetClosedTab(42);
|
||||||
|
sm.forgetClosedTab(999);
|
||||||
|
|
||||||
|
expect(isAgentControlledTab(a, 42)).toBe(false);
|
||||||
|
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||||
|
expect([...a.borrowedTabs.keys()]).toEqual([43]);
|
||||||
|
expect([...b.borrowedTabs.keys()]).toEqual([44]);
|
||||||
|
expect(sm.list()).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||||
|
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||||
|
|
||||||
|
sm.forgetClosedTab(42);
|
||||||
|
|
||||||
|
expect(() =>
|
||||||
|
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
|
||||||
|
).toThrow(/reservation disappeared/);
|
||||||
|
reservation.release();
|
||||||
|
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||||
|
expect(ctx.borrowedTabs.has(42)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("findBorrowingSession", () => {
|
describe("findBorrowingSession", () => {
|
||||||
it("returns null when no session has borrowed the tab", async () => {
|
it("returns null when no session has borrowed the tab", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
|||||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
|||||||
expect(s.resolve("@e11")).toBe(11);
|
expect(s.resolve("@e11")).toBe(11);
|
||||||
expect(s.size()).toBe(2);
|
expect(s.size()).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves the child CDP session as part of ref identity", () => {
|
||||||
|
const s = new RefStore();
|
||||||
|
s.set("e1", 42, {
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
expect(s.resolveEntry("e1")).toMatchObject({
|
||||||
|
backendNodeId: 42,
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -14,8 +14,11 @@ export interface AgentWindowApi {
|
|||||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||||
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
|
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
|
||||||
* so sessions bootstrap with `about:blank` instead.
|
* so sessions bootstrap with `about:blank` instead.
|
||||||
|
*
|
||||||
|
* Resolves with the id of the activated (or newly created) tab, so callers
|
||||||
|
* can track the session's home tab without re-querying Chrome.
|
||||||
*/
|
*/
|
||||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
ensureActiveTab(windowId: number, url: string): Promise<number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Creation hints for a new Agent Window. */
|
/** Creation hints for a new Agent Window. */
|
||||||
@@ -49,15 +52,19 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
|||||||
// cancellation success while the Agent Window remains open.
|
// cancellation success while the Agent Window remains open.
|
||||||
await chrome.windows.remove(windowId);
|
await chrome.windows.remove(windowId);
|
||||||
},
|
},
|
||||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
async ensureActiveTab(windowId: number, url: string): Promise<number> {
|
||||||
const tabs = await chrome.tabs.query({ windowId });
|
const tabs = await chrome.tabs.query({ windowId });
|
||||||
const first = tabs.find((t) => typeof t.id === "number");
|
const first = tabs.find((t) => typeof t.id === "number");
|
||||||
if (first?.id) {
|
if (first?.id !== undefined) {
|
||||||
if (!first.active) {
|
if (!first.active) {
|
||||||
await chrome.tabs.update(first.id, { active: true });
|
await chrome.tabs.update(first.id, { active: true });
|
||||||
}
|
}
|
||||||
return;
|
return first.id;
|
||||||
}
|
}
|
||||||
await chrome.tabs.create({ windowId, url, active: true });
|
const created = await chrome.tabs.create({ windowId, url, active: true });
|
||||||
|
if (typeof created?.id !== "number") {
|
||||||
|
throw new Error("[bh] chrome.tabs.create returned no tab id");
|
||||||
|
}
|
||||||
|
return created.id;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -6,9 +6,20 @@ export interface SessionContext {
|
|||||||
agentWindowId: number;
|
agentWindowId: number;
|
||||||
refStore: RefStore;
|
refStore: RefStore;
|
||||||
borrowedTabs: Map<number, BorrowedTab>;
|
borrowedTabs: Map<number, BorrowedTab>;
|
||||||
|
/**
|
||||||
|
* Tabs explicitly claimed by the agent because it created them. This
|
||||||
|
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
|
||||||
|
* Tabs opened through Chrome UI never enter this set.
|
||||||
|
*/
|
||||||
|
agentCreatedTabs: Set<number>;
|
||||||
createdAtMs: number;
|
createdAtMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Whether this session has explicitly claimed control of `tabId`. */
|
||||||
|
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
|
||||||
|
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
export interface BorrowedTab {
|
export interface BorrowedTab {
|
||||||
tabId: number;
|
tabId: number;
|
||||||
originalWindowId: number;
|
originalWindowId: number;
|
||||||
@@ -106,6 +117,19 @@ export class SessionManager {
|
|||||||
return Array.from(this.sessions.values());
|
return Array.from(this.sessions.values());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Forget a tab Chrome has removed, including any uncommitted borrow.
|
||||||
|
* Whole-window closures keep committed borrows until the window-removed
|
||||||
|
* handler reports which user tabs could not be returned.
|
||||||
|
*/
|
||||||
|
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
|
||||||
|
this.borrowReservations.delete(tabId);
|
||||||
|
for (const ctx of this.sessions.values()) {
|
||||||
|
ctx.agentCreatedTabs.delete(tabId);
|
||||||
|
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Look up whether `tabId` is currently borrowed by some *other*
|
* Look up whether `tabId` is currently borrowed by some *other*
|
||||||
* session than the one calling. Used by M8 `tab_borrow` to refuse
|
* session than the one calling. Used by M8 `tab_borrow` to refuse
|
||||||
@@ -179,7 +203,7 @@ export class SessionManager {
|
|||||||
const { signal: _signal, ...createOptions } = opts;
|
const { signal: _signal, ...createOptions } = opts;
|
||||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||||
throwIfSessionStartAborted(opts.signal);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||||
throwIfSessionStartAborted(opts.signal);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
|
|
||||||
const ctx: SessionContext = {
|
const ctx: SessionContext = {
|
||||||
@@ -187,6 +211,10 @@ export class SessionManager {
|
|||||||
agentWindowId: windowId,
|
agentWindowId: windowId,
|
||||||
refStore: new RefStore(),
|
refStore: new RefStore(),
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
|
// The home tab is the session's first explicit claim. Every other
|
||||||
|
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||||
|
// by its concrete Chrome tab id.
|
||||||
|
agentCreatedTabs: new Set([homeTabId]),
|
||||||
createdAtMs: this.now(),
|
createdAtMs: this.now(),
|
||||||
};
|
};
|
||||||
this.sessions.set(sessionId, ctx);
|
this.sessions.set(sessionId, ctx);
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
/**
|
/**
|
||||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||||
*
|
*
|
||||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
* `replace(...)` with the new ref → node address pairs. A node address
|
||||||
* `tool.click` consume the store via `resolve("@e1")`.
|
* includes the owning flat CDP session/frame when the element lives in
|
||||||
|
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||||
*
|
*
|
||||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||||
* returns `null`, never silently leaks. Storing values in different
|
* returns `null`, never silently leaks. Storing values in different
|
||||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
|||||||
export interface RefEntry {
|
export interface RefEntry {
|
||||||
backendNodeId: BackendNodeId;
|
backendNodeId: BackendNodeId;
|
||||||
tabId: number | null;
|
tabId: number | null;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
generation: number;
|
generation: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
export type RefInput =
|
||||||
|
| BackendNodeId
|
||||||
|
| {
|
||||||
|
backendNodeId: BackendNodeId;
|
||||||
|
tabId: number;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
};
|
||||||
|
|
||||||
export class RefStore {
|
export class RefStore {
|
||||||
private readonly map = new Map<string, RefEntry>();
|
private readonly map = new Map<string, RefEntry>();
|
||||||
@@ -44,7 +54,7 @@ export class RefStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||||
* Used after every fresh `tool.snapshot`.
|
* Used after every fresh `tool.snapshot`.
|
||||||
*/
|
*/
|
||||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||||
@@ -53,10 +63,20 @@ export class RefStore {
|
|||||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
set(
|
||||||
|
ref: string,
|
||||||
|
id: BackendNodeId,
|
||||||
|
opts: {
|
||||||
|
tabId?: number;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
} = {},
|
||||||
|
): void {
|
||||||
this.map.set(normaliseRef(ref), {
|
this.map.set(normaliseRef(ref), {
|
||||||
backendNodeId: id,
|
backendNodeId: id,
|
||||||
tabId: opts.tabId ?? null,
|
tabId: opts.tabId ?? null,
|
||||||
|
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||||
|
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -80,6 +100,8 @@ export class RefStore {
|
|||||||
return {
|
return {
|
||||||
backendNodeId: input.backendNodeId,
|
backendNodeId: input.backendNodeId,
|
||||||
tabId: input.tabId,
|
tabId: input.tabId,
|
||||||
|
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||||
|
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
|
||||||
|
|
||||||
|
const RECORDING_DOCUMENT_ID_PATTERN =
|
||||||
|
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||||
|
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
|
||||||
|
|
||||||
|
export function recordingDocumentMarkerValue(identity: string): string {
|
||||||
|
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
|
||||||
|
throw new TypeError("recording Document identity must be a random UUID");
|
||||||
|
}
|
||||||
|
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
|
||||||
|
export function readRecordingDocumentIdentity(
|
||||||
|
attrs: Readonly<Record<string, string>>,
|
||||||
|
): string | undefined {
|
||||||
|
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
|
||||||
|
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
|
||||||
|
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
|
||||||
|
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
|
||||||
|
}
|
||||||
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
|
|||||||
expect(message.requestId.length).toBeGreaterThan(0);
|
expect(message.requestId.length).toBeGreaterThan(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("proactively focuses the user window before sending the overlay message", async () => {
|
||||||
|
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||||
|
windows.getLastFocused.mockResolvedValueOnce(
|
||||||
|
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||||
|
);
|
||||||
|
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||||
|
|
||||||
|
const pending = requestBorrowConfirmation(42, {
|
||||||
|
deps: { tabs, windows, notifications },
|
||||||
|
});
|
||||||
|
await vi.runAllTimersAsync();
|
||||||
|
await pending;
|
||||||
|
|
||||||
|
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still proceeds when proactive window focus fails", async () => {
|
||||||
|
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||||
|
windows.getLastFocused.mockResolvedValueOnce(
|
||||||
|
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||||
|
);
|
||||||
|
windows.update.mockRejectedValueOnce(new Error("window gone"));
|
||||||
|
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||||
|
|
||||||
|
const pending = requestBorrowConfirmation(42, {
|
||||||
|
deps: { tabs, windows, notifications },
|
||||||
|
});
|
||||||
|
await vi.runAllTimersAsync();
|
||||||
|
|
||||||
|
expect(await pending).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("sends confirmation to the borrow target when it is the active tab", async () => {
|
it("sends confirmation to the borrow target when it is the active tab", async () => {
|
||||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
|
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
|
||||||
windows.getLastFocused.mockResolvedValueOnce(
|
windows.getLastFocused.mockResolvedValueOnce(
|
||||||
@@ -612,4 +644,14 @@ describe("isInjectableContentScriptUrl", () => {
|
|||||||
).toBe(false);
|
).toBe(false);
|
||||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects the Edge Add-ons store", () => {
|
||||||
|
expect(
|
||||||
|
isInjectableContentScriptUrl(
|
||||||
|
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||||
|
),
|
||||||
|
).toBe(false);
|
||||||
|
// Non-store paths on the same host stay injectable.
|
||||||
|
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
return id;
|
return id;
|
||||||
}),
|
}),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import type {
|
|||||||
RequestFrame,
|
RequestFrame,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
|
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
|
||||||
|
|
||||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
@@ -44,16 +45,120 @@ function makeRequest(method: string, params: unknown): RequestFrame {
|
|||||||
|
|
||||||
describe("ToolDispatcher", () => {
|
describe("ToolDispatcher", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
resetBrowserObservationForTests();
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"tool.snapshot",
|
||||||
|
"tool.evaluate",
|
||||||
|
"tool.get_html",
|
||||||
|
"tool.console",
|
||||||
|
])("annotates Chrome extension-access failures from %s", async (method) => {
|
||||||
|
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
get: vi.fn(async () => tab),
|
||||||
|
query: vi.fn(async () => [tab]),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(),
|
||||||
|
ensureActiveTab: vi.fn(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("test");
|
||||||
|
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn().mockRejectedValue(error),
|
||||||
|
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
|
||||||
|
consoleEntriesSince: vi.fn(),
|
||||||
|
} as unknown as TestDispatcherCdp;
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||||
|
dispatcher.start();
|
||||||
|
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
expect(sent[0]).toMatchObject({
|
||||||
|
error: {
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "cdp_extension_access_denied" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
dispatcher.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses the configured recording runtime for start and stop", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(),
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const tab = {
|
||||||
|
id: 7,
|
||||||
|
windowId: 4242,
|
||||||
|
active: true,
|
||||||
|
status: "complete",
|
||||||
|
url: "https://example.com/start",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const frameCoordinator = {
|
||||||
|
begin: vi.fn(),
|
||||||
|
armTab: vi.fn(async () => true),
|
||||||
|
sourceFor: vi.fn(() => null),
|
||||||
|
stop: vi.fn(async () => true),
|
||||||
|
cancel: vi.fn(),
|
||||||
|
};
|
||||||
|
const sendToTab = vi.fn(async () => ({ ok: true }));
|
||||||
|
setBrowserObservationAttachForTests(
|
||||||
|
() => () => {},
|
||||||
|
() => () => {},
|
||||||
|
);
|
||||||
|
const dispatcher = new ToolDispatcher({
|
||||||
|
transport,
|
||||||
|
sessions,
|
||||||
|
recording: {
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(async () => tab),
|
||||||
|
query: vi.fn(async () => [tab]),
|
||||||
|
},
|
||||||
|
frameCoordinator,
|
||||||
|
sendToTab,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(
|
||||||
|
makeRequest("tool.record_start", {
|
||||||
|
session_id: "aa11",
|
||||||
|
url: "https://example.com/start",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
|
||||||
|
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
|
||||||
|
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
|
||||||
|
|
||||||
|
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(2));
|
||||||
|
|
||||||
|
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
|
||||||
|
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
|
||||||
|
});
|
||||||
|
|
||||||
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -76,7 +181,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create,
|
create,
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -94,7 +199,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
await sessions.start("aa11");
|
await sessions.start("aa11");
|
||||||
@@ -108,6 +213,47 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
|
||||||
|
// Regression guard for the deps that session_stop reads directly: when
|
||||||
|
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
|
||||||
|
// the agent-tab cleanup and the release decision silently no-op and the
|
||||||
|
// Agent Window is closed even though a user tab is still open (issue #57).
|
||||||
|
const removeTab = vi.fn(async () => {});
|
||||||
|
const closeWindow = vi.fn(async () => {});
|
||||||
|
// After cleanup the window still holds one tab (id 99) that is neither
|
||||||
|
// the home tab nor an agent-created tab — i.e. a genuine user tab.
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
remove: removeTab,
|
||||||
|
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: closeWindow,
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
// Register the exact tab id returned by the agent's creation path.
|
||||||
|
sessions.get("aa11")?.agentCreatedTabs.add(7);
|
||||||
|
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
// The agent tab is closed through the real chrome.tabs surface...
|
||||||
|
expect(removeTab).toHaveBeenCalledWith(7);
|
||||||
|
// ...and the window is released rather than closed, so the user's tab survives.
|
||||||
|
expect(closeWindow).not.toHaveBeenCalled();
|
||||||
|
expect(sessions.has("aa11")).toBe(false);
|
||||||
|
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
|
||||||
|
});
|
||||||
|
|
||||||
it("routes tool.console through the CDP console buffer", async () => {
|
it("routes tool.console through the CDP console buffer", async () => {
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
tabs: {
|
tabs: {
|
||||||
@@ -119,7 +265,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
await sessions.start("aa11");
|
await sessions.start("aa11");
|
||||||
@@ -168,13 +314,114 @@ describe("ToolDispatcher", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||||
|
const sendMessage = vi.fn(async () => undefined);
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })),
|
||||||
|
query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]),
|
||||||
|
sendMessage,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const ctx = await sessions.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||||
|
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T;
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
|
||||||
|
}
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
return { object: { objectId: "trigger-object" } } as T;
|
||||||
|
}
|
||||||
|
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
|
if (declaration.includes("count: state.inputs.length")) {
|
||||||
|
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||||
|
}
|
||||||
|
if (declaration.includes("inputs[0]")) {
|
||||||
|
return { result: { objectId: "input-object" } } as T;
|
||||||
|
}
|
||||||
|
return { result: { value: true } } as T;
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string }).expression ?? "";
|
||||||
|
if (expression.includes("overlayDetails")) {
|
||||||
|
return { result: { value: { hitIndex: 0 } } } as T;
|
||||||
|
}
|
||||||
|
if (expression.includes("overlayHostPresent")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||||
|
},
|
||||||
|
} as T;
|
||||||
|
}
|
||||||
|
if (expression.includes("count:")) {
|
||||||
|
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||||
|
}
|
||||||
|
if (expression.includes("?.inputs[0]")) {
|
||||||
|
return { result: { objectId: "input-object" } } as T;
|
||||||
|
}
|
||||||
|
return { result: { value: true } } as T;
|
||||||
|
}
|
||||||
|
return {} as T;
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send,
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(
|
||||||
|
makeRequest("tool.upload", {
|
||||||
|
session_id: "aa11",
|
||||||
|
ref: "@e1",
|
||||||
|
files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await flushMicrotasks();
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||||
|
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||||
|
type: "bh-automation-bypass",
|
||||||
|
enabled: true,
|
||||||
|
});
|
||||||
|
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||||
|
type: "bh-automation-bypass",
|
||||||
|
enabled: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("detaches CDP state before stopping a session", async () => {
|
it("detaches CDP state before stopping a session", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
await sessions.start("aa11");
|
await sessions.start("aa11");
|
||||||
@@ -203,13 +450,62 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const remove = vi.fn(async () => {});
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove,
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
let finishDetach: () => void = () => {};
|
||||||
|
const detach = new Promise<void>((resolve) => {
|
||||||
|
finishDetach = resolve;
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(),
|
||||||
|
detachSession: vi.fn(() => detach),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
|
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||||
|
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||||
|
await flushMicrotasks();
|
||||||
|
finishDetach();
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||||
|
expect(sent).toContainEqual({
|
||||||
|
id: "r-1",
|
||||||
|
error: expect.objectContaining({ code: "cancelled" }),
|
||||||
|
});
|
||||||
|
expect(sessions.has("aa11")).toBe(true);
|
||||||
|
expect(remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("returns not_found when stopping an unknown session", async () => {
|
it("returns not_found when stopping an unknown session", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -229,7 +525,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -253,7 +549,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -270,7 +566,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -287,7 +583,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const onSessionsChanged = vi.fn();
|
const onSessionsChanged = vi.fn();
|
||||||
@@ -304,16 +600,32 @@ describe("ToolDispatcher", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
|
||||||
|
id: 7,
|
||||||
|
windowId: props.windowId,
|
||||||
|
url: props.url,
|
||||||
|
active: props.active,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
});
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const onBrowserControlResumed = vi.fn();
|
const onBrowserControlResumed = vi.fn();
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
const onAgentTabClaimed = vi.fn();
|
||||||
|
const dispatcher = new ToolDispatcher({
|
||||||
|
transport,
|
||||||
|
sessions,
|
||||||
|
onBrowserControlResumed,
|
||||||
|
onAgentTabClaimed,
|
||||||
|
});
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||||
@@ -330,6 +642,7 @@ describe("ToolDispatcher", () => {
|
|||||||
});
|
});
|
||||||
await flushMicrotasks();
|
await flushMicrotasks();
|
||||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||||
|
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||||
@@ -343,7 +656,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const order: string[] = [];
|
const order: string[] = [];
|
||||||
@@ -425,7 +738,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const cdp = {
|
const cdp = {
|
||||||
@@ -475,6 +788,135 @@ describe("ToolDispatcher", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
{ tab_id: 7, code: "not_found" },
|
||||||
|
{ tab_id: undefined, code: "invalid_params" },
|
||||||
|
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
|
||||||
|
const sendMessage = vi.fn(async () => undefined);
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage,
|
||||||
|
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
|
||||||
|
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
releaseSessionTab: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({
|
||||||
|
transport,
|
||||||
|
sessions,
|
||||||
|
cdp: cdp as unknown as TestDispatcherCdp,
|
||||||
|
});
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
dispatcher.start();
|
||||||
|
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ error: { code } });
|
||||||
|
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
|
||||||
|
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
dispatcher.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
|
||||||
|
const sendMessage = vi.fn(async () => undefined);
|
||||||
|
const move = vi.fn(async () => {
|
||||||
|
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||||
|
return { id: 7, windowId: 200, index: 4 };
|
||||||
|
});
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: { sendMessage, move },
|
||||||
|
windows: { get: vi.fn(async () => ({ id: 200 })) },
|
||||||
|
});
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const ctx = await sessions.start("aa11");
|
||||||
|
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
releaseSessionTab: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({
|
||||||
|
transport,
|
||||||
|
sessions,
|
||||||
|
cdp: cdp as unknown as TestDispatcherCdp,
|
||||||
|
});
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
for (const [sessionId, tabId] of [
|
||||||
|
["aa11", 7],
|
||||||
|
["aa11", 8],
|
||||||
|
["bb22", 9],
|
||||||
|
] as const) {
|
||||||
|
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass(sessionId, tabId, true);
|
||||||
|
}
|
||||||
|
dispatcher.start();
|
||||||
|
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
|
||||||
|
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
for (const [sessionId, tabId] of [
|
||||||
|
["aa11", 8],
|
||||||
|
["bb22", 9],
|
||||||
|
] as const) {
|
||||||
|
expect(sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
tabId,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
|
||||||
|
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
dispatcher.stop();
|
||||||
|
});
|
||||||
|
|
||||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
tabs: {
|
tabs: {
|
||||||
@@ -486,7 +928,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -523,7 +965,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
await sessions.start("aa11");
|
await sessions.start("aa11");
|
||||||
@@ -575,7 +1017,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -609,7 +1051,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 4242),
|
create: vi.fn(async () => 4242),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
transport.send = () => {
|
transport.send = () => {
|
||||||
@@ -631,7 +1073,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 5555),
|
create: vi.fn(async () => 5555),
|
||||||
remove,
|
remove,
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
transport.send = () => {
|
transport.send = () => {
|
||||||
@@ -659,7 +1101,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(() => createPromise),
|
create: vi.fn(() => createPromise),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -710,7 +1152,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(async () => 1),
|
create: vi.fn(async () => 1),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
@@ -731,7 +1173,7 @@ describe("ToolDispatcher", () => {
|
|||||||
agentWindow: {
|
agentWindow: {
|
||||||
create: vi.fn(() => createPromise),
|
create: vi.fn(() => createPromise),
|
||||||
remove: vi.fn(),
|
remove: vi.fn(),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
|||||||
@@ -1,24 +1,54 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
import { nodeContentRegion } from "../element-geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
describe("quadBoundingRect", () => {
|
describe("nodeContentRegion", () => {
|
||||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
const send = async (_tabId: number, method: string) => {
|
||||||
x: 10,
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||||
y: 20,
|
if (method === "DOM.getBoxModel") {
|
||||||
width: 100,
|
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||||
height: 40,
|
}
|
||||||
|
throw new Error(`unexpected CDP call ${method}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||||
|
[
|
||||||
|
{ x: 5, y: 10 },
|
||||||
|
{ x: 25, y: 10 },
|
||||||
|
{ x: 25, y: 30 },
|
||||||
|
{ x: 5, y: 30 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves every visible content fragment", async () => {
|
||||||
|
const send = async () => ({
|
||||||
|
quads: [
|
||||||
|
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||||
|
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||||
|
],
|
||||||
});
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||||
|
[
|
||||||
|
{ x: 10, y: 20 },
|
||||||
|
{ x: 60, y: 20 },
|
||||||
|
{ x: 60, y: 40 },
|
||||||
|
{ x: 10, y: 40 },
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{ x: 15, y: 50 },
|
||||||
|
{ x: 90, y: 50 },
|
||||||
|
{ x: 90, y: 80 },
|
||||||
|
{ x: 15, y: 80 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns null for degenerate quads", () => {
|
|
||||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
|
||||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("nodeBoundingRect", () => {
|
|
||||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||||
const calls: Array<{ method: string; params?: object }> = [];
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
const send = async (_tabId: number, method: string, params?: object) => {
|
const send = async (_tabId: number, method: string, params?: object) => {
|
||||||
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
|
|||||||
};
|
};
|
||||||
const cdp = { send: send as CdpRunner["send"] };
|
const cdp = { send: send as CdpRunner["send"] };
|
||||||
|
|
||||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||||
x: 242,
|
[
|
||||||
y: 468,
|
{ x: 242, y: 468 },
|
||||||
width: 769,
|
{ x: 1011, y: 468 },
|
||||||
height: 180,
|
{ x: 1011, y: 648 },
|
||||||
});
|
{ x: 242, y: 648 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
expect(calls.map((c) => c.method)).toEqual([
|
expect(calls.map((c) => c.method)).toEqual([
|
||||||
"DOM.getContentQuads",
|
"DOM.getContentQuads",
|
||||||
"DOM.getBoxModel",
|
"DOM.getBoxModel",
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
return id;
|
return id;
|
||||||
}),
|
}),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { cdpError, classifyCdpError } from "../errors";
|
||||||
|
|
||||||
|
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||||
|
|
||||||
|
describe("CDP extension access errors", () => {
|
||||||
|
it("classifies both Chrome Error instances and string failures", () => {
|
||||||
|
for (const error of [new Error(denied), denied]) {
|
||||||
|
expect(cdpError(error)).toEqual({
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: denied,
|
||||||
|
data: { reason: "cdp_extension_access_denied" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
|
||||||
|
const error: RpcError = {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: denied,
|
||||||
|
data: {
|
||||||
|
reason: "transfer_outcome_unknown",
|
||||||
|
effect_state: "unknown",
|
||||||
|
cleanup_state: "failed",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
expect(classifyCdpError(error)).toBe(error);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves other error codes and unrelated debugger failures unchanged", () => {
|
||||||
|
for (const error of [
|
||||||
|
{ code: "permission_denied", message: denied },
|
||||||
|
{ code: "cdp_failed", message: "Another debugger is already attached" },
|
||||||
|
] satisfies RpcError[]) {
|
||||||
|
expect(classifyCdpError(error)).toBe(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
return id;
|
return id;
|
||||||
}),
|
}),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,988 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { type DownloadsApi, handleDownload } from "../download";
|
||||||
|
import { captureBrowserDownload } from "../download-capture";
|
||||||
|
import { uploadThroughFileDrop } from "../file-drop-transaction";
|
||||||
|
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||||
|
import type { ResolvedActionTarget } from "../interaction";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
import { handleUpload } from "../upload";
|
||||||
|
|
||||||
|
function sessions() {
|
||||||
|
return new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function tabsApi() {
|
||||||
|
return {
|
||||||
|
get: vi.fn(
|
||||||
|
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||||
|
const listeners = new Set<T>();
|
||||||
|
return {
|
||||||
|
addListener: (listener: T) => listeners.add(listener),
|
||||||
|
removeListener: (listener: T) => listeners.delete(listener),
|
||||||
|
emit: (...args: Parameters<T>) => {
|
||||||
|
for (const listener of listeners) listener(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
|
||||||
|
return {
|
||||||
|
tab: { tabId: 4, windowId: 100, active: true },
|
||||||
|
backendNodeId: 123,
|
||||||
|
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
|
||||||
|
...(frameId ? { frameId } : {}),
|
||||||
|
usedRef: "e3",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
|
||||||
|
return operation();
|
||||||
|
}
|
||||||
|
|
||||||
|
function uploadCdp(
|
||||||
|
options: {
|
||||||
|
inputCount?: number;
|
||||||
|
multiple?: boolean;
|
||||||
|
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
|
||||||
|
pendingResolve?: boolean;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
calls.push({ method, params });
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics")
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
if (options.pendingResolve) return new Promise<never>(() => {});
|
||||||
|
return { object: { objectId: "trigger-object" } };
|
||||||
|
}
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
|
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
|
||||||
|
if (declaration.includes("count: state.inputs.length")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
count: options.inputCount ?? 1,
|
||||||
|
multiple: options.multiple ?? true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (declaration.includes("inputs[0]")) {
|
||||||
|
return { result: { objectId: "input-object" } };
|
||||||
|
}
|
||||||
|
return { result: { value: true } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchMouseEvent" &&
|
||||||
|
(params as { type?: string }).type === "mousePressed" &&
|
||||||
|
options.chooser
|
||||||
|
) {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
calls,
|
||||||
|
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
|
||||||
|
cdp,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function dropCdp(
|
||||||
|
options: {
|
||||||
|
targetContainsPoint?: boolean;
|
||||||
|
dispatchError?: (type: string) => Error | Promise<never> | undefined;
|
||||||
|
onCall?: (method: string) => void;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
|
const detach = vi.fn(async () => {});
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
calls.push({ method, params });
|
||||||
|
options.onCall?.(method);
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||||
|
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||||
|
}
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
return { result: { value: options.targetContainsPoint ?? true } };
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchDragEvent") {
|
||||||
|
const type = (params as { type?: string }).type ?? "";
|
||||||
|
const error = options.dispatchError?.(type);
|
||||||
|
if (error instanceof Promise) return error;
|
||||||
|
if (error) throw error;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.releaseObjectGroup") return {};
|
||||||
|
throw new Error(`unexpected drop CDP command ${method}`);
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
detach,
|
||||||
|
};
|
||||||
|
return { calls, cdp, detach };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("file transfer tools", () => {
|
||||||
|
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||||
|
const manager = sessions();
|
||||||
|
const ctx = await manager.start("s1");
|
||||||
|
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||||
|
const { cdp, calls } = uploadCdp();
|
||||||
|
|
||||||
|
const result = await handleUpload(
|
||||||
|
manager,
|
||||||
|
{
|
||||||
|
session_id: "s1",
|
||||||
|
ref: "@e3",
|
||||||
|
files: [
|
||||||
|
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
|
||||||
|
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{ cdp, tabsApi: tabsApi() },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
|
||||||
|
expect(calls[0]).toEqual({
|
||||||
|
method: "Page.setInterceptFileChooserDialog",
|
||||||
|
params: { enabled: true },
|
||||||
|
});
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "Page.setInterceptFileChooserDialog",
|
||||||
|
params: { enabled: false, cancel: true },
|
||||||
|
});
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "DOM.setFileInputFiles",
|
||||||
|
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails immediately when the trigger does not activate a file input", async () => {
|
||||||
|
const { cdp } = uploadCdp({ inputCount: 0 });
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "unsupported",
|
||||||
|
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails before clicking when chooser interception is unavailable", async () => {
|
||||||
|
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.setInterceptFileChooserDialog") {
|
||||||
|
throw new Error("method unavailable");
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { effect_state: "none", phase: "arm_interception" },
|
||||||
|
});
|
||||||
|
expect(trigger).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses an exact chooser event as an independent input-location signal", async () => {
|
||||||
|
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget("f1"),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => {
|
||||||
|
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ multiple: false });
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "DOM.setFileInputFiles",
|
||||||
|
params: { files: ["/private/stage/one"], backendNodeId: 789 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a File System Access picker without waiting for a timeout", async () => {
|
||||||
|
const { cdp, emitChooser } = uploadCdp({
|
||||||
|
inputCount: 0,
|
||||||
|
});
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget("f1"),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => {
|
||||||
|
emitChooser({ frameId: "f1", mode: "selectSingle" });
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "unsupported",
|
||||||
|
message: "upload trigger invoked a non-input file picker",
|
||||||
|
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds a stuck file-input probe", async () => {
|
||||||
|
const { cdp } = uploadCdp({ pendingResolve: true });
|
||||||
|
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 5,
|
||||||
|
trigger: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
data: { reason: "file_input_probe_failed", phase: "arm_input_probe" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||||
|
const fixture = uploadCdp();
|
||||||
|
const originalSend = fixture.cdp.send;
|
||||||
|
const detach = vi.fn(async () => {});
|
||||||
|
fixture.cdp.detach = detach;
|
||||||
|
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
|
||||||
|
return originalSend(tabId, method, params);
|
||||||
|
}) as CdpRunner["send"];
|
||||||
|
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp: fixture.cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 10,
|
||||||
|
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
data: { effect_state: "unknown", phase: "set_files" },
|
||||||
|
});
|
||||||
|
expect(detach).toHaveBeenCalledWith(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("delivers staged files through one explicit native drop without clicking", async () => {
|
||||||
|
const manager = sessions();
|
||||||
|
const ctx = await manager.start("s1");
|
||||||
|
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||||
|
const lifecycle: string[] = [];
|
||||||
|
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
|
||||||
|
const overlayPhases: string[] = [];
|
||||||
|
|
||||||
|
const result = await handleUpload(
|
||||||
|
manager,
|
||||||
|
{
|
||||||
|
session_id: "s1",
|
||||||
|
ref: "@e3",
|
||||||
|
mode: "drop",
|
||||||
|
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
cdp,
|
||||||
|
tabsApi: tabsApi(),
|
||||||
|
sendToTab: vi.fn(async (_tabId, message) => {
|
||||||
|
overlayPhases.push(message.phase);
|
||||||
|
lifecycle.push(`overlay:${message.phase}`);
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
|
||||||
|
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
|
||||||
|
{
|
||||||
|
method: "Input.dispatchDragEvent",
|
||||||
|
params: {
|
||||||
|
type: "dragEnter",
|
||||||
|
x: 110,
|
||||||
|
y: 70,
|
||||||
|
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: "Input.dispatchDragEvent",
|
||||||
|
params: {
|
||||||
|
type: "dragOver",
|
||||||
|
x: 110,
|
||||||
|
y: 70,
|
||||||
|
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
method: "Input.dispatchDragEvent",
|
||||||
|
params: {
|
||||||
|
type: "drop",
|
||||||
|
x: 110,
|
||||||
|
y: 70,
|
||||||
|
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||||
|
expect(overlayPhases).toEqual(["begin", "end"]);
|
||||||
|
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
|
||||||
|
lifecycle.indexOf("overlay:begin"),
|
||||||
|
);
|
||||||
|
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
|
||||||
|
lifecycle.indexOf("DOM.getNodeForLocation"),
|
||||||
|
);
|
||||||
|
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
|
||||||
|
lifecycle.indexOf("overlay:end"),
|
||||||
|
);
|
||||||
|
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
|
||||||
|
lifecycle.indexOf("Runtime.releaseObjectGroup"),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed before dragging when another element owns the action point", async () => {
|
||||||
|
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
|
||||||
|
const result = await uploadThroughFileDrop({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/stage/one.png"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
withOverlayHidden: withoutOverlay,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "permission_denied",
|
||||||
|
data: {
|
||||||
|
reason: "file_drop_target_unavailable",
|
||||||
|
mechanism: "drop",
|
||||||
|
effect_state: "none",
|
||||||
|
phase: "resolve_target",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
|
||||||
|
const rootCalls: Array<{ method: string; params?: object }> = [];
|
||||||
|
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method, params) => {
|
||||||
|
rootCalls.push({ method, params });
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method, params) => {
|
||||||
|
childCalls.push({ method, params });
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getNodeForLocation") {
|
||||||
|
return { backendNodeId: 124, frameId: "child" };
|
||||||
|
}
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||||
|
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||||
|
}
|
||||||
|
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
|
||||||
|
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await uploadThroughFileDrop({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget("child", "child-session"),
|
||||||
|
files: ["/stage/one.png"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
withOverlayHidden: withoutOverlay,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ x: 110, y: 70 });
|
||||||
|
expect(
|
||||||
|
childCalls
|
||||||
|
.filter((call) => call.method === "Input.dispatchDragEvent")
|
||||||
|
.map((call) => call.params),
|
||||||
|
).toEqual([
|
||||||
|
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
|
||||||
|
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
|
||||||
|
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
|
||||||
|
]);
|
||||||
|
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a missing native drag command as unsupported before commit", async () => {
|
||||||
|
const { cdp } = dropCdp({
|
||||||
|
dispatchError: (type) =>
|
||||||
|
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
|
||||||
|
});
|
||||||
|
const result = await uploadThroughFileDrop({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/stage/one.png"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
withOverlayHidden: withoutOverlay,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "unsupported",
|
||||||
|
data: {
|
||||||
|
reason: "upload_mechanism_unsupported",
|
||||||
|
effect_state: "none",
|
||||||
|
phase: "drag_enter",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
|
||||||
|
const { cdp, calls } = dropCdp({
|
||||||
|
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
|
||||||
|
});
|
||||||
|
const result = await uploadThroughFileDrop({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/stage/one.png"],
|
||||||
|
timeoutMs: 10,
|
||||||
|
withOverlayHidden: withoutOverlay,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
|
||||||
|
});
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "Input.dispatchDragEvent",
|
||||||
|
params: expect.objectContaining({ type: "dragCancel" }),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("marks an unresolved drop unknown and detaches browser state", async () => {
|
||||||
|
const { cdp, detach } = dropCdp({
|
||||||
|
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
|
||||||
|
});
|
||||||
|
const result = await uploadThroughFileDrop({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/stage/one.png"],
|
||||||
|
timeoutMs: 10,
|
||||||
|
withOverlayHidden: withoutOverlay,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
|
||||||
|
});
|
||||||
|
expect(detach).toHaveBeenCalledWith(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||||
|
const manager = sessions();
|
||||||
|
const ctx = await manager.start("s1");
|
||||||
|
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const initial = {
|
||||||
|
id: 9,
|
||||||
|
url: "https://example.test/result.zip",
|
||||||
|
finalUrl: "https://example.test/result.zip",
|
||||||
|
filename: "result.zip",
|
||||||
|
state: "in_progress",
|
||||||
|
fileSize: -1,
|
||||||
|
totalBytes: 12,
|
||||||
|
mime: "application/zip",
|
||||||
|
danger: "safe",
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const completed = {
|
||||||
|
...initial,
|
||||||
|
filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip",
|
||||||
|
state: "complete",
|
||||||
|
fileSize: 12,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => [completed]),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "Page.getLayoutMetrics")
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchMouseEvent" &&
|
||||||
|
(params as { type?: string }).type === "mousePressed"
|
||||||
|
) {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: initial.url,
|
||||||
|
suggestedFilename: "result.zip",
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
onDeterminingFilename.emit(initial, (value) => {
|
||||||
|
suggested = value;
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
onCreated.emit(completed);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await handleDownload(
|
||||||
|
manager,
|
||||||
|
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
|
||||||
|
{ cdp, tabsApi: tabsApi(), downloads },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(suggested).toEqual({
|
||||||
|
filename: "BrowserSkill/tr_1/result.zip",
|
||||||
|
conflictAction: "overwrite",
|
||||||
|
});
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
tab_id: 4,
|
||||||
|
suggested_filename: "result.zip",
|
||||||
|
byte_size: 12,
|
||||||
|
browser_path: completed.filename,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not claim a download without an intent from the exact target", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => []),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as unknown as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const unrelated = {
|
||||||
|
id: 17,
|
||||||
|
url: "https://example.test/unrelated.zip",
|
||||||
|
finalUrl: "https://example.test/unrelated.zip",
|
||||||
|
filename: "unrelated.zip",
|
||||||
|
state: "in_progress",
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
let defaultSuggestionCalled = false;
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4, sessionId: "expected-child" },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_1",
|
||||||
|
timeoutMs: 5,
|
||||||
|
trigger: async () => {
|
||||||
|
cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", {
|
||||||
|
url: unrelated.url,
|
||||||
|
suggestedFilename: unrelated.filename,
|
||||||
|
});
|
||||||
|
onDeterminingFilename.emit(unrelated, (suggestion) => {
|
||||||
|
defaultSuggestionCalled = suggestion === undefined;
|
||||||
|
});
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(defaultSuggestionCalled).toBe(true);
|
||||||
|
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "download_capture_failed" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("correlates a filename candidate that arrives before the CDP intent", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const initial = {
|
||||||
|
id: 21,
|
||||||
|
url: "https://example.test/candidate-first.bin",
|
||||||
|
finalUrl: "https://example.test/candidate-first.bin",
|
||||||
|
filename: "candidate-first.bin",
|
||||||
|
state: "in_progress",
|
||||||
|
fileSize: -1,
|
||||||
|
totalBytes: 4,
|
||||||
|
bytesReceived: 0,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const complete = {
|
||||||
|
...initial,
|
||||||
|
state: "complete",
|
||||||
|
fileSize: 4,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => [complete]),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_21",
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
trigger: async () => {
|
||||||
|
const suggested = new Promise<void>((resolve) => {
|
||||||
|
onDeterminingFilename.emit(initial, (value) => {
|
||||||
|
suggestion = value;
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: initial.url,
|
||||||
|
suggestedFilename: initial.filename,
|
||||||
|
});
|
||||||
|
await suggested;
|
||||||
|
onCreated.emit(complete);
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(suggestion).toEqual({
|
||||||
|
filename: "BrowserSkill/tr_21/candidate-first.bin",
|
||||||
|
conflictAction: "overwrite",
|
||||||
|
});
|
||||||
|
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes a completed download when the final size exceeds the limit", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const initial = {
|
||||||
|
id: 22,
|
||||||
|
url: "https://example.test/oversized.bin",
|
||||||
|
finalUrl: "https://example.test/oversized.bin",
|
||||||
|
filename: "oversized.bin",
|
||||||
|
state: "in_progress",
|
||||||
|
fileSize: -1,
|
||||||
|
totalBytes: -1,
|
||||||
|
bytesReceived: 0,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const complete = {
|
||||||
|
...initial,
|
||||||
|
filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin",
|
||||||
|
state: "complete",
|
||||||
|
fileSize: 8,
|
||||||
|
totalBytes: 8,
|
||||||
|
bytesReceived: 8,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => [complete]),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_22",
|
||||||
|
maxByteSize: 4,
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
trigger: async () => {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: initial.url,
|
||||||
|
suggestedFilename: initial.filename,
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
onDeterminingFilename.emit(initial, () => resolve());
|
||||||
|
});
|
||||||
|
onCreated.emit(initial);
|
||||||
|
onChanged.emit({ id: initial.id, state: { current: "complete" } });
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||||
|
});
|
||||||
|
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||||
|
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reconciles a download that completes while cancellation is being requested", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const initial = {
|
||||||
|
id: 23,
|
||||||
|
url: "https://example.test/racing.bin",
|
||||||
|
finalUrl: "https://example.test/racing.bin",
|
||||||
|
filename: "racing.bin",
|
||||||
|
state: "in_progress",
|
||||||
|
fileSize: -1,
|
||||||
|
totalBytes: -1,
|
||||||
|
bytesReceived: 0,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const complete = {
|
||||||
|
...initial,
|
||||||
|
filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin",
|
||||||
|
state: "complete",
|
||||||
|
fileSize: 4,
|
||||||
|
totalBytes: 4,
|
||||||
|
bytesReceived: 4,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
|
||||||
|
cancel: vi.fn(async () => {
|
||||||
|
throw new Error("download already complete");
|
||||||
|
}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_23",
|
||||||
|
timeoutMs: 80,
|
||||||
|
trigger: async () => {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: initial.url,
|
||||||
|
suggestedFilename: initial.filename,
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
onDeterminingFilename.emit(initial, () => resolve());
|
||||||
|
});
|
||||||
|
onCreated.emit(initial);
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||||
|
});
|
||||||
|
expect(downloads.cancel).toHaveBeenCalledWith(initial.id);
|
||||||
|
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||||
|
expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => []),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const defaults: number[] = [];
|
||||||
|
const candidate = (id: number) =>
|
||||||
|
({
|
||||||
|
id,
|
||||||
|
url: "https://example.test/same.bin",
|
||||||
|
finalUrl: "https://example.test/same.bin",
|
||||||
|
filename: "same.bin",
|
||||||
|
state: "in_progress",
|
||||||
|
}) as chrome.downloads.DownloadItem;
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_ambiguous",
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: "https://example.test/same.bin",
|
||||||
|
suggestedFilename: "same.bin",
|
||||||
|
});
|
||||||
|
for (const id of [31, 32]) {
|
||||||
|
onDeterminingFilename.emit(candidate(id), (value) => {
|
||||||
|
if (value === undefined) defaults.push(id);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(defaults.sort()).toEqual([31, 32]);
|
||||||
|
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { effect_state: "unknown", phase: "attribution" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,573 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { handleFill } from "../interaction";
|
||||||
|
|
||||||
|
interface ScriptParams {
|
||||||
|
functionDeclaration: string;
|
||||||
|
arguments?: Array<{ value: unknown }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setup(markup = '<input value="old">') {
|
||||||
|
document.body.innerHTML = markup;
|
||||||
|
const element = document.body.firstElementChild as HTMLInputElement;
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: async () => 100,
|
||||||
|
remove: async () => {},
|
||||||
|
ensureActiveTab: async () => 4,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const ctx = await manager.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 12, { tabId: 4 });
|
||||||
|
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
|
||||||
|
try {
|
||||||
|
const fn = new Function(`return (${params.functionDeclaration})`)();
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: await fn.apply(
|
||||||
|
element,
|
||||||
|
(params.arguments ?? []).map((arg) => arg.value),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return { exceptionDetails: { text: "test page script failed" } };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// Number/email controls have a browser caret even though their selection
|
||||||
|
// getters return null. Model that caret instead of assuming it is at the end.
|
||||||
|
const hiddenCaret = { start: 0, end: 0 };
|
||||||
|
const moveToEnd = vi.fn(async () => {
|
||||||
|
if (typeof element.selectionStart === "number") {
|
||||||
|
element.setSelectionRange(element.value.length, element.value.length);
|
||||||
|
} else {
|
||||||
|
hiddenCaret.start = hiddenCaret.end = element.value.length;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const insert = vi.fn(async (text: string) => {
|
||||||
|
const focused = document.activeElement as HTMLInputElement;
|
||||||
|
if (focused.readOnly || focused.disabled) return;
|
||||||
|
const start = focused.selectionStart ?? hiddenCaret.start;
|
||||||
|
const end = focused.selectionEnd ?? hiddenCaret.end;
|
||||||
|
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
|
||||||
|
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
|
||||||
|
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
|
||||||
|
focused.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
});
|
||||||
|
const release = vi.fn(async () => ({}));
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
switch (method) {
|
||||||
|
case "DOM.describeNode":
|
||||||
|
return {
|
||||||
|
node: {
|
||||||
|
nodeName: element.tagName,
|
||||||
|
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
case "DOM.scrollIntoViewIfNeeded":
|
||||||
|
return {};
|
||||||
|
case "DOM.focus":
|
||||||
|
element.focus();
|
||||||
|
return {};
|
||||||
|
case "DOM.resolveNode":
|
||||||
|
return { object: { objectId: "fill-target" } };
|
||||||
|
case "Runtime.callFunctionOn":
|
||||||
|
return script(params as ScriptParams);
|
||||||
|
case "Input.insertText":
|
||||||
|
await insert((params as { text: string }).text);
|
||||||
|
return {};
|
||||||
|
case "Input.dispatchKeyEvent":
|
||||||
|
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
|
||||||
|
await moveToEnd();
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
case "Runtime.releaseObject":
|
||||||
|
return release();
|
||||||
|
default:
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
element,
|
||||||
|
hiddenCaret,
|
||||||
|
moveToEnd,
|
||||||
|
script,
|
||||||
|
insert,
|
||||||
|
release,
|
||||||
|
send,
|
||||||
|
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
|
||||||
|
handleFill(
|
||||||
|
manager,
|
||||||
|
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
|
||||||
|
{
|
||||||
|
cdp: { send: send as CdpRunner["send"] },
|
||||||
|
tabsApi: {
|
||||||
|
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
document.body.innerHTML = "";
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("fill result verification", () => {
|
||||||
|
it.each([
|
||||||
|
"readonly",
|
||||||
|
"disabled",
|
||||||
|
])("preserves a %s field before rejecting it", async (attribute) => {
|
||||||
|
const h = await setup(`<input ${attribute} value="old">`);
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
data: { reason: "target_not_fillable" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("old");
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"checkbox",
|
||||||
|
"radio",
|
||||||
|
"range",
|
||||||
|
"date",
|
||||||
|
"file",
|
||||||
|
])("rejects an unsupported %s input before clearing", async (type) => {
|
||||||
|
const h = await setup(`<input type="${type}">`);
|
||||||
|
const before = h.element.value;
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
data: { reason: "target_not_fillable" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe(before);
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an overlong value before clearing", async () => {
|
||||||
|
const h = await setup('<input maxlength="3" value="old">');
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
message: /maxlength/,
|
||||||
|
data: { reason: "fill_value_invalid" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("old");
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
|
||||||
|
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
|
||||||
|
const send = h.send.getMockImplementation()!;
|
||||||
|
h.send.mockImplementation(async (tabId, method, params) => {
|
||||||
|
if (method === "DOM.focus") throw new Error("Element is not focusable");
|
||||||
|
return send(tabId, method, params);
|
||||||
|
});
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: disabled ? "invalid_params" : "cdp_failed",
|
||||||
|
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("old");
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fills a background document without requiring foreground browser focus", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||||
|
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||||
|
expect(h.element.value).toBe("hello");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restores focus once after clearing without typing into another field", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
const other = document.createElement("input");
|
||||||
|
document.body.append(other);
|
||||||
|
h.element.addEventListener("input", () => other.focus());
|
||||||
|
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||||
|
expect(h.element.value).toBe("hello");
|
||||||
|
expect(other.value).toBe("");
|
||||||
|
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"focus",
|
||||||
|
"value",
|
||||||
|
])("rechecks %s after restoring focus, without retrying again", async (action) => {
|
||||||
|
const h = await setup();
|
||||||
|
const other = document.createElement("input");
|
||||||
|
document.body.append(other);
|
||||||
|
h.element.addEventListener(
|
||||||
|
"input",
|
||||||
|
() => {
|
||||||
|
other.focus();
|
||||||
|
h.element.addEventListener("focus", () => {
|
||||||
|
if (action === "focus") queueMicrotask(() => other.focus());
|
||||||
|
else h.element.value = "changed";
|
||||||
|
});
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
|
||||||
|
});
|
||||||
|
expect(other.value).toBe("");
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"remove",
|
||||||
|
"readonly",
|
||||||
|
"restore",
|
||||||
|
"microtask-restore",
|
||||||
|
])("stops when clearing causes the page to %s the target", async (action) => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.addEventListener("input", () => {
|
||||||
|
if (action === "remove") h.element.remove();
|
||||||
|
if (action === "readonly") h.element.readOnly = true;
|
||||||
|
if (action === "restore") h.element.value = "restored";
|
||||||
|
if (action === "microtask-restore")
|
||||||
|
queueMicrotask(() => {
|
||||||
|
h.element.value = "restored";
|
||||||
|
});
|
||||||
|
});
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "fill_target_changed" },
|
||||||
|
});
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails when insertText is acknowledged without changing the value", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
h.insert.mockImplementation(async () => {});
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "fill_value_mismatch" },
|
||||||
|
});
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("detects truncation when maxlength changes after preparation", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.addEventListener(
|
||||||
|
"input",
|
||||||
|
() => {
|
||||||
|
h.element.maxLength = 3;
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "fill_value_mismatch" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("hel");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"input",
|
||||||
|
"change",
|
||||||
|
"microtask-change",
|
||||||
|
])("detects a value reverted by the %s handler", async (event) => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
|
||||||
|
if (event === "microtask-change")
|
||||||
|
queueMicrotask(() => {
|
||||||
|
h.element.value = "";
|
||||||
|
});
|
||||||
|
else h.element.value = "";
|
||||||
|
});
|
||||||
|
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("compares content rather than length without including field values in the error", async () => {
|
||||||
|
const h = await setup('<input type="password">');
|
||||||
|
h.element.addEventListener("change", () => {
|
||||||
|
h.element.value = "other-secret";
|
||||||
|
});
|
||||||
|
const result = await h.fill("given-secret");
|
||||||
|
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||||
|
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not report success for a target replaced by a change handler", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "fill_target_changed" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
1, 2, 3, 4, 5,
|
||||||
|
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
|
||||||
|
const h = await setup();
|
||||||
|
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||||
|
const run = h.script.getMockImplementation()!;
|
||||||
|
let count = 0;
|
||||||
|
h.script.mockImplementation(async (params) =>
|
||||||
|
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
|
||||||
|
);
|
||||||
|
const result = await h.fill();
|
||||||
|
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
|
||||||
|
expect(JSON.stringify(result)).not.toContain("page secret");
|
||||||
|
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
|
||||||
|
const h = await setup();
|
||||||
|
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||||
|
const run = h.script.getMockImplementation()!;
|
||||||
|
let count = 0;
|
||||||
|
h.script.mockImplementation(async (params) =>
|
||||||
|
++count === phase ? { result: { type: "undefined" } } : run(params),
|
||||||
|
);
|
||||||
|
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||||
|
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.addEventListener(event, () => {
|
||||||
|
queueMicrotask(() =>
|
||||||
|
queueMicrotask(() => {
|
||||||
|
h.element.value = "reverted";
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||||
|
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"<input>",
|
||||||
|
'<input type="password">',
|
||||||
|
"<textarea></textarea>",
|
||||||
|
])("verifies successful text in %s", async (markup) => {
|
||||||
|
const h = await setup(markup);
|
||||||
|
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
|
||||||
|
expect(h.element.value).toBe("你好🙂");
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("verifies append against the old value plus the requested text", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.focus();
|
||||||
|
h.element.setSelectionRange(3, 3);
|
||||||
|
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
|
||||||
|
expect(h.element.value).toBe("old!");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["<textarea></textarea>", "a\r\nb", "a\nb"],
|
||||||
|
["<input>", "a\nb", "a b"],
|
||||||
|
["<input>", "a\r\nb", "a b"],
|
||||||
|
[
|
||||||
|
'<input type="email" multiple>',
|
||||||
|
" a@example.com , b@example.com ",
|
||||||
|
"a@example.com,b@example.com",
|
||||||
|
],
|
||||||
|
])("accepts native normalization for %s", async (markup, requested, normalized) => {
|
||||||
|
if (markup.startsWith("<textarea")) {
|
||||||
|
// Happy DOM does not implement Chrome's textarea line-ending normalization.
|
||||||
|
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
|
||||||
|
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
|
||||||
|
this: HTMLTextAreaElement,
|
||||||
|
value,
|
||||||
|
) {
|
||||||
|
setter.call(this, value.replace(/\r\n?/g, "\n"));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const h = await setup(markup);
|
||||||
|
h.insert.mockImplementation(async () => {
|
||||||
|
h.element.value = normalized;
|
||||||
|
});
|
||||||
|
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not count an editable padding break as an extra typed character", async () => {
|
||||||
|
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
|
||||||
|
// These two layout/editability properties are missing in Happy DOM.
|
||||||
|
let innerText = "";
|
||||||
|
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
|
||||||
|
Object.defineProperty(h.element, "innerText", { get: () => innerText });
|
||||||
|
h.insert.mockImplementation(async () => {
|
||||||
|
h.element.innerHTML = "one<div><br></div>";
|
||||||
|
innerText = "one\n\n";
|
||||||
|
});
|
||||||
|
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
[0, 0],
|
||||||
|
[1, 1],
|
||||||
|
[0, 3],
|
||||||
|
])("appends after moving selection %s:%s to the end", async (start, end) => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.focus();
|
||||||
|
h.element.setSelectionRange(start, end);
|
||||||
|
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
|
||||||
|
expect(h.element.value).toBe("old🙂");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(
|
||||||
|
["number", "email"].flatMap((type) =>
|
||||||
|
[
|
||||||
|
[0, 0],
|
||||||
|
[1, 1],
|
||||||
|
[0, 2],
|
||||||
|
].map(([start, end]) => ({ type, start, end })),
|
||||||
|
),
|
||||||
|
)("appends to $type from selection $start:$end without using its unsupported API", async ({
|
||||||
|
type,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
}) => {
|
||||||
|
const h = await setup(`<input type="${type}" value="12">`);
|
||||||
|
Object.assign(h.hiddenCaret, { start, end });
|
||||||
|
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||||
|
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||||
|
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
|
||||||
|
throw new DOMException("unsupported", "InvalidStateError");
|
||||||
|
});
|
||||||
|
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
|
||||||
|
expect(h.element.value).toBe("123");
|
||||||
|
expect(selection).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"number",
|
||||||
|
"email",
|
||||||
|
])("does not report successful %s append when the caret command is ignored", async (type) => {
|
||||||
|
const h = await setup(`<input type="${type}" value="12">`);
|
||||||
|
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||||
|
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||||
|
h.moveToEnd.mockResolvedValueOnce(undefined);
|
||||||
|
expect(await h.fill("3", false)).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "fill_value_mismatch" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("312");
|
||||||
|
expect(h.insert).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects invalid number text before clearing", async () => {
|
||||||
|
const h = await setup('<input type="number" value="12">');
|
||||||
|
expect(await h.fill("abc")).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
data: { reason: "fill_value_invalid" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("12");
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports page formatting as unconfirmed without retrying the write", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
h.element.addEventListener("change", () => {
|
||||||
|
h.element.value = h.element.value.toUpperCase();
|
||||||
|
});
|
||||||
|
expect(await h.fill()).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "fill_value_mismatch" },
|
||||||
|
});
|
||||||
|
expect(h.element.value).toBe("HELLO");
|
||||||
|
expect(h.insert).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["focus", false],
|
||||||
|
["focus", true],
|
||||||
|
["value", false],
|
||||||
|
["value", true],
|
||||||
|
["cancel", false],
|
||||||
|
["cancel", true],
|
||||||
|
["throw", false],
|
||||||
|
["throw", true],
|
||||||
|
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
|
||||||
|
const h = await setup();
|
||||||
|
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||||
|
const controller = new AbortController();
|
||||||
|
const other = document.createElement("input");
|
||||||
|
document.body.append(other);
|
||||||
|
const send = h.send.getMockImplementation()!;
|
||||||
|
h.send.mockImplementation(async (tabId, method, params) => {
|
||||||
|
const reply = await send(tabId, method, params);
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchKeyEvent" &&
|
||||||
|
(params as { type: string }).type === "rawKeyDown"
|
||||||
|
) {
|
||||||
|
if (action === "focus") other.focus();
|
||||||
|
if (action === "value") h.element.value = "changed";
|
||||||
|
if (action === "cancel") controller.abort();
|
||||||
|
if (action === "throw") throw new Error("dispatch failed");
|
||||||
|
}
|
||||||
|
return reply;
|
||||||
|
});
|
||||||
|
const result = await h.fill("!", clearBefore, controller.signal);
|
||||||
|
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
expect(other.value).toBe("");
|
||||||
|
expect(h.send).toHaveBeenCalledWith(
|
||||||
|
4,
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
expect.objectContaining({ type: "keyUp" }),
|
||||||
|
);
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
|
||||||
|
const h = await setup();
|
||||||
|
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
|
||||||
|
expect(h.element.value).toBe(clearBefore ? "" : "old");
|
||||||
|
expect(h.insert).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels after insertion without sending change events", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
const controller = new AbortController();
|
||||||
|
h.insert.mockImplementation(async () => controller.abort());
|
||||||
|
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(h.script).toHaveBeenCalledTimes(2);
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels after resolving the remote object without clearing the field", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
const controller = new AbortController();
|
||||||
|
const send = h.send.getMockImplementation()!;
|
||||||
|
h.send.mockImplementation(async (tabId, method, params) => {
|
||||||
|
const result = await send(tabId, method, params);
|
||||||
|
if (method === "DOM.resolveNode") controller.abort();
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(h.element.value).toBe("old");
|
||||||
|
expect(h.script).not.toHaveBeenCalled();
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honors cancellation during verification", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
const controller = new AbortController();
|
||||||
|
h.element.addEventListener("change", () => controller.abort());
|
||||||
|
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(h.release).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the verified outcome when releasing the remote object fails", async () => {
|
||||||
|
const h = await setup();
|
||||||
|
h.release.mockRejectedValueOnce(new Error("target closed"));
|
||||||
|
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||||
|
import {
|
||||||
|
clipPolygon,
|
||||||
|
polygonArea,
|
||||||
|
polygonBounds,
|
||||||
|
projectAndClipRegion,
|
||||||
|
projectUnitPoint,
|
||||||
|
type Quad,
|
||||||
|
rectPolygon,
|
||||||
|
regionBounds,
|
||||||
|
} from "../geometry";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
describe("frame geometry projection", () => {
|
||||||
|
it("keeps region bounds separate from polygon area", () => {
|
||||||
|
const region = [
|
||||||
|
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||||
|
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||||
|
];
|
||||||
|
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||||
|
expect(polygonArea(region[0])).toBe(100);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps through the iframe content box instead of its border box", () => {
|
||||||
|
const contentQuad: Quad = [
|
||||||
|
{ x: 204, y: 306 },
|
||||||
|
{ x: 604, y: 306 },
|
||||||
|
{ x: 604, y: 506 },
|
||||||
|
{ x: 204, y: 506 },
|
||||||
|
];
|
||||||
|
const projected = projectAndClipRegion(
|
||||||
|
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||||
|
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||||
|
{ width: 1000, height: 800 },
|
||||||
|
);
|
||||||
|
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||||
|
const quad: Quad = [
|
||||||
|
{ x: 0, y: 0 },
|
||||||
|
{ x: 200, y: 20 },
|
||||||
|
{ x: 180, y: 140 },
|
||||||
|
{ x: 20, y: 100 },
|
||||||
|
];
|
||||||
|
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||||
|
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||||
|
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||||
|
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||||
|
const frameQuad: Quad = [
|
||||||
|
{ x: 50, y: 0 },
|
||||||
|
{ x: 100, y: 50 },
|
||||||
|
{ x: 50, y: 100 },
|
||||||
|
{ x: 0, y: 50 },
|
||||||
|
];
|
||||||
|
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||||
|
const send = vi.fn(async (_tabId, method, params) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||||
|
return backendNodeId === 20
|
||||||
|
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||||
|
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
const projection = await resolveFrameProjection(
|
||||||
|
cdp,
|
||||||
|
{
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "same-process-parent",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "oopif",
|
||||||
|
parentFrameId: "same-process-parent",
|
||||||
|
ownerBackendNodeId: 20,
|
||||||
|
target: { tabId: 4, sessionId: "oopif-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"oopif",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(projection?.edges).toEqual([
|
||||||
|
{
|
||||||
|
sourceViewport: { width: 200, height: 100 },
|
||||||
|
destinationQuad: [
|
||||||
|
{ x: 110, y: 70 },
|
||||||
|
{ x: 310, y: 70 },
|
||||||
|
{ x: 310, y: 170 },
|
||||||
|
{ x: 110, y: 170 },
|
||||||
|
],
|
||||||
|
destinationClips: [
|
||||||
|
[
|
||||||
|
{ x: 100, y: 50 },
|
||||||
|
{ x: 400, y: 50 },
|
||||||
|
{ x: 400, y: 250 },
|
||||||
|
{ x: 100, y: 250 },
|
||||||
|
],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(geometry).toMatchObject({
|
||||||
|
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||||
|
actionPoint: { x: 200, y: 150 },
|
||||||
|
targetActionPoint: { x: 100, y: 50 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||||
|
const getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph,
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveNodeGeometry(
|
||||||
|
cdp,
|
||||||
|
4,
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
},
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
),
|
||||||
|
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||||
|
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4 },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(geometry).toMatchObject({
|
||||||
|
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||||
|
actionPoint: { x: 200, y: 150 },
|
||||||
|
targetActionPoint: { x: 200, y: 150 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(),
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
}),
|
||||||
|
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { RefStore } from "@/session-manager/ref-store";
|
||||||
import type { RequestHelpParams } from "@/transport/types";
|
import type { RequestHelpParams } from "@/transport/types";
|
||||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||||
|
|
||||||
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||||
|
const refStore = new RefStore();
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === sessionId
|
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
|
||||||
: null,
|
|
||||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
return mgr;
|
return mgr;
|
||||||
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
|
|||||||
|
|
||||||
ac.abort();
|
ac.abort();
|
||||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
expect(cdpSend).toHaveBeenCalledWith(
|
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||||
5,
|
expect.anything(),
|
||||||
"DOM.querySelectorAll",
|
"DOM.removeAttribute",
|
||||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
expect.anything(),
|
||||||
);
|
|
||||||
expect(cdpSend).toHaveBeenCalledWith(
|
|
||||||
6,
|
|
||||||
"DOM.querySelectorAll",
|
|
||||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("tags ref targets via CDP and reports them matched", async () => {
|
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||||
|
const refStore = new RefStore();
|
||||||
|
refStore.set("e1", 42, { tabId: 5 });
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore: { resolve: () => 42 },
|
refStore,
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
|
|||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
cdp: {
|
cdp: {
|
||||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}),
|
||||||
} as unknown as RequestHelpDeps["cdp"],
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
|
|||||||
deps,
|
deps,
|
||||||
);
|
);
|
||||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
expect(sentMsg.selectors).toEqual([]);
|
||||||
|
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||||
expect(res).toMatchObject({
|
expect(res).toMatchObject({
|
||||||
outcome: "continued",
|
outcome: "continued",
|
||||||
tab_id: 5,
|
tab_id: 5,
|
||||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
resolved_targets: [
|
||||||
|
{
|
||||||
|
matched: true,
|
||||||
|
ref: "@e1",
|
||||||
|
},
|
||||||
|
],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||||
|
const mgr = {
|
||||||
|
get: (id: string) =>
|
||||||
|
id === "abcd"
|
||||||
|
? {
|
||||||
|
sessionId: "abcd",
|
||||||
|
agentWindowId: 99,
|
||||||
|
refStore: {
|
||||||
|
resolveEntry: () => ({
|
||||||
|
backendNodeId: 42,
|
||||||
|
tabId: 5,
|
||||||
|
frameId: "child",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
generation: 1,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
borrowedTabs: new Map(),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||||
|
} as unknown as SessionManager;
|
||||||
|
const send = vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root ${method}`);
|
||||||
|
});
|
||||||
|
const sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child ${method}`);
|
||||||
|
});
|
||||||
|
const deps = baseDeps({
|
||||||
|
cdp: {
|
||||||
|
send,
|
||||||
|
sendToTarget,
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 5 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 5, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
mgr,
|
||||||
|
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
|
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||||
|
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 5, sessionId: "child-session" },
|
||||||
|
"DOM.getContentQuads",
|
||||||
|
{ backendNodeId: 42 },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||||
const deps = baseDeps();
|
const deps = baseDeps();
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||||
|
const refStore = new RefStore();
|
||||||
|
refStore.set("e1", 42, { tabId: 4 });
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore: {
|
refStore,
|
||||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
|
||||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
|
||||||
},
|
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
return id;
|
return id;
|
||||||
}),
|
}),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||||
sent.push({ tabId, method, params });
|
sent.push({ tabId, method, params });
|
||||||
const h = handlers[method];
|
const h = handlers[method];
|
||||||
|
if (!h && method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return h(params);
|
return h(params);
|
||||||
};
|
};
|
||||||
@@ -164,6 +167,63 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Input.dispatchMouseEvent": () => ({}),
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getBoxModel": () => ({
|
||||||
|
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||||
|
targetCalls.push({ sessionId: target.sessionId, method });
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child CDP call ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handleClick(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e3" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||||
|
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||||
|
expect(targetCalls).toEqual([
|
||||||
|
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||||
|
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||||
|
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||||
|
]);
|
||||||
|
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||||
|
});
|
||||||
|
|
||||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||||
const order: string[] = [];
|
const order: string[] = [];
|
||||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||||
@@ -480,6 +540,21 @@ describe("handleHover", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function successfulFillScript(params: unknown) {
|
||||||
|
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
|
||||||
|
const args = script.arguments ?? [];
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value:
|
||||||
|
args.length === 2
|
||||||
|
? { before: "", expected: args[0].value }
|
||||||
|
: script.functionDeclaration.startsWith("function(expected)")
|
||||||
|
? { connected: true, matches: true, valueLength: String(args[0].value).length }
|
||||||
|
: "ready",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe("handleFill", () => {
|
describe("handleFill", () => {
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
@@ -526,7 +601,8 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
"Runtime.callFunctionOn": successfulFillScript,
|
||||||
|
"Runtime.releaseObject": () => ({}),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
@@ -540,13 +616,12 @@ describe("handleFill", () => {
|
|||||||
expect(res.used_ref).toBe("e1");
|
expect(res.used_ref).toBe("e1");
|
||||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||||
expect(insert?.params).toEqual({ text: "hello" });
|
expect(insert?.params).toEqual({ text: "hello" });
|
||||||
// clear_before defaults to true → callFunctionOn invoked once to
|
// Foreground replacement needs no extra caret-positioning round trip.
|
||||||
// clear, once to fire input/change after typing.
|
|
||||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
expect(callFns).toHaveLength(4);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clear_before=false skips the wipe call", async () => {
|
it("passes clear_before=false to preparation and verifies the result", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||||
@@ -558,12 +633,12 @@ describe("handleFill", () => {
|
|||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||||
"Runtime.callFunctionOn": (p) => {
|
"Runtime.callFunctionOn": (p) => {
|
||||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
|
||||||
// No "this.value = ''" clearing on a no-clear path; only the
|
if (args.length === 2) expect(args[1].value).toBe(false);
|
||||||
// post-input dispatchEvent.
|
return successfulFillScript(p);
|
||||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
|
||||||
return { result: { type: "undefined" } };
|
|
||||||
},
|
},
|
||||||
|
"Input.dispatchKeyEvent": () => ({}),
|
||||||
|
"Runtime.releaseObject": () => ({}),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
@@ -590,7 +665,8 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
"Runtime.callFunctionOn": successfulFillScript,
|
||||||
|
"Runtime.releaseObject": () => ({}),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
|
|||||||
@@ -0,0 +1,266 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import type { BrowserNavigationApi } from "../browser-navigation";
|
||||||
|
import { handleNavigate, handleReload } from "../navigation";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||||
|
const url = "https://example.test/recovered";
|
||||||
|
|
||||||
|
function event<T extends (...args: never[]) => void>() {
|
||||||
|
const listeners = new Set<T>();
|
||||||
|
return {
|
||||||
|
listeners,
|
||||||
|
addListener: (listener: T) => listeners.add(listener),
|
||||||
|
removeListener: (listener: T) => listeners.delete(listener),
|
||||||
|
fire: (...args: Parameters<T>) => {
|
||||||
|
for (const listener of [...listeners]) listener(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fixture() {
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => 4),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await manager.start("test");
|
||||||
|
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
|
||||||
|
const events = {
|
||||||
|
onCommitted: event<Listener>(),
|
||||||
|
onDOMContentLoaded: event<Listener>(),
|
||||||
|
onCompleted: event<Listener>(),
|
||||||
|
onErrorOccurred: event<Listener>(),
|
||||||
|
};
|
||||||
|
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
|
||||||
|
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (state.blocked) throw new Error(denied);
|
||||||
|
if (method === "Page.getFrameTree") {
|
||||||
|
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
|
||||||
|
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
|
||||||
|
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
|
||||||
|
frameId: "main",
|
||||||
|
loaderId: "new-loader",
|
||||||
|
name: "networkIdle",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
onEvent: (listener) => {
|
||||||
|
cdpEvents.addListener(listener);
|
||||||
|
return { dispose: () => cdpEvents.removeListener(listener) };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
|
||||||
|
const navigate = async () => {
|
||||||
|
state.blocked = state.destinationBlocked;
|
||||||
|
events.onCommitted.fire(main);
|
||||||
|
events.onDOMContentLoaded.fire(main);
|
||||||
|
events.onCompleted.fire(main);
|
||||||
|
};
|
||||||
|
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
|
||||||
|
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
|
||||||
|
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
|
||||||
|
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
|
||||||
|
const expectCleanedUp = () => {
|
||||||
|
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
|
||||||
|
expect(cdpEvents.listeners.size).toBe(0);
|
||||||
|
};
|
||||||
|
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("navigation after Chrome denies extension-frame access", () => {
|
||||||
|
it.each([
|
||||||
|
"load",
|
||||||
|
"commit",
|
||||||
|
"domcontentloaded",
|
||||||
|
"networkidle",
|
||||||
|
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
|
||||||
|
const f = await fixture();
|
||||||
|
const result = await handleNavigate(
|
||||||
|
f.manager,
|
||||||
|
{
|
||||||
|
session_id: "test",
|
||||||
|
url,
|
||||||
|
wait_until: waitUntil,
|
||||||
|
},
|
||||||
|
f.deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
|
||||||
|
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
|
||||||
|
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
|
||||||
|
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("recovers reload through tabs.reload and preserves hard reload", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
|
||||||
|
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
|
||||||
|
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
|
||||||
|
bypassCache: true,
|
||||||
|
});
|
||||||
|
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports the restriction when reload leaves the conflicting frame present", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.state.destinationBlocked = true;
|
||||||
|
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "cdp_extension_access_denied" },
|
||||||
|
});
|
||||||
|
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not fall back for unrelated debugger failures", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
|
||||||
|
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: "Another debugger is already attached",
|
||||||
|
});
|
||||||
|
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not replay navigation when a dispatched CDP command fails", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.state.blocked = false;
|
||||||
|
const send = f.send.getMockImplementation()!;
|
||||||
|
f.send.mockImplementation(async (tabId, method) => {
|
||||||
|
if (method === "Page.navigate") throw new Error(denied);
|
||||||
|
return send(tabId, method);
|
||||||
|
});
|
||||||
|
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "cdp_extension_access_denied" },
|
||||||
|
});
|
||||||
|
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the Agent Window guard in front of the recovery", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.deps.tabsApi.get.mockResolvedValue({
|
||||||
|
id: 4,
|
||||||
|
windowId: 200,
|
||||||
|
active: true,
|
||||||
|
url,
|
||||||
|
} as chrome.tabs.Tab);
|
||||||
|
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
|
||||||
|
expect(result).toMatchObject({ code: "permission_denied" });
|
||||||
|
expect(f.send).not.toHaveBeenCalled();
|
||||||
|
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores old-document, subframe, and other-tab completion events", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||||
|
let settled = false;
|
||||||
|
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
|
||||||
|
settled = true;
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
|
||||||
|
f.events.onCompleted.fire(f.main);
|
||||||
|
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
|
||||||
|
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
|
||||||
|
f.events.onCompleted.fire(f.main);
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
f.events.onCommitted.fire(f.main);
|
||||||
|
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
f.state.blocked = false;
|
||||||
|
f.events.onCompleted.fire(f.main);
|
||||||
|
expect(await work).toMatchObject({ reached: "load" });
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not infer network idle from a browser load event", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.state.emitIdle = false;
|
||||||
|
const result = await handleNavigate(
|
||||||
|
f.manager,
|
||||||
|
{
|
||||||
|
session_id: "test",
|
||||||
|
url,
|
||||||
|
wait_until: "networkidle",
|
||||||
|
timeout_ms: 10,
|
||||||
|
},
|
||||||
|
f.deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ reached: "timeout" });
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cleans up browser listeners on timeout", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||||
|
const result = await handleNavigate(
|
||||||
|
f.manager,
|
||||||
|
{ session_id: "test", url, timeout_ms: 10 },
|
||||||
|
f.deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ reached: "timeout" });
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not navigate an already-cancelled request", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const result = await handleNavigate(
|
||||||
|
f.manager,
|
||||||
|
{ session_id: "test", url },
|
||||||
|
{
|
||||||
|
...f.deps,
|
||||||
|
signal: AbortSignal.abort(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cleans up browser listeners when cancelled after dispatch", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const ac = new AbortController();
|
||||||
|
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||||
|
ac.abort();
|
||||||
|
});
|
||||||
|
const result = await handleNavigate(
|
||||||
|
f.manager,
|
||||||
|
{ session_id: "test", url },
|
||||||
|
{ ...f.deps, signal: ac.signal },
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports browser navigation errors and releases listeners", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||||
|
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
|
||||||
|
});
|
||||||
|
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||||
|
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
|
||||||
|
f.expectCleanedUp();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
return id;
|
return id;
|
||||||
}),
|
}),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
return id;
|
return id;
|
||||||
}),
|
}),
|
||||||
remove: vi.fn(async () => {}),
|
remove: vi.fn(async () => {}),
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
ensureActiveTab: vi.fn(async () => 1),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user