mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
74
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
388c1283bc | ||
|
|
b449be1544 | ||
|
|
f06a514bd2 | ||
|
|
1e53ee63ae | ||
|
|
120dc441bb | ||
|
|
710b1219f4 | ||
|
|
177ecd3ebf | ||
|
|
0e6b9c4542 | ||
|
|
506adbe708 | ||
|
|
984d017b0f | ||
|
|
5df93143c5 | ||
|
|
7163f2f9b0 | ||
|
|
e9f0584af3 | ||
|
|
2997f495db | ||
|
|
f96bd72c3c | ||
|
|
329c68054b | ||
|
|
5cd43553ed | ||
|
|
9ad33801c7 | ||
|
|
7aaa5a2d01 | ||
|
|
210b21b637 | ||
|
|
48f64ec0df | ||
|
|
c48a53269e | ||
|
|
3bbd6ab19f | ||
|
|
1a70d2200e | ||
|
|
554861d6b1 | ||
|
|
0d7f976335 | ||
|
|
d1d3eb6a94 | ||
|
|
38fd7d3f4c | ||
|
|
b5062ac19b | ||
|
|
41cb576256 | ||
|
|
f060a7c074 | ||
|
|
60b62d5420 | ||
|
|
70b851a3e8 | ||
|
|
da3534d066 | ||
|
|
a004291848 | ||
|
|
60fbf30e24 | ||
|
|
4b6cdde168 | ||
|
|
4aade4b94f | ||
|
|
616af1099f | ||
|
|
3656f0dd3c | ||
|
|
9742d83213 | ||
|
|
615e866608 | ||
|
|
c08fed5de3 | ||
|
|
887d1ed0b9 | ||
|
|
1a3d3b7de8 | ||
|
|
72834980d2 | ||
|
|
38b0debba9 | ||
|
|
4677149e5e | ||
|
|
8ea5477e11 | ||
|
|
17af1af65a | ||
|
|
6300eda0aa | ||
|
|
87cdcfe5ca | ||
|
|
21476c3884 | ||
|
|
c2a225377b | ||
|
|
e64f4cac9f | ||
|
|
368507b07d | ||
|
|
b750eed5d4 | ||
|
|
3d159c0254 | ||
|
|
524516cb56 | ||
|
|
f98198949a | ||
|
|
d05279f991 | ||
|
|
a4b6a61d29 | ||
|
|
cad0b5cbc9 | ||
|
|
2719e76438 | ||
|
|
dd9fc9edba | ||
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
8352c3c67a | ||
|
|
4eedd98c1d | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
7035e46024 |
@@ -325,3 +325,180 @@ jobs:
|
|||||||
|
|
||||||
echo "Chrome Web Store submission created successfully:"
|
echo "Chrome Web Store submission created successfully:"
|
||||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||||
|
|
||||||
|
publish-edge-addons:
|
||||||
|
needs: [resolve, build-extension]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: edge-addons
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Download build artifact
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: extension-zip
|
||||||
|
path: dist
|
||||||
|
|
||||||
|
- name: Validate Microsoft Edge Add-ons configuration
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||||
|
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||||
|
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "$API_KEY" ]; then
|
||||||
|
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$CLIENT_ID" ]; then
|
||||||
|
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$PRODUCT_ID" ]; then
|
||||||
|
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
||||||
|
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||||
|
if [ -z "$ARCHIVE" ]; then
|
||||||
|
echo "::error::No extension zip found in the downloaded artifact"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Upload and publish Microsoft Edge Add-ons extension
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||||
|
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||||
|
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||||
|
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
||||||
|
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
||||||
|
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
||||||
|
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
||||||
|
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
||||||
|
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
||||||
|
|
||||||
|
operation_id_from_headers() {
|
||||||
|
local headers="$1"
|
||||||
|
local location
|
||||||
|
location="$(awk '
|
||||||
|
tolower($1) == "location:" {
|
||||||
|
sub(/\r$/, "")
|
||||||
|
sub(/^[^:]*:[[:space:]]*/, "")
|
||||||
|
print
|
||||||
|
exit
|
||||||
|
}
|
||||||
|
' "$headers")"
|
||||||
|
location="${location%%\?*}"
|
||||||
|
printf '%s' "${location##*/}"
|
||||||
|
}
|
||||||
|
|
||||||
|
poll_operation() {
|
||||||
|
local operation_name="$1"
|
||||||
|
local status_url="$2"
|
||||||
|
local status="InProgress"
|
||||||
|
local http_status
|
||||||
|
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
sleep 5
|
||||||
|
http_status="$(curl --silent --show-error \
|
||||||
|
--output "$STATUS_RESPONSE" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
"$status_url")"
|
||||||
|
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
||||||
|
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
||||||
|
echo "$operation_name status check $attempt/30: $status"
|
||||||
|
case "$status" in
|
||||||
|
Succeeded)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
InProgress)
|
||||||
|
;;
|
||||||
|
Failed)
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$UPLOAD_RESPONSE" \
|
||||||
|
--dump-header "$UPLOAD_HEADERS" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
--header "Content-Type: application/zip" \
|
||||||
|
--data-binary "@$ARCHIVE" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
||||||
|
if [ "$HTTP_STATUS" != "202" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
||||||
|
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
||||||
|
cat "$UPLOAD_HEADERS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
poll_operation \
|
||||||
|
"package upload" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$PUBLISH_RESPONSE" \
|
||||||
|
--dump-header "$PUBLISH_HEADERS" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
--header "Content-Type: application/json" \
|
||||||
|
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
||||||
|
if [ "$HTTP_STATUS" != "202" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
||||||
|
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
||||||
|
cat "$PUBLISH_HEADERS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
poll_operation \
|
||||||
|
"submission" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
||||||
|
|
||||||
|
echo "Microsoft Edge Add-ons submission created successfully:"
|
||||||
|
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
||||||
|
|||||||
@@ -11,6 +11,10 @@ dist/
|
|||||||
# Logs
|
# Logs
|
||||||
*.log
|
*.log
|
||||||
|
|
||||||
|
# Local browser evaluation artifacts
|
||||||
|
evals/browser/results/
|
||||||
|
evals/browser/agents.local.json
|
||||||
|
|
||||||
# macOS
|
# macOS
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
|||||||
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk"
|
name = "bsk"
|
||||||
version = "0.1.10"
|
version = "0.1.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"bsk-protocol",
|
"bsk-protocol",
|
||||||
@@ -168,7 +168,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk-protocol"
|
name = "bsk-protocol"
|
||||||
version = "0.1.10"
|
version = "0.1.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"schemars",
|
"schemars",
|
||||||
"semver",
|
"semver",
|
||||||
|
|||||||
+1
-1
@@ -7,7 +7,7 @@ members = [
|
|||||||
default-members = ["crates/bsk-cli"]
|
default-members = ["crates/bsk-cli"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.1.10"
|
version = "0.1.11"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
|
|||||||
@@ -208,6 +208,7 @@ The repository is a Cargo + pnpm workspace:
|
|||||||
- `apps/extension` — browser extension
|
- `apps/extension` — browser extension
|
||||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
@@ -178,6 +178,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
|||||||
- `apps/extension` — 浏览器扩展
|
- `apps/extension` — 浏览器扩展
|
||||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
||||||
|
|
||||||
## 许可证
|
## 许可证
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@browser-skill/extension",
|
"name": "@browser-skill/extension",
|
||||||
"version": "0.1.6",
|
"version": "0.1.7",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
import { startRecordCapture } from "../record-capture";
|
||||||
|
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
runtime: {
|
runtime: {
|
||||||
sendMessage: vi.fn(() => Promise.resolve()),
|
sendMessage: vi.fn((message: { sequence?: number }) =>
|
||||||
|
Promise.resolve({ ok: true, sequence: message.sequence }),
|
||||||
|
),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -46,58 +48,6 @@ function click(el: Element): void {
|
|||||||
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleRecordContentMessage stop/cancel", () => {
|
|
||||||
it("ignores STOP when no recording is active", () => {
|
|
||||||
const dispose = vi.fn();
|
|
||||||
const onStop = vi.fn();
|
|
||||||
const setActiveRequestId = vi.fn();
|
|
||||||
const setCapture = vi.fn();
|
|
||||||
const sendResponse = vi.fn();
|
|
||||||
|
|
||||||
const needsAsync = handleRecordContentMessage(
|
|
||||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
|
||||||
{
|
|
||||||
activeRequestId: null,
|
|
||||||
capture: { dispose },
|
|
||||||
setActiveRequestId,
|
|
||||||
setCapture,
|
|
||||||
onStart: vi.fn(),
|
|
||||||
onStop,
|
|
||||||
},
|
|
||||||
sendResponse,
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(needsAsync).toBe(false);
|
|
||||||
expect(dispose).not.toHaveBeenCalled();
|
|
||||||
expect(onStop).not.toHaveBeenCalled();
|
|
||||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
|
||||||
expect(setCapture).not.toHaveBeenCalled();
|
|
||||||
expect(sendResponse).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("ignores STOP for a mismatched requestId", () => {
|
|
||||||
const dispose = vi.fn();
|
|
||||||
const onStop = vi.fn();
|
|
||||||
|
|
||||||
const needsAsync = handleRecordContentMessage(
|
|
||||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
|
||||||
{
|
|
||||||
activeRequestId: "rec-1",
|
|
||||||
capture: { dispose },
|
|
||||||
setActiveRequestId: vi.fn(),
|
|
||||||
setCapture: vi.fn(),
|
|
||||||
onStart: vi.fn(),
|
|
||||||
onStop,
|
|
||||||
},
|
|
||||||
vi.fn(),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(needsAsync).toBe(false);
|
|
||||||
expect(dispose).not.toHaveBeenCalled();
|
|
||||||
expect(onStop).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("record-capture semantic", () => {
|
describe("record-capture semantic", () => {
|
||||||
let steps: RecordStepPayload[];
|
let steps: RecordStepPayload[];
|
||||||
|
|
||||||
@@ -174,6 +124,19 @@ describe("record-capture semantic", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not emit page navigation from a child Document capture", () => {
|
||||||
|
const originalUrl = location.href;
|
||||||
|
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
|
||||||
|
captureNavigation: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
history.pushState({}, "", "#inside-frame");
|
||||||
|
expect(steps).toEqual([]);
|
||||||
|
|
||||||
|
capture.dispose();
|
||||||
|
history.replaceState({}, "", originalUrl);
|
||||||
|
});
|
||||||
|
|
||||||
it("does not record clicks on anonymous layout divs", () => {
|
it("does not record clicks on anonymous layout divs", () => {
|
||||||
document.body.innerHTML = `
|
document.body.innerHTML = `
|
||||||
<div id="chrome">page chrome</div>
|
<div id="chrome">page chrome</div>
|
||||||
@@ -228,6 +191,7 @@ describe("record-capture semantic", () => {
|
|||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { role: "button", name: "Open user navigation menu" },
|
target: { role: "button", name: "Open user navigation menu" },
|
||||||
|
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||||
});
|
});
|
||||||
expect(steps[1]).toMatchObject({
|
expect(steps[1]).toMatchObject({
|
||||||
op: "click",
|
op: "click",
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
|
||||||
|
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
|
||||||
|
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
|
||||||
|
import { RecordFrameAgent } from "../recording/frame-agent";
|
||||||
|
|
||||||
|
class PortListeners<T extends (...args: never[]) => unknown> {
|
||||||
|
readonly values = new Set<T>();
|
||||||
|
addListener = (listener: T) => this.values.add(listener);
|
||||||
|
removeListener = (listener: T) => this.values.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
function portHarness() {
|
||||||
|
const onMessage = new PortListeners<(message: unknown) => void>();
|
||||||
|
const onDisconnect = new PortListeners<() => void>();
|
||||||
|
const outbound: RecordFramePortMessage[] = [];
|
||||||
|
const port = {
|
||||||
|
name: "bsk-record-frame",
|
||||||
|
onMessage,
|
||||||
|
onDisconnect,
|
||||||
|
postMessage(message: RecordFramePortMessage) {
|
||||||
|
outbound.push(message);
|
||||||
|
if (message.type === "ready") {
|
||||||
|
queueMicrotask(() => {
|
||||||
|
for (const listener of onMessage.values) {
|
||||||
|
listener({
|
||||||
|
type: "ready_ack",
|
||||||
|
requestId: message.requestId,
|
||||||
|
producerId: message.producerId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
disconnect: vi.fn(),
|
||||||
|
} as unknown as chrome.runtime.Port;
|
||||||
|
return {
|
||||||
|
port,
|
||||||
|
outbound,
|
||||||
|
receive(message: RecordFramePortMessage) {
|
||||||
|
for (const listener of onMessage.values) listener(message);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("RecordFrameAgent", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
document.body.replaceChildren();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||||
|
const harness = portHarness();
|
||||||
|
const sendMessage = vi
|
||||||
|
.fn<(message: { sequence: number }) => Promise<unknown>>()
|
||||||
|
.mockRejectedValueOnce(new Error("offline"))
|
||||||
|
.mockRejectedValueOnce(new Error("still offline"))
|
||||||
|
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: {
|
||||||
|
connect: vi.fn(() => harness.port),
|
||||||
|
sendMessage,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
|
||||||
|
const agent = new RecordFrameAgent();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
|
||||||
|
).resolves.toEqual({ ok: true });
|
||||||
|
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||||
|
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "final value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
|
||||||
|
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(harness.outbound).toContainEqual({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: "rec-1",
|
||||||
|
commandId: "stop-1",
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||||
|
|
||||||
|
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(harness.outbound).toContainEqual({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: "rec-1",
|
||||||
|
commandId: "stop-2",
|
||||||
|
ok: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(sendMessage).toHaveBeenCalledTimes(3);
|
||||||
|
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { RecordStepMessage } from "@/lib/record-bridge";
|
||||||
|
import { RecordStepDelivery } from "../record-step-delivery";
|
||||||
|
|
||||||
|
describe("RecordStepDelivery", () => {
|
||||||
|
it("retries from the first unacknowledged sequence before sending later steps", async () => {
|
||||||
|
const sent: RecordStepMessage[] = [];
|
||||||
|
const send = vi.fn(async (message: RecordStepMessage) => {
|
||||||
|
sent.push(message);
|
||||||
|
if (sent.length === 1) throw new Error("service worker unavailable");
|
||||||
|
return { ok: true, sequence: message.sequence };
|
||||||
|
});
|
||||||
|
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
|
||||||
|
|
||||||
|
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
|
||||||
|
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
|
||||||
|
|
||||||
|
await expect(delivery.flush()).resolves.toBe(true);
|
||||||
|
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
|
||||||
|
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps unacknowledged steps pending for a later flush", async () => {
|
||||||
|
const send = vi
|
||||||
|
.fn<(message: RecordStepMessage) => Promise<unknown>>()
|
||||||
|
.mockRejectedValueOnce(new Error("offline"))
|
||||||
|
.mockRejectedValueOnce(new Error("still offline"))
|
||||||
|
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||||
|
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
|
||||||
|
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
|
||||||
|
|
||||||
|
await expect(delivery.flush()).resolves.toBe(false);
|
||||||
|
await expect(delivery.flush()).resolves.toBe(true);
|
||||||
|
expect(send).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import {
|
import {
|
||||||
|
type CaptureTargetDescriptor,
|
||||||
describeEventTarget,
|
describeEventTarget,
|
||||||
describeTarget,
|
describeTarget,
|
||||||
resolveClickableElement,
|
resolveClickableElement,
|
||||||
resolveHoverElement,
|
resolveHoverElement,
|
||||||
type TargetDescriptor,
|
|
||||||
} from "@/lib/describe-target";
|
} from "@/lib/describe-target";
|
||||||
import {
|
import {
|
||||||
evaluateHoverTrigger,
|
evaluateHoverTrigger,
|
||||||
@@ -12,22 +12,8 @@ import {
|
|||||||
hasDirectHoverInteractiveSignal,
|
hasDirectHoverInteractiveSignal,
|
||||||
hasStrongHoverExpansionSignal,
|
hasStrongHoverExpansionSignal,
|
||||||
} from "@/lib/hover-trigger-policy";
|
} from "@/lib/hover-trigger-policy";
|
||||||
import {
|
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||||
isRecordCancelMessage,
|
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||||
isRecordStartMessage,
|
|
||||||
isRecordStopMessage,
|
|
||||||
RECORD_CANCEL,
|
|
||||||
RECORD_START,
|
|
||||||
RECORD_STEP,
|
|
||||||
RECORD_STOP,
|
|
||||||
type RecordCancelMessage,
|
|
||||||
type RecordStartAck,
|
|
||||||
type RecordStartMessage,
|
|
||||||
type RecordStepPayload,
|
|
||||||
type RecordStopAck,
|
|
||||||
type RecordStopMessage,
|
|
||||||
} from "@/lib/record-bridge";
|
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
|
||||||
import {
|
import {
|
||||||
closestHoverSurfaceCandidate,
|
closestHoverSurfaceCandidate,
|
||||||
collectHoverSurfaceStates,
|
collectHoverSurfaceStates,
|
||||||
@@ -37,42 +23,21 @@ import {
|
|||||||
isLikelyHoverSurfaceOwner,
|
isLikelyHoverSurfaceOwner,
|
||||||
} from "./record-hover-surface";
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
|
||||||
const failedStepDeliveries = new Set<string>();
|
|
||||||
const knownRecordRequests = new Set<string>();
|
|
||||||
|
|
||||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
|
||||||
const payload = { type: RECORD_STEP, requestId, step };
|
|
||||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
|
||||||
() => true,
|
|
||||||
() => {
|
|
||||||
failedStepDeliveries.add(requestId);
|
|
||||||
return false;
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
|
||||||
sends.add(pending);
|
|
||||||
pendingStepSends.set(requestId, sends);
|
|
||||||
void pending.then(() => {
|
|
||||||
sends.delete(pending);
|
|
||||||
if (sends.size === 0) pendingStepSends.delete(requestId);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RecordCaptureController {
|
export interface RecordCaptureController {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface FillSession {
|
interface FillSession {
|
||||||
element: FillableElement;
|
element: FillableElement;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
baselineValue: string;
|
baselineValue: string;
|
||||||
lastValue: string;
|
lastValue: string;
|
||||||
|
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
element: Element;
|
element: Element;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
recordedAt: number;
|
recordedAt: number;
|
||||||
score: number;
|
score: number;
|
||||||
eligible: boolean;
|
eligible: boolean;
|
||||||
@@ -157,6 +122,24 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return {
|
||||||
|
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function geometryForEventTarget(
|
||||||
|
target: EventTarget | null,
|
||||||
|
): RecordStepPayload["geometry"] | undefined {
|
||||||
|
if (!(target instanceof Element)) return undefined;
|
||||||
|
const clickable = target.closest(
|
||||||
|
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||||
|
);
|
||||||
|
return captureGeometry(clickable ?? target);
|
||||||
|
}
|
||||||
|
|
||||||
function fillableValue(el: FillableElement): string {
|
function fillableValue(el: FillableElement): string {
|
||||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||||
return el.value;
|
return el.value;
|
||||||
@@ -232,7 +215,10 @@ function collectHoverTriggerLabelText(root: Element): string {
|
|||||||
return normalizeLabelText(text);
|
return normalizeLabelText(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
function compactHoverTargetName(
|
||||||
|
el: Element,
|
||||||
|
desc: CaptureTargetDescriptor,
|
||||||
|
): CaptureTargetDescriptor {
|
||||||
if (!desc.name) return desc;
|
if (!desc.name) return desc;
|
||||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
const compactName = collectHoverTriggerLabelText(el);
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
@@ -250,7 +236,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
return !target.role && !target.name && target.tag === "div";
|
return !target.role && !target.name && target.tag === "div";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,9 +247,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
|||||||
|
|
||||||
function normalizeHoverTarget(
|
function normalizeHoverTarget(
|
||||||
el: Element,
|
el: Element,
|
||||||
desc: TargetDescriptor,
|
desc: CaptureTargetDescriptor,
|
||||||
decision: HoverTriggerDecision,
|
decision: HoverTriggerDecision,
|
||||||
): TargetDescriptor {
|
): CaptureTargetDescriptor {
|
||||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
return { ...desc, name: "image" };
|
return { ...desc, name: "image" };
|
||||||
}
|
}
|
||||||
@@ -277,7 +263,7 @@ function normalizeHoverTarget(
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||||
if (!(el instanceof HTMLElement)) return null;
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
const style = hoverTriggerStyle(el);
|
const style = hoverTriggerStyle(el);
|
||||||
return {
|
return {
|
||||||
@@ -373,6 +359,7 @@ function scheduleInputCompletionCommit(
|
|||||||
export function startRecordCapture(
|
export function startRecordCapture(
|
||||||
_requestId: string,
|
_requestId: string,
|
||||||
sendStep: (step: RecordStepPayload) => void,
|
sendStep: (step: RecordStepPayload) => void,
|
||||||
|
options: { captureNavigation?: boolean } = {},
|
||||||
): RecordCaptureController {
|
): RecordCaptureController {
|
||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
@@ -411,14 +398,17 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: session.target,
|
target: session.target,
|
||||||
|
geometry: captureGeometry(session.element),
|
||||||
value,
|
value,
|
||||||
|
commit: session.pendingCommit ?? "blur",
|
||||||
...(isPassword ? { redacted: true } : {}),
|
...(isPassword ? { redacted: true } : {}),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const commitFillSession = () => {
|
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||||
if (!fillSession || composing) return;
|
if (!fillSession || composing) return;
|
||||||
const session = fillSession;
|
const session = fillSession;
|
||||||
|
session.pendingCommit = commit;
|
||||||
fillSession = null;
|
fillSession = null;
|
||||||
emitFill(session);
|
emitFill(session);
|
||||||
committedValues.set(session.element, session.lastValue);
|
committedValues.set(session.element, session.lastValue);
|
||||||
@@ -624,6 +614,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: hover.target,
|
target: hover.target,
|
||||||
|
geometry: captureGeometry(hover.element),
|
||||||
});
|
});
|
||||||
emittedHoverElements.add(hover.element);
|
emittedHoverElements.add(hover.element);
|
||||||
};
|
};
|
||||||
@@ -678,6 +669,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "click",
|
op: "click",
|
||||||
target,
|
target,
|
||||||
|
geometry: geometryForEventTarget(eventTarget(event)),
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
@@ -779,7 +771,7 @@ export function startRecordCapture(
|
|||||||
scheduleInputCompletionCommit(
|
scheduleInputCompletionCommit(
|
||||||
sessionElement,
|
sessionElement,
|
||||||
syncFillSessionValue,
|
syncFillSessionValue,
|
||||||
commitFillSession,
|
() => commitFillSession("suggestion"),
|
||||||
(el) => fillSession?.element === el,
|
(el) => fillSession?.element === el,
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
@@ -839,6 +831,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "select",
|
op: "select",
|
||||||
target: desc,
|
target: desc,
|
||||||
|
geometry: captureGeometry(target),
|
||||||
values,
|
values,
|
||||||
labels,
|
labels,
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
@@ -879,7 +872,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
commitFillSession();
|
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
@@ -887,7 +880,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
...(desc ? { target: desc } : {}),
|
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||||
...(modifiers.length ? { modifiers } : {}),
|
...(modifiers.length ? { modifiers } : {}),
|
||||||
expects_navigation: event.key === "Enter",
|
expects_navigation: event.key === "Enter",
|
||||||
});
|
});
|
||||||
@@ -903,23 +896,33 @@ export function startRecordCapture(
|
|||||||
document.addEventListener("change", onChange, true);
|
document.addEventListener("change", onChange, true);
|
||||||
document.addEventListener("keydown", onKeyDown, true);
|
document.addEventListener("keydown", onKeyDown, true);
|
||||||
|
|
||||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
const captureNavigation = options.captureNavigation ?? true;
|
||||||
urlObserver.observe(document, { subtree: true, childList: true });
|
const urlObserver = captureNavigation
|
||||||
|
? new MutationObserver(() => emitNavigateIfChanged())
|
||||||
|
: null;
|
||||||
|
urlObserver?.observe(document, { subtree: true, childList: true });
|
||||||
const onUrlEvent = () => emitNavigateIfChanged();
|
const onUrlEvent = () => emitNavigateIfChanged();
|
||||||
window.addEventListener("hashchange", onUrlEvent);
|
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||||
window.addEventListener("popstate", onUrlEvent);
|
// the `commit` argument and stamp it onto the recorded fill step.
|
||||||
window.addEventListener("pagehide", commitFillSession);
|
const onPageHide = () => commitFillSession();
|
||||||
|
if (captureNavigation) {
|
||||||
|
window.addEventListener("hashchange", onUrlEvent);
|
||||||
|
window.addEventListener("popstate", onUrlEvent);
|
||||||
|
}
|
||||||
|
window.addEventListener("pagehide", onPageHide);
|
||||||
|
|
||||||
const originalPushState = history.pushState;
|
const originalPushState = history.pushState;
|
||||||
const originalReplaceState = history.replaceState;
|
const originalReplaceState = history.replaceState;
|
||||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
if (captureNavigation) {
|
||||||
originalPushState.apply(this, args);
|
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
originalPushState.apply(this, args);
|
||||||
};
|
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
};
|
||||||
originalReplaceState.apply(this, args);
|
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
originalReplaceState.apply(this, args);
|
||||||
};
|
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
dispose() {
|
dispose() {
|
||||||
@@ -933,93 +936,16 @@ export function startRecordCapture(
|
|||||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||||
document.removeEventListener("change", onChange, true);
|
document.removeEventListener("change", onChange, true);
|
||||||
document.removeEventListener("keydown", onKeyDown, true);
|
document.removeEventListener("keydown", onKeyDown, true);
|
||||||
urlObserver.disconnect();
|
urlObserver?.disconnect();
|
||||||
window.removeEventListener("hashchange", onUrlEvent);
|
if (captureNavigation) {
|
||||||
window.removeEventListener("popstate", onUrlEvent);
|
window.removeEventListener("hashchange", onUrlEvent);
|
||||||
window.removeEventListener("pagehide", commitFillSession);
|
window.removeEventListener("popstate", onUrlEvent);
|
||||||
history.pushState = originalPushState;
|
}
|
||||||
history.replaceState = originalReplaceState;
|
window.removeEventListener("pagehide", onPageHide);
|
||||||
|
if (captureNavigation) {
|
||||||
|
history.pushState = originalPushState;
|
||||||
|
history.replaceState = originalReplaceState;
|
||||||
|
}
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
|
||||||
|
|
||||||
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
|
||||||
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function handleRecordContentMessage(
|
|
||||||
message: RecordContentMessage,
|
|
||||||
state: {
|
|
||||||
activeRequestId: string | null;
|
|
||||||
capture: RecordCaptureController | null;
|
|
||||||
setActiveRequestId(id: string | null): void;
|
|
||||||
setCapture(capture: RecordCaptureController | null): void;
|
|
||||||
onStart(requestId: string, startedAtMs?: number): void;
|
|
||||||
onStop(): void;
|
|
||||||
},
|
|
||||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
|
||||||
): boolean {
|
|
||||||
if (isRecordStartMessage(message)) {
|
|
||||||
if (!knownRecordRequests.has(message.requestId)) {
|
|
||||||
knownRecordRequests.add(message.requestId);
|
|
||||||
failedStepDeliveries.delete(message.requestId);
|
|
||||||
}
|
|
||||||
state.capture?.dispose();
|
|
||||||
state.setCapture(
|
|
||||||
startRecordCapture(message.requestId, (step) => {
|
|
||||||
sendRecordStep(message.requestId, step);
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
state.setActiveRequestId(message.requestId);
|
|
||||||
state.onStart(message.requestId, message.startedAtMs);
|
|
||||||
sendResponse?.({ ok: true });
|
|
||||||
return sendResponse !== undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
|
||||||
// Require an active recording that matches this requestId — otherwise a
|
|
||||||
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
|
||||||
// clear overlay state even though nothing was capturing.
|
|
||||||
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
state.capture?.dispose();
|
|
||||||
state.setCapture(null);
|
|
||||||
const finishStop = () => {
|
|
||||||
state.onStop();
|
|
||||||
state.setActiveRequestId(null);
|
|
||||||
};
|
|
||||||
if (isRecordStopMessage(message) && sendResponse) {
|
|
||||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
|
||||||
void Promise.all(pending).then((delivered) => {
|
|
||||||
finishStop();
|
|
||||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
|
||||||
if (succeeded) {
|
|
||||||
failedStepDeliveries.delete(message.requestId);
|
|
||||||
knownRecordRequests.delete(message.requestId);
|
|
||||||
}
|
|
||||||
sendResponse(
|
|
||||||
succeeded
|
|
||||||
? { ok: true }
|
|
||||||
: {
|
|
||||||
ok: false,
|
|
||||||
error: "failed to deliver one or more recorded steps",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (isRecordCancelMessage(message)) {
|
|
||||||
failedStepDeliveries.delete(message.requestId);
|
|
||||||
knownRecordRequests.delete(message.requestId);
|
|
||||||
}
|
|
||||||
finishStop();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import {
|
||||||
|
isAcceptedRecordStepAck,
|
||||||
|
RECORD_STEP,
|
||||||
|
type RecordStepMessage,
|
||||||
|
type RecordStepPayload,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
|
||||||
|
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
|
||||||
|
|
||||||
|
export class RecordStepDelivery {
|
||||||
|
readonly #requestId: string;
|
||||||
|
readonly #producerId: string;
|
||||||
|
readonly #send: SendRecordStepMessage;
|
||||||
|
readonly #pending = new Map<number, RecordStepPayload>();
|
||||||
|
#nextSequence = 1;
|
||||||
|
#tail = Promise.resolve();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
requestId: string,
|
||||||
|
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||||
|
producerId: string = crypto.randomUUID(),
|
||||||
|
) {
|
||||||
|
this.#requestId = requestId;
|
||||||
|
this.#producerId = producerId;
|
||||||
|
this.#send = send;
|
||||||
|
}
|
||||||
|
|
||||||
|
enqueue(step: RecordStepPayload): void {
|
||||||
|
const sequence = this.#nextSequence;
|
||||||
|
this.#nextSequence += 1;
|
||||||
|
this.#pending.set(sequence, step);
|
||||||
|
void this.#schedule();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<boolean> {
|
||||||
|
await this.#tail;
|
||||||
|
await this.#schedule();
|
||||||
|
return this.#pending.size === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#schedule(): Promise<void> {
|
||||||
|
const drain = () => this.#drain();
|
||||||
|
this.#tail = this.#tail.then(drain, drain);
|
||||||
|
return this.#tail;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #drain(): Promise<void> {
|
||||||
|
for (const [sequence, step] of this.#pending) {
|
||||||
|
const message: RecordStepMessage = {
|
||||||
|
type: RECORD_STEP,
|
||||||
|
requestId: this.#requestId,
|
||||||
|
producerId: this.#producerId,
|
||||||
|
sequence,
|
||||||
|
step,
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const response = await this.#send(message);
|
||||||
|
if (!isAcceptedRecordStepAck(response, sequence)) return;
|
||||||
|
this.#pending.delete(sequence);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
import {
|
||||||
|
markRecordingDocument,
|
||||||
|
type RecordingDocumentMarker,
|
||||||
|
waitForDocumentElement,
|
||||||
|
} from "@/lib/recording/document-marker";
|
||||||
|
import {
|
||||||
|
isRecordFrameStartMessage,
|
||||||
|
RECORD_FRAME_PORT,
|
||||||
|
RECORD_FRAME_QUERY,
|
||||||
|
RECORD_FRAME_START,
|
||||||
|
type RecordFramePortMessage,
|
||||||
|
type RecordFrameQueryResponse,
|
||||||
|
type RecordFrameStartMessage,
|
||||||
|
} from "@/lib/recording/frame-bridge";
|
||||||
|
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
|
||||||
|
import { RecordStepDelivery } from "../record-step-delivery";
|
||||||
|
|
||||||
|
interface ActiveFrameRecording {
|
||||||
|
requestId: string;
|
||||||
|
producerId: string;
|
||||||
|
port: chrome.runtime.Port;
|
||||||
|
delivery: RecordStepDelivery;
|
||||||
|
capture: RecordCaptureController;
|
||||||
|
marker: RecordingDocumentMarker;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
|
||||||
|
|
||||||
|
export class RecordFrameAgent {
|
||||||
|
#active: ActiveFrameRecording | null = null;
|
||||||
|
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
|
||||||
|
#disposed = false;
|
||||||
|
|
||||||
|
async start(
|
||||||
|
message: RecordFrameStartMessage,
|
||||||
|
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||||
|
if (this.#active?.requestId === message.requestId) return { ok: true };
|
||||||
|
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||||
|
if (this.#startPromise) {
|
||||||
|
await this.#startPromise;
|
||||||
|
return this.start(message);
|
||||||
|
}
|
||||||
|
if (this.#active) this.cancel(this.#active.requestId);
|
||||||
|
const starting = this.#start(message);
|
||||||
|
this.#startPromise = starting;
|
||||||
|
try {
|
||||||
|
return await starting;
|
||||||
|
} finally {
|
||||||
|
if (this.#startPromise === starting) this.#startPromise = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(requestId: string): void {
|
||||||
|
const active = this.#active;
|
||||||
|
if (!active || active.requestId !== requestId) return;
|
||||||
|
active.capture.dispose();
|
||||||
|
active.marker.restore();
|
||||||
|
active.port.disconnect();
|
||||||
|
this.#active = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
dispose(): void {
|
||||||
|
this.#disposed = true;
|
||||||
|
if (this.#active) this.cancel(this.#active.requestId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async #start(
|
||||||
|
message: RecordFrameStartMessage,
|
||||||
|
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||||
|
const producerId = crypto.randomUUID();
|
||||||
|
const root = await waitForDocumentElement();
|
||||||
|
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||||
|
const marker = markRecordingDocument(producerId, root);
|
||||||
|
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
|
||||||
|
try {
|
||||||
|
await registerPort(port, message.requestId, producerId);
|
||||||
|
} catch {
|
||||||
|
marker.restore();
|
||||||
|
port.disconnect();
|
||||||
|
return { ok: false, error: "failed to register recording document" };
|
||||||
|
}
|
||||||
|
if (this.#disposed) {
|
||||||
|
marker.restore();
|
||||||
|
port.disconnect();
|
||||||
|
return { ok: false, error: "recording document is disposed" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
|
||||||
|
const capture = startRecordCapture(
|
||||||
|
message.requestId,
|
||||||
|
(step) => {
|
||||||
|
marker.ensure();
|
||||||
|
delivery.enqueue(step);
|
||||||
|
},
|
||||||
|
{ captureNavigation: window.top === window },
|
||||||
|
);
|
||||||
|
const active: ActiveFrameRecording = {
|
||||||
|
requestId: message.requestId,
|
||||||
|
producerId,
|
||||||
|
port,
|
||||||
|
delivery,
|
||||||
|
capture,
|
||||||
|
marker,
|
||||||
|
};
|
||||||
|
this.#active = active;
|
||||||
|
port.onMessage.addListener((raw: unknown) => {
|
||||||
|
const command = raw as Partial<RecordFramePortMessage>;
|
||||||
|
if (command.type === "stop" && command.requestId === active.requestId) {
|
||||||
|
void this.#stop(active, command.commandId);
|
||||||
|
} else if (command.type === "cancel" && command.requestId === active.requestId) {
|
||||||
|
this.cancel(active.requestId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
port.onDisconnect.addListener(() => {
|
||||||
|
if (this.#active !== active) return;
|
||||||
|
active.capture.dispose();
|
||||||
|
active.marker.restore();
|
||||||
|
this.#active = null;
|
||||||
|
});
|
||||||
|
return { ok: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
|
||||||
|
if (!commandId || this.#active !== active) return;
|
||||||
|
active.capture.dispose();
|
||||||
|
const succeeded = await active.delivery.flush();
|
||||||
|
try {
|
||||||
|
active.port.postMessage({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: active.requestId,
|
||||||
|
commandId,
|
||||||
|
ok: succeeded,
|
||||||
|
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!succeeded || this.#active !== active) return;
|
||||||
|
active.marker.restore();
|
||||||
|
this.#active = null;
|
||||||
|
active.port.disconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function registerPort(
|
||||||
|
port: chrome.runtime.Port,
|
||||||
|
requestId: string,
|
||||||
|
producerId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
reject(new Error("recording document registration timed out"));
|
||||||
|
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
|
||||||
|
const cleanup = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
port.onMessage.removeListener(onMessage);
|
||||||
|
port.onDisconnect.removeListener(onDisconnect);
|
||||||
|
};
|
||||||
|
const onMessage = (raw: unknown) => {
|
||||||
|
const message = raw as Partial<RecordFramePortMessage>;
|
||||||
|
if (
|
||||||
|
message.type !== "ready_ack" ||
|
||||||
|
message.requestId !== requestId ||
|
||||||
|
message.producerId !== producerId
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const onDisconnect = () => {
|
||||||
|
cleanup();
|
||||||
|
reject(new Error("recording document port disconnected"));
|
||||||
|
};
|
||||||
|
port.onMessage.addListener(onMessage);
|
||||||
|
port.onDisconnect.addListener(onDisconnect);
|
||||||
|
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function attachRecordFrameAgent(): () => void {
|
||||||
|
const agent = new RecordFrameAgent();
|
||||||
|
const onMessage = (
|
||||||
|
message: unknown,
|
||||||
|
_sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
|
||||||
|
) => {
|
||||||
|
if (!isRecordFrameStartMessage(message)) return false;
|
||||||
|
void agent.start(message).then(sendResponse);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
chrome.runtime.onMessage.addListener(onMessage);
|
||||||
|
void chrome.runtime
|
||||||
|
.sendMessage({ type: RECORD_FRAME_QUERY })
|
||||||
|
.then((response: RecordFrameQueryResponse | undefined) => {
|
||||||
|
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
|
||||||
|
return agent.start({
|
||||||
|
type: RECORD_FRAME_START,
|
||||||
|
requestId: response.requestId,
|
||||||
|
startedAtMs: response.startedAtMs,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch(() => {});
|
||||||
|
return () => {
|
||||||
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
|
agent.dispose();
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
import { ConnectionController } from "@/lib/connection-controller";
|
import { ConnectionController } from "@/lib/connection-controller";
|
||||||
|
import { DiagnosticLogger } from "@/lib/diagnostics";
|
||||||
import { startHeartbeat } from "@/lib/heartbeat";
|
import { startHeartbeat } from "@/lib/heartbeat";
|
||||||
import {
|
import {
|
||||||
getConnectionEnabled,
|
getConnectionEnabled,
|
||||||
@@ -21,6 +22,7 @@ import {
|
|||||||
type OverlayMode,
|
type OverlayMode,
|
||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||||
|
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||||
@@ -38,15 +40,27 @@ import {
|
|||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
attachRecordQueryListener,
|
attachRecordQueryListener,
|
||||||
attachRecordStepListener,
|
attachRecordStepListener,
|
||||||
|
type RecordRuntimeDeps,
|
||||||
} from "@/tools/record";
|
} from "@/tools/record";
|
||||||
import { detectBrowserMeta } from "@/transport/handshake";
|
import { detectBrowserMeta } from "@/transport/handshake";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
import { WSTransport } from "@/transport/ws-transport";
|
import { WSTransport } from "@/transport/ws-transport";
|
||||||
|
|
||||||
export default defineBackground(() => {
|
export default defineBackground(() => {
|
||||||
const controller = new ConnectionController();
|
const diagnostics = new DiagnosticLogger();
|
||||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
diagnostics.record("extension.worker.started", {
|
||||||
const sessions = new SessionManager();
|
diagnostic_build: "lumi-session-loss-v2",
|
||||||
|
extension_version: __BSK_EXT_VERSION__,
|
||||||
|
extension_runtime_id: chrome.runtime.id,
|
||||||
|
daemon_ws_url: __BSK_DAEMON_WS_URL__,
|
||||||
|
});
|
||||||
|
const controller = new ConnectionController(diagnostics.sink);
|
||||||
|
const transport = new WSTransport({
|
||||||
|
url: __BSK_DAEMON_WS_URL__,
|
||||||
|
diagnostics: diagnostics.sink,
|
||||||
|
});
|
||||||
|
diagnostics.bindTransport((frame) => transport.send(frame));
|
||||||
|
const sessions = new SessionManager({ diagnostics: diagnostics.sink });
|
||||||
const cdp = new ChromiumCdp();
|
const cdp = new ChromiumCdp();
|
||||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||||
let overlayGeneration = 0;
|
let overlayGeneration = 0;
|
||||||
@@ -107,6 +121,9 @@ export default defineBackground(() => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function onOverlaySessionStateChanged(): void {
|
function onOverlaySessionStateChanged(): void {
|
||||||
|
diagnostics.record("background.sessions.changed", {
|
||||||
|
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||||
for (const sessionId of controlModes.keys()) {
|
for (const sessionId of controlModes.keys()) {
|
||||||
@@ -144,11 +161,35 @@ export default defineBackground(() => {
|
|||||||
onSessionsChanged: () => {
|
onSessionsChanged: () => {
|
||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
},
|
},
|
||||||
|
diagnostics: diagnostics.sink,
|
||||||
});
|
});
|
||||||
|
const recordDeps = {
|
||||||
|
tabsApi: chrome.tabs,
|
||||||
|
cdp,
|
||||||
|
frameCoordinator: recordFrameCoordinator,
|
||||||
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
},
|
||||||
|
} satisfies RecordRuntimeDeps;
|
||||||
|
recordFrameCoordinator.attach();
|
||||||
|
attachRecordStepListener(recordDeps);
|
||||||
|
attachRecordFinishListener(recordDeps);
|
||||||
|
attachRecordQueryListener(recordDeps);
|
||||||
|
|
||||||
const dispatcher = new ToolDispatcher({
|
const dispatcher = new ToolDispatcher({
|
||||||
transport,
|
transport,
|
||||||
sessions,
|
sessions,
|
||||||
cdp,
|
cdp,
|
||||||
|
recording: recordDeps,
|
||||||
onSessionsChanged: onOverlaySessionStateChanged,
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
onBrowserControlResumed,
|
onBrowserControlResumed,
|
||||||
approveBorrow: (ctx) =>
|
approveBorrow: (ctx) =>
|
||||||
@@ -168,29 +209,9 @@ export default defineBackground(() => {
|
|||||||
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
|
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
|
||||||
body: "",
|
body: "",
|
||||||
}),
|
}),
|
||||||
|
diagnostics: diagnostics.sink,
|
||||||
});
|
});
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
const recordDeps = {
|
|
||||||
tabsApi: chrome.tabs,
|
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
|
||||||
try {
|
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
// Message listeners stay up (cheap; fire only for record message types).
|
|
||||||
// Tab / webNavigation observation attaches lazily while a recording is
|
|
||||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
|
||||||
attachRecordStepListener();
|
|
||||||
attachRecordFinishListener(recordDeps);
|
|
||||||
attachRecordQueryListener(recordDeps);
|
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
attachBorrowNotificationClickHandler({
|
attachBorrowNotificationClickHandler({
|
||||||
onClicked: chrome.notifications.onClicked,
|
onClicked: chrome.notifications.onClicked,
|
||||||
@@ -221,6 +242,7 @@ export default defineBackground(() => {
|
|||||||
transport,
|
transport,
|
||||||
cdp,
|
cdp,
|
||||||
onSessionsChanged: onOverlaySessionStateChanged,
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
|
diagnostics: diagnostics.sink,
|
||||||
});
|
});
|
||||||
|
|
||||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||||
@@ -232,6 +254,7 @@ export default defineBackground(() => {
|
|||||||
startKeepalive({
|
startKeepalive({
|
||||||
transport,
|
transport,
|
||||||
shouldConnect: () => controller.isConnectionEnabled,
|
shouldConnect: () => controller.isConnectionEnabled,
|
||||||
|
diagnostics: diagnostics.sink,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
||||||
@@ -245,6 +268,19 @@ export default defineBackground(() => {
|
|||||||
controller.subscribe((snap) =>
|
controller.subscribe((snap) =>
|
||||||
cb(snap.state === "connected" || snap.state === "version_skew"),
|
cb(snap.state === "connected" || snap.state === "version_skew"),
|
||||||
),
|
),
|
||||||
|
diagnostics: diagnostics.sink,
|
||||||
|
});
|
||||||
|
|
||||||
|
controller.subscribe((snapshot) => {
|
||||||
|
const ready = snapshot.state === "connected" || snapshot.state === "version_skew";
|
||||||
|
diagnostics.setTransportReady(ready);
|
||||||
|
diagnostics.record("background.controller.snapshot", {
|
||||||
|
state: snapshot.state,
|
||||||
|
connection_enabled: snapshot.connectionEnabled,
|
||||||
|
instance_id: snapshot.instanceId,
|
||||||
|
last_error: snapshot.lastError,
|
||||||
|
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
||||||
@@ -253,7 +289,12 @@ export default defineBackground(() => {
|
|||||||
// the worker and let us reconnect immediately instead of waiting for
|
// the worker and let us reconnect immediately instead of waiting for
|
||||||
// the next 30s alarm tick. They only help when the daemon is actually
|
// the next 30s alarm tick. They only help when the daemon is actually
|
||||||
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
||||||
const reconnectIfNeeded = () => {
|
const reconnectIfNeeded = (source: string) => {
|
||||||
|
diagnostics.record("background.wake_reconnect.requested", {
|
||||||
|
source,
|
||||||
|
controller_enabled: controller.isConnectionEnabled,
|
||||||
|
transport_state: transport.state,
|
||||||
|
});
|
||||||
if (!controller.isConnectionEnabled) return;
|
if (!controller.isConnectionEnabled) return;
|
||||||
if (transport.state === "connected") return;
|
if (transport.state === "connected") return;
|
||||||
void transport.connect().catch((err) => {
|
void transport.connect().catch((err) => {
|
||||||
@@ -261,13 +302,14 @@ export default defineBackground(() => {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
||||||
chrome.runtime.onStartup.addListener(reconnectIfNeeded);
|
chrome.runtime.onStartup.addListener(() => reconnectIfNeeded("runtime.onStartup"));
|
||||||
}
|
}
|
||||||
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
||||||
chrome.idle.onStateChanged.addListener((state) => {
|
chrome.idle.onStateChanged.addListener((state) => {
|
||||||
|
diagnostics.record("chrome.idle.state_changed", { state });
|
||||||
// "active" fires when the user returns from idle/locked — the most
|
// "active" fires when the user returns from idle/locked — the most
|
||||||
// reliable "machine just woke" signal we get.
|
// reliable "machine just woke" signal we get.
|
||||||
if (state === "active") reconnectIfNeeded();
|
if (state === "active") reconnectIfNeeded("idle.active");
|
||||||
});
|
});
|
||||||
// Treat >60s of no input as idle so the active transition is timely.
|
// Treat >60s of no input as idle so the active transition is timely.
|
||||||
chrome.idle.setDetectionInterval?.(60);
|
chrome.idle.setDetectionInterval?.(60);
|
||||||
@@ -275,14 +317,21 @@ export default defineBackground(() => {
|
|||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const connectionEnabled = await getConnectionEnabled();
|
const connectionEnabled = await getConnectionEnabled();
|
||||||
|
diagnostics.record("background.controller.attach_started", { connectionEnabled });
|
||||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||||
beforeDisconnect: async () => {
|
beforeDisconnect: async () => {
|
||||||
|
diagnostics.record("background.before_disconnect_cleanup", {
|
||||||
|
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
const report = await cleanupAfterDisconnect();
|
const report = await cleanupAfterDisconnect();
|
||||||
if (report.failures.length > 0) {
|
if (report.failures.length > 0) {
|
||||||
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
onDisconnected: async () => {
|
onDisconnected: async () => {
|
||||||
|
diagnostics.record("background.after_disconnect_cleanup", {
|
||||||
|
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
const report = await cleanupAfterDisconnect();
|
const report = await cleanupAfterDisconnect();
|
||||||
if (report.failures.length > 0) {
|
if (report.failures.length > 0) {
|
||||||
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||||
@@ -290,6 +339,7 @@ export default defineBackground(() => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
})().catch((err) => {
|
})().catch((err) => {
|
||||||
|
diagnostics.record("background.controller.attach_failed", { error: err });
|
||||||
console.error("[browser-skill] controller failed to attach", err);
|
console.error("[browser-skill] controller failed to attach", err);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -370,6 +420,15 @@ export default defineBackground(() => {
|
|||||||
dbg.__bhDispatcher = dispatcher;
|
dbg.__bhDispatcher = dispatcher;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (typeof chrome.runtime?.onSuspend?.addListener === "function") {
|
||||||
|
chrome.runtime.onSuspend.addListener(() => {
|
||||||
|
diagnostics.record("extension.worker.suspending", {
|
||||||
|
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
transport_state: transport.state,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
console.info("[browser-skill] background worker initialised");
|
console.info("[browser-skill] background worker initialised");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request";
|
|||||||
import overlayCss from "@/content/overlay.css?inline";
|
import overlayCss from "@/content/overlay.css?inline";
|
||||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||||
import {
|
|
||||||
handleRecordContentMessage,
|
|
||||||
isRecordContentMessage,
|
|
||||||
type RecordCaptureController,
|
|
||||||
} from "@/content/record-capture";
|
|
||||||
import {
|
import {
|
||||||
type CaptureSuppressAck,
|
type CaptureSuppressAck,
|
||||||
type CaptureSuppressMessage,
|
type CaptureSuppressMessage,
|
||||||
@@ -45,11 +40,17 @@ import {
|
|||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||||
import {
|
import {
|
||||||
|
isRecordCancelMessage,
|
||||||
|
isRecordStartMessage,
|
||||||
|
isRecordStopMessage,
|
||||||
RECORD_FINISH,
|
RECORD_FINISH,
|
||||||
RECORD_QUERY,
|
RECORD_QUERY,
|
||||||
|
type RecordCancelMessage,
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
|
type RecordStartMessage,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import type {
|
import type {
|
||||||
BorrowCancelMessage,
|
BorrowCancelMessage,
|
||||||
@@ -69,7 +70,6 @@ export default defineContentScript({
|
|||||||
if (window.top !== window) return;
|
if (window.top !== window) return;
|
||||||
|
|
||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
let overlayHost: HTMLElement | null = null;
|
let overlayHost: HTMLElement | null = null;
|
||||||
@@ -228,8 +228,6 @@ export default defineContentScript({
|
|||||||
if (previousHelp) {
|
if (previousHelp) {
|
||||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
recordCapture?.dispose();
|
|
||||||
recordCapture = null;
|
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
renderAll();
|
renderAll();
|
||||||
}
|
}
|
||||||
@@ -259,6 +257,9 @@ export default defineContentScript({
|
|||||||
| HelpRequestMessage
|
| HelpRequestMessage
|
||||||
| HelpCancelMessage
|
| HelpCancelMessage
|
||||||
| CaptureSuppressMessage
|
| CaptureSuppressMessage
|
||||||
|
| RecordStartMessage
|
||||||
|
| RecordStopMessage
|
||||||
|
| RecordCancelMessage
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAgentStateMessage
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
@@ -269,41 +270,32 @@ export default defineContentScript({
|
|||||||
return captureSuppress.handleMessage(message, sendResponse);
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordStartMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
activeRecordRequestId = message.requestId;
|
||||||
message,
|
overlays.setAgentRecordRequest({
|
||||||
{
|
id: message.requestId,
|
||||||
activeRequestId: activeRecordRequestId,
|
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
|
||||||
capture: recordCapture,
|
onFinish: () => {
|
||||||
setActiveRequestId: (id) => {
|
void chrome.runtime.sendMessage({
|
||||||
activeRecordRequestId = id;
|
type: RECORD_FINISH,
|
||||||
},
|
requestId: message.requestId,
|
||||||
setCapture: (capture) => {
|
});
|
||||||
recordCapture = capture;
|
|
||||||
},
|
|
||||||
onStart: (requestId, startedAtMs) => {
|
|
||||||
overlays.setAgentRecordRequest({
|
|
||||||
id: requestId,
|
|
||||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
|
||||||
onFinish: () => {
|
|
||||||
void chrome.runtime.sendMessage({
|
|
||||||
type: RECORD_FINISH,
|
|
||||||
requestId,
|
|
||||||
});
|
|
||||||
},
|
|
||||||
});
|
|
||||||
renderAll();
|
|
||||||
},
|
|
||||||
onStop: () => {
|
|
||||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
|
||||||
renderAll();
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
sendResponse as unknown as
|
});
|
||||||
| ((response: RecordStartAck | RecordStopAck) => void)
|
renderAll();
|
||||||
| undefined,
|
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
|
||||||
);
|
return false;
|
||||||
return needsAsync;
|
}
|
||||||
|
|
||||||
|
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||||
|
if (activeRecordRequestId !== message.requestId) return false;
|
||||||
|
overlays.clearAgentRecordRequest(message.requestId);
|
||||||
|
activeRecordRequestId = null;
|
||||||
|
renderAll();
|
||||||
|
if (isRecordStopMessage(message)) {
|
||||||
|
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -526,9 +518,6 @@ export default defineContentScript({
|
|||||||
chrome.runtime.onMessage.removeListener(onMessage);
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||||
window.removeEventListener("pageshow", onPageShow);
|
window.removeEventListener("pageshow", onPageShow);
|
||||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
|
||||||
recordCapture?.dispose();
|
|
||||||
recordCapture = null;
|
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -200,6 +200,9 @@ describe("App", () => {
|
|||||||
expect(copied).toContain("--browser 03c3e47f");
|
expect(copied).toContain("--browser 03c3e47f");
|
||||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||||
|
expect(copied).toContain("./trace");
|
||||||
|
expect(copied).toContain("trace.json");
|
||||||
|
expect(copied).toContain("states/");
|
||||||
// Button label stays static; a transient toast confirms the copy.
|
// Button label stays static; a transient toast confirms the copy.
|
||||||
expect(copyButton.textContent).toContain("复制录制指令");
|
expect(copyButton.textContent).toContain("复制录制指令");
|
||||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||||
@@ -244,6 +247,41 @@ describe("App", () => {
|
|||||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||||
|
mockUseConnectionState.mockReturnValue({
|
||||||
|
snapshot: {
|
||||||
|
...baseSnapshot,
|
||||||
|
state: "version_skew",
|
||||||
|
instanceId: "03c3e47f",
|
||||||
|
handshake: {
|
||||||
|
server: "bh",
|
||||||
|
version: mockDaemonVersion,
|
||||||
|
protocol_version: "1.0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
statusState: "version_skew",
|
||||||
|
setLabel,
|
||||||
|
setConnectionEnabled,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
expect(screen.getByText("已连接")).toBeTruthy();
|
||||||
|
expect(screen.getByText("可升级")).toBeTruthy();
|
||||||
|
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||||
|
expect(screen.queryByText("Action needed")).toBeNull();
|
||||||
|
expect(screen.queryByText("兼容")).toBeNull();
|
||||||
|
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||||
|
expect(warning.textContent).toContain("CLI 协议");
|
||||||
|
expect(warning.textContent).toContain("扩展协议");
|
||||||
|
|
||||||
|
openRecordView();
|
||||||
|
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||||
|
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||||
|
fireEvent.click(copyButton);
|
||||||
|
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("control hints toggle", () => {
|
describe("control hints toggle", () => {
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ export function App() {
|
|||||||
}, [copiedTick]);
|
}, [copiedTick]);
|
||||||
|
|
||||||
const isSkewed = statusState === "version_skew";
|
const isSkewed = statusState === "version_skew";
|
||||||
|
const connectionLive = statusState === "connected" || isSkewed;
|
||||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||||
const extensionVersion = snapshot.extensionVersion || "—";
|
const extensionVersion = snapshot.extensionVersion || "—";
|
||||||
@@ -90,7 +91,7 @@ export function App() {
|
|||||||
setCopiedInstanceId(true);
|
setCopiedInstanceId(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||||
const recordPurpose = purposeDraft.trim();
|
const recordPurpose = purposeDraft.trim();
|
||||||
const recordStartUrl = startUrlDraft.trim();
|
const recordStartUrl = startUrlDraft.trim();
|
||||||
const recordCommand = snapshot.instanceId
|
const recordCommand = snapshot.instanceId
|
||||||
@@ -206,12 +207,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
<p
|
<p
|
||||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||||
data-slot="popup-version-skew-warning"
|
data-slot="popup-version-skew-warning"
|
||||||
>
|
>
|
||||||
{t("popup.versionSkewWarning", {
|
{t("popup.versionSkewWarning", {
|
||||||
extensionProtocol: PROTOCOL_VERSION,
|
extensionProtocol: PROTOCOL_VERSION,
|
||||||
daemonProtocol,
|
cliProtocol: daemonProtocol,
|
||||||
})}
|
})}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
|
||||||
|
|
||||||
|
export default defineContentScript({
|
||||||
|
matches: ["<all_urls>"],
|
||||||
|
runAt: "document_start",
|
||||||
|
allFrames: true,
|
||||||
|
matchAboutBlank: true,
|
||||||
|
matchOriginAsFallback: true,
|
||||||
|
|
||||||
|
main(ctx) {
|
||||||
|
const dispose = attachRecordFrameAgent();
|
||||||
|
ctx.onInvalidated(dispose);
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -26,20 +26,20 @@ function handshake(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("computeConnectedState (protocol-based compat)", () => {
|
describe("computeConnectedState (protocol-based compat)", () => {
|
||||||
it("returns connected when protocol strings match", () => {
|
it("returns connected when daemon protocol equals extension protocol", () => {
|
||||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||||
kind: "connected",
|
kind: "connected",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("min_compatible_protocol");
|
expect(result.reason).toContain("min_compatible_protocol");
|
||||||
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState({
|
const result = computeConnectedState({
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.1.0",
|
min_compatible_peer: "0.1.0",
|
||||||
});
|
});
|
||||||
expect(result).toEqual({ kind: "connected" });
|
expect(result).toEqual({ kind: "connected" });
|
||||||
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
expect(result.reason).toContain("daemon min_compatible_protocol");
|
||||||
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||||
expect(second.id).not.toBe(first.id);
|
expect(second.id).not.toBe(first.id);
|
||||||
|
|
||||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
expect(controller.snapshot().state).not.toBe("connected");
|
expect(controller.snapshot().state).not.toBe("connected");
|
||||||
|
|
||||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
|
||||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { DiagnosticLogger } from "../diagnostics";
|
||||||
|
|
||||||
|
class MemoryStorage {
|
||||||
|
value: Record<string, unknown> = {};
|
||||||
|
|
||||||
|
async get(key: string): Promise<Record<string, unknown>> {
|
||||||
|
return { [key]: this.value[key] };
|
||||||
|
}
|
||||||
|
|
||||||
|
async set(items: Record<string, unknown>): Promise<void> {
|
||||||
|
this.value = { ...this.value, ...items };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("DiagnosticLogger", () => {
|
||||||
|
it("persists evidence before transport readiness and flushes it after reconnect", async () => {
|
||||||
|
const storage = new MemoryStorage();
|
||||||
|
const send = vi.fn();
|
||||||
|
const logger = new DiagnosticLogger(storage);
|
||||||
|
logger.bindTransport(send);
|
||||||
|
|
||||||
|
logger.record("transport.socket.closed", { code: 1006 });
|
||||||
|
await logger.flush();
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
logger.setTransportReady(true);
|
||||||
|
await logger.flush();
|
||||||
|
|
||||||
|
expect(send).toHaveBeenCalledOnce();
|
||||||
|
expect(send.mock.calls[0]?.[0]).toMatchObject({
|
||||||
|
event: "browser.diagnostic",
|
||||||
|
payload: {
|
||||||
|
schema_version: 1,
|
||||||
|
entries: [
|
||||||
|
{
|
||||||
|
event: "transport.socket.closed",
|
||||||
|
fields: { code: 1006 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps persisted evidence when the transport rejects the flush", async () => {
|
||||||
|
const storage = new MemoryStorage();
|
||||||
|
const logger = new DiagnosticLogger(storage);
|
||||||
|
logger.bindTransport(() => {
|
||||||
|
throw new Error("socket closed");
|
||||||
|
});
|
||||||
|
logger.record("session.stop.requested", { session_id: "abcd" });
|
||||||
|
|
||||||
|
logger.setTransportReady(true);
|
||||||
|
await logger.flush();
|
||||||
|
|
||||||
|
expect(storage.value.bsk_diagnostic_events_v1).toMatchObject([
|
||||||
|
{ event: "session.stop.requested", fields: { session_id: "abcd" } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not resend an acknowledged event when send records a new event", async () => {
|
||||||
|
const storage = new MemoryStorage();
|
||||||
|
const batches: Array<{ payload?: { entries?: Array<{ event: string }> } }> = [];
|
||||||
|
const logger = new DiagnosticLogger(storage);
|
||||||
|
logger.record("dispatcher.rpc.received", { rpc_id: "one" });
|
||||||
|
await logger.flush();
|
||||||
|
logger.bindTransport((frame) => {
|
||||||
|
batches.push(frame as { payload?: { entries?: Array<{ event: string }> } });
|
||||||
|
if (batches.length === 1) {
|
||||||
|
logger.record("dispatcher.rpc.response_sent", { rpc_id: "one" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.setTransportReady(true);
|
||||||
|
await logger.flush();
|
||||||
|
await logger.flush();
|
||||||
|
|
||||||
|
expect(batches.map((batch) => batch.payload?.entries?.map((entry) => entry.event))).toEqual([
|
||||||
|
["dispatcher.rpc.received"],
|
||||||
|
["dispatcher.rpc.response_sent"],
|
||||||
|
]);
|
||||||
|
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { waitForDocumentSettled } from "../recording/document-settle";
|
||||||
|
|
||||||
|
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||||
|
const send = vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||||
|
send,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("waitForDocumentSettled", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels before probing a superseded observation", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const settled = waitForDocumentSettled(
|
||||||
|
cdp,
|
||||||
|
{ target: { tabId: 7 } },
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
controller.abort();
|
||||||
|
await vi.advanceTimersByTimeAsync(60);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("cancelled");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes a same-process iframe through its isolated execution context", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, {
|
||||||
|
target: { tabId: 7 },
|
||||||
|
frameId: "child-frame",
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
expect.objectContaining({ frameId: "child-frame" }),
|
||||||
|
);
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 91 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes an OOPIF probe through its CDP target session", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 27 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
isAcceptedRecordStepAck,
|
||||||
|
isRecordStepMessage,
|
||||||
|
RECORD_STEP,
|
||||||
|
type RecordStepMessage,
|
||||||
|
} from "../record-bridge";
|
||||||
|
|
||||||
|
const message: RecordStepMessage = {
|
||||||
|
type: RECORD_STEP,
|
||||||
|
requestId: "rec-1",
|
||||||
|
producerId: "document-1",
|
||||||
|
sequence: 1,
|
||||||
|
step: { op: "click", target: { tag: "button", name: "Submit" } },
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("record step delivery protocol", () => {
|
||||||
|
it("requires a producer and a positive integer sequence", () => {
|
||||||
|
expect(isRecordStepMessage(message)).toBe(true);
|
||||||
|
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
|
||||||
|
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
|
||||||
|
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts only the acknowledgement for the sent sequence", () => {
|
||||||
|
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
|
||||||
|
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
|
||||||
|
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
|
||||||
|
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
|
||||||
|
|
||||||
|
class ListenerSet<T extends (...args: never[]) => unknown> {
|
||||||
|
readonly listeners = new Set<T>();
|
||||||
|
addListener = (listener: T) => this.listeners.add(listener);
|
||||||
|
removeListener = (listener: T) => this.listeners.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakePort(sender: chrome.runtime.MessageSender) {
|
||||||
|
const messages: unknown[] = [];
|
||||||
|
const onMessage = new ListenerSet<(message: unknown) => void>();
|
||||||
|
const onDisconnect = new ListenerSet<() => void>();
|
||||||
|
return {
|
||||||
|
port: {
|
||||||
|
name: RECORD_FRAME_PORT,
|
||||||
|
sender,
|
||||||
|
onMessage,
|
||||||
|
onDisconnect,
|
||||||
|
postMessage: vi.fn((message: unknown) => messages.push(message)),
|
||||||
|
disconnect: vi.fn(),
|
||||||
|
} as unknown as chrome.runtime.Port,
|
||||||
|
messages,
|
||||||
|
receive(message: unknown) {
|
||||||
|
for (const listener of onMessage.listeners) listener(message);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("RecordFrameCoordinator", () => {
|
||||||
|
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
|
||||||
|
const onMessage = new ListenerSet<
|
||||||
|
(
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: unknown) => void,
|
||||||
|
) => boolean
|
||||||
|
>();
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
onConnect.listeners.clear();
|
||||||
|
onMessage.listeners.clear();
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: { onConnect, onMessage },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("answers child-document queries for the initial tab before it is armed", () => {
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [],
|
||||||
|
sendToDocument: async () => ({ ok: true }),
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
coordinator.begin("rec-1", 10, 3);
|
||||||
|
|
||||||
|
const query = (tabId: number) => {
|
||||||
|
let response: unknown;
|
||||||
|
for (const listener of onMessage.listeners) {
|
||||||
|
listener(
|
||||||
|
{ type: RECORD_FRAME_QUERY },
|
||||||
|
{
|
||||||
|
tab: { id: tabId },
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "child-document",
|
||||||
|
} as chrome.runtime.MessageSender,
|
||||||
|
(value) => {
|
||||||
|
response = value;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
|
||||||
|
expect(query(4)).toEqual({ active: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts a child Document that appears after the initial frame snapshot", async () => {
|
||||||
|
let notifyFrameNavigation:
|
||||||
|
| ((frame: {
|
||||||
|
tabId: number;
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
lifecycle: "committed" | "completed";
|
||||||
|
}) => void)
|
||||||
|
| undefined;
|
||||||
|
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||||
|
sendToDocument,
|
||||||
|
subscribeFrameNavigation(listener) {
|
||||||
|
notifyFrameNavigation = listener;
|
||||||
|
return () => {};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
const onDocumentReady = vi.fn();
|
||||||
|
coordinator.begin("rec-1", 10, 3, onDocumentReady);
|
||||||
|
await coordinator.armTab("rec-1", 3);
|
||||||
|
sendToDocument.mockClear();
|
||||||
|
|
||||||
|
notifyFrameNavigation?.({
|
||||||
|
tabId: 3,
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "late-child",
|
||||||
|
lifecycle: "completed",
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(sendToDocument).toHaveBeenCalledWith(
|
||||||
|
3,
|
||||||
|
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
|
||||||
|
{ documentId: "late-child" },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
expect(onDocumentReady).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const child = fakePort({
|
||||||
|
tab: { id: 3 },
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "late-child",
|
||||||
|
} as chrome.runtime.MessageSender);
|
||||||
|
for (const listener of onConnect.listeners) listener(child.port);
|
||||||
|
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||||
|
notifyFrameNavigation?.({
|
||||||
|
tabId: 3,
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "late-child",
|
||||||
|
lifecycle: "completed",
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(onDocumentReady).toHaveBeenCalledWith({
|
||||||
|
tabId: 3,
|
||||||
|
documentId: "late-child",
|
||||||
|
browserFrameId: 7,
|
||||||
|
producerId: "producer-1",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
|
||||||
|
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [
|
||||||
|
{ frameId: 0, documentId: "top-document" },
|
||||||
|
{ frameId: 7, documentId: "child-document" },
|
||||||
|
],
|
||||||
|
sendToDocument,
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
coordinator.begin("rec-1", 10, 3);
|
||||||
|
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
|
||||||
|
|
||||||
|
const sender = {
|
||||||
|
tab: { id: 3 },
|
||||||
|
frameId: 7,
|
||||||
|
documentId: "child-document",
|
||||||
|
} as chrome.runtime.MessageSender;
|
||||||
|
const frame = fakePort(sender);
|
||||||
|
for (const listener of onConnect.listeners) listener(frame.port);
|
||||||
|
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||||
|
|
||||||
|
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
|
||||||
|
tabId: 3,
|
||||||
|
documentId: "child-document",
|
||||||
|
browserFrameId: 7,
|
||||||
|
producerId: "producer-1",
|
||||||
|
});
|
||||||
|
|
||||||
|
const stopping = coordinator.stop("rec-1");
|
||||||
|
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
|
||||||
|
const stop = frame.messages.find(
|
||||||
|
(message): message is { type: "stop"; commandId: string } =>
|
||||||
|
typeof message === "object" &&
|
||||||
|
message !== null &&
|
||||||
|
(message as { type?: string }).type === "stop",
|
||||||
|
);
|
||||||
|
expect(stop).toBeDefined();
|
||||||
|
frame.receive({
|
||||||
|
type: "stopped",
|
||||||
|
requestId: "rec-1",
|
||||||
|
commandId: stop!.commandId,
|
||||||
|
ok: true,
|
||||||
|
});
|
||||||
|
await expect(stopping).resolves.toBe(true);
|
||||||
|
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a producer used from another Document", async () => {
|
||||||
|
const coordinator = new RecordFrameCoordinator({
|
||||||
|
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||||
|
sendToDocument: async () => ({ ok: true }),
|
||||||
|
});
|
||||||
|
coordinator.attach();
|
||||||
|
coordinator.begin("rec-1", 10, 3);
|
||||||
|
await coordinator.armTab("rec-1", 3);
|
||||||
|
const sender = {
|
||||||
|
tab: { id: 3 },
|
||||||
|
frameId: 0,
|
||||||
|
documentId: "top-document",
|
||||||
|
} as chrome.runtime.MessageSender;
|
||||||
|
const frame = fakePort(sender);
|
||||||
|
for (const listener of onConnect.listeners) listener(frame.port);
|
||||||
|
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
coordinator.sourceFor("rec-1", "producer-1", {
|
||||||
|
...sender,
|
||||||
|
documentId: "other-document",
|
||||||
|
}),
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
RECORD_DOCUMENT_ATTRIBUTE,
|
||||||
|
recordingDocumentMarkerValue,
|
||||||
|
} from "@/shared/recording-document-identity";
|
||||||
|
import { markRecordingDocument } from "../recording/document-marker";
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
|
||||||
|
describe("recording document marker", () => {
|
||||||
|
it("rejects a non-random Document identity", () => {
|
||||||
|
expect(() => markRecordingDocument("user@example.com")).toThrow(
|
||||||
|
"recording Document identity must be a random UUID",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restores the Document attribute exactly", () => {
|
||||||
|
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
|
||||||
|
const producerId = "123e4567-e89b-42d3-a456-426614174000";
|
||||||
|
const marker = markRecordingDocument(producerId);
|
||||||
|
|
||||||
|
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
|
||||||
|
recordingDocumentMarkerValue(producerId),
|
||||||
|
);
|
||||||
|
marker.restore();
|
||||||
|
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
|
||||||
|
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("indexes the CDP scope from the safe frame identity", () => {
|
||||||
|
const index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 3, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId: "producer-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [],
|
||||||
|
refs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(index.documentScope("producer-1")).toEqual({
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 3, sessionId: "oopif-session" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when a recording identity maps to multiple Documents", () => {
|
||||||
|
const index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "left",
|
||||||
|
target: { tabId: 3, sessionId: "left-session" },
|
||||||
|
recordingDocumentId: "duplicate",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "right",
|
||||||
|
target: { tabId: 3, sessionId: "right-session" },
|
||||||
|
recordingDocumentId: "duplicate",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [],
|
||||||
|
refs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(index.documentScope("duplicate")).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||||
|
const session = new RecordingObservationSession({ redactValues });
|
||||||
|
const state = session.registry.register({
|
||||||
|
url: URL,
|
||||||
|
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||||
|
});
|
||||||
|
session.cursor.lastSettled = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "input",
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
}),
|
||||||
|
url: URL,
|
||||||
|
};
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillTrace(value: string, redactValues: boolean) {
|
||||||
|
const session = sessionWithInput(redactValues);
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
||||||
|
value,
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
draft.postStateId = draft.preStateId;
|
||||||
|
return buildTraceV3({
|
||||||
|
registry: session.registry,
|
||||||
|
drafts: [draft],
|
||||||
|
annotations: session.annotations,
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
redactValues,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
return finalizedFillTrace(value, redactValues).states[0]!.body;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
it("omits a fill literal when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
|
||||||
|
expect(dumped).toContain("step 1: fill");
|
||||||
|
expect(dumped).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps ordinary fill details", () => {
|
||||||
|
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({
|
||||||
|
url: "https://example.com/a\nb",
|
||||||
|
title: "hello\nworld",
|
||||||
|
vomText: "@vom 1",
|
||||||
|
});
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
||||||
|
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("recording state ownership", () => {
|
||||||
|
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||||
|
const first = new RecordingStateRegistry();
|
||||||
|
const second = new RecordingStateRegistry();
|
||||||
|
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||||
|
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||||
|
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
registry.register({ url: URL, vomText: "same" });
|
||||||
|
const state = registry.register({
|
||||||
|
url: URL,
|
||||||
|
title: "Login",
|
||||||
|
vomText: "same",
|
||||||
|
truncated: true,
|
||||||
|
});
|
||||||
|
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("draft binding", () => {
|
||||||
|
it("keeps capture semantics when no observation exists", () => {
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not bind an unmatched new action to a stale observation", () => {
|
||||||
|
const session = sessionWithInput();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 2, true);
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "Confirm",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
|
||||||
|
const captureRecordingObservation = vi.hoisted(() => vi.fn());
|
||||||
|
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
|
||||||
|
|
||||||
|
vi.mock("../recording/observation-capture", async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
|
||||||
|
return { ...actual, captureRecordingObservation };
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock("../recording/document-settle", async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import("../recording/document-settle")>();
|
||||||
|
return { ...actual, waitForDocumentSettled };
|
||||||
|
});
|
||||||
|
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
function observation(url = "https://example.com/") {
|
||||||
|
return {
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
|
url,
|
||||||
|
title: "Example",
|
||||||
|
vomText: '@vom 1\nRootWebArea "Example"',
|
||||||
|
truncated: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function runtime(): RecordingObservationRuntime {
|
||||||
|
return new RecordingObservationRuntime({
|
||||||
|
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
|
||||||
|
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("RecordingObservationRuntime", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
captureRecordingObservation.mockReset();
|
||||||
|
waitForDocumentSettled.mockClear();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shares one initial capture and includes it in flush", async () => {
|
||||||
|
let release!: (value: ReturnType<typeof observation>) => void;
|
||||||
|
captureRecordingObservation.mockReturnValueOnce(
|
||||||
|
new Promise((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
const first = recording.captureInitial(7);
|
||||||
|
const second = recording.captureInitial(7);
|
||||||
|
const flushed = recording.flush();
|
||||||
|
let flushCompleted = false;
|
||||||
|
void flushed.then(() => {
|
||||||
|
flushCompleted = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
|
||||||
|
expect(flushCompleted).toBe(false);
|
||||||
|
|
||||||
|
release(observation());
|
||||||
|
await Promise.all([first, second, flushed]);
|
||||||
|
expect(flushCompleted).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels an in-flight initial capture", async () => {
|
||||||
|
captureRecordingObservation.mockImplementationOnce(
|
||||||
|
(input: { signal?: AbortSignal }) =>
|
||||||
|
new Promise((_, reject) => {
|
||||||
|
input.signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
const pending = recording.captureInitial(7);
|
||||||
|
recording.cancel();
|
||||||
|
|
||||||
|
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries a failed initial capture when the recording settles at stop", async () => {
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockRejectedValueOnce(new Error("document swapped"))
|
||||||
|
.mockResolvedValueOnce(observation());
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
|
||||||
|
await recording.settleTrailing(7, []);
|
||||||
|
const trace = recording.buildTrace({
|
||||||
|
drafts: [],
|
||||||
|
startedAt: "2026-08-19T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "0.1.6",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
||||||
|
expect(trace.states).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps runtime matching data out of the serialized trace", async () => {
|
||||||
|
const captured = observation();
|
||||||
|
captured.index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
|
||||||
|
});
|
||||||
|
captureRecordingObservation.mockResolvedValueOnce(captured);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
const dumped = JSON.stringify(
|
||||||
|
recording.buildTrace({
|
||||||
|
drafts: [],
|
||||||
|
startedAt: "2026-08-19T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "0.1.6",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(dumped).toContain("RootWebArea");
|
||||||
|
expect(dumped).not.toContain("matchNodes");
|
||||||
|
expect(dumped).not.toContain("localRect");
|
||||||
|
expect(dumped).not.toContain("backendNodeId");
|
||||||
|
expect(dumped).not.toContain("ObservationNodeIndex");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("captures a tab transition without sharing observation cursors", async () => {
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockResolvedValueOnce(observation("https://example.com/first"))
|
||||||
|
.mockResolvedValueOnce(observation("https://example.com/second"));
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(4);
|
||||||
|
const transition = await recording.captureTabTransition(4, 5);
|
||||||
|
await recording.captureInitial(4);
|
||||||
|
await recording.captureInitial(5);
|
||||||
|
|
||||||
|
expect(transition).toEqual({
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s2",
|
||||||
|
targetUrl: "https://example.com/second",
|
||||||
|
});
|
||||||
|
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("binds an iframe draft to its marked CDP Document scope", async () => {
|
||||||
|
captureRecordingObservation.mockResolvedValueOnce({
|
||||||
|
...observation(),
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 7, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId: "producer-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const recording = runtime();
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "click" as const,
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||||
|
recording.cancel();
|
||||||
|
|
||||||
|
const draft = drafts[0];
|
||||||
|
expect(draft?.op).toBe("click");
|
||||||
|
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||||
|
expect(draft.targetHint).toMatchObject({
|
||||||
|
frameId: "child",
|
||||||
|
geometrySpace: "local",
|
||||||
|
});
|
||||||
|
expect(draft.matchedTarget?.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refreshes a late iframe before matching its first action", async () => {
|
||||||
|
const iframeObservation = {
|
||||||
|
...observation(),
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frames: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 7, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId: "producer-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [
|
||||||
|
{
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
role: "button",
|
||||||
|
name: "表格视图",
|
||||||
|
line: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockResolvedValueOnce(observation())
|
||||||
|
.mockResolvedValueOnce(iframeObservation)
|
||||||
|
.mockResolvedValueOnce(iframeObservation);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
await recording.refreshDocument(7, "producer-1");
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "表格视图" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||||
|
recording.cancel();
|
||||||
|
|
||||||
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
|
||||||
|
expect(waitForDocumentSettled).toHaveBeenCalledWith(
|
||||||
|
expect.anything(),
|
||||||
|
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
|
||||||
|
{ signal: expect.any(AbortSignal) },
|
||||||
|
);
|
||||||
|
const draft = drafts[0];
|
||||||
|
expect(draft?.op).toBe("click");
|
||||||
|
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||||
|
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not poison final flush when a frame readiness refresh fails", async () => {
|
||||||
|
captureRecordingObservation
|
||||||
|
.mockResolvedValueOnce(observation())
|
||||||
|
.mockRejectedValueOnce(new Error("child Document replaced"));
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
|
||||||
|
"child Document replaced",
|
||||||
|
);
|
||||||
|
await expect(recording.flush()).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
|
||||||
|
|
||||||
describe("recording-step-buffer", () => {
|
|
||||||
it("stores semantic click without summary", () => {
|
|
||||||
const buffer = { steps: [], pendingNavigation: false };
|
|
||||||
appendRecordedPayload(buffer, {
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
expects_navigation: true,
|
|
||||||
});
|
|
||||||
expect(buffer.steps).toEqual([
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
expect(buffer.pendingNavigation).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [
|
|
||||||
{
|
|
||||||
op: "click" as const,
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
currentUrl: "https://example.com/a",
|
|
||||||
pendingNavigation: true,
|
|
||||||
pendingNavigationDeadline: Date.now() + 5_000,
|
|
||||||
};
|
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
|
||||||
expect(buffer.steps).toEqual([
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
navigated_to: "https://example.com/b",
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [
|
|
||||||
{
|
|
||||||
op: "select" as const,
|
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
|
||||||
values: ["tech"],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
currentUrl: "https://example.com/list",
|
|
||||||
pendingNavigation: true,
|
|
||||||
pendingNavigationDeadline: Date.now() + 5_000,
|
|
||||||
};
|
|
||||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
|
||||||
expect(buffer.steps).toEqual([
|
|
||||||
{
|
|
||||||
op: "select",
|
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
|
||||||
values: ["tech"],
|
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("emits navigate for uncaused URL changes", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [],
|
|
||||||
currentUrl: "https://example.com/a",
|
|
||||||
pendingNavigation: false,
|
|
||||||
};
|
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
|
||||||
expect(buffer.steps[0]).toMatchObject({
|
|
||||||
op: "navigate",
|
|
||||||
url: "https://example.com/b",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { SettleController } from "../recording/settle-controller";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const OBSERVATION: RegisteredObservation = {
|
||||||
|
stateId: "s-next",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
|
url: "https://example.com/next",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("SettleController", () => {
|
||||||
|
afterEach(() => vi.useRealTimers());
|
||||||
|
|
||||||
|
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
let firstSignal: AbortSignal | undefined;
|
||||||
|
vi.spyOn(session, "capture")
|
||||||
|
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
||||||
|
firstSignal = signal;
|
||||||
|
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce(OBSERVATION);
|
||||||
|
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
controller.schedule(drafts, 0);
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
drafts.push({
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", name: "Second" },
|
||||||
|
preStateId: "s-next",
|
||||||
|
});
|
||||||
|
controller.schedule(drafts, 1);
|
||||||
|
expect(firstSignal?.aborted).toBe(true);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flush();
|
||||||
|
expect(drafts[1]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||||
|
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||||
|
resolveFirstTab = resolve;
|
||||||
|
});
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi
|
||||||
|
.fn()
|
||||||
|
.mockImplementationOnce(() => firstTab)
|
||||||
|
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
session.cursor.lastSettled = OBSERVATION;
|
||||||
|
const drafts: RecordingDraftStep[] = [];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
||||||
|
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flushRedirects();
|
||||||
|
|
||||||
|
expect(drafts).toEqual([]);
|
||||||
|
expect(controller.hasPending).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("settles an action in its own OOPIF document scope", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
|
||||||
|
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
await controller.flush();
|
||||||
|
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 39 }),
|
||||||
|
);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
||||||
|
|
||||||
|
describe("recording-step-buffer", () => {
|
||||||
|
it("stores semantic click without summary", () => {
|
||||||
|
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||||
|
appendRecordedPayload(buffer, {
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "button", role: "button", name: "发布" },
|
||||||
|
expects_navigation: true,
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(buffer.navigation.pendingNavigation).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the hovered element description as a capture fallback", () => {
|
||||||
|
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||||
|
appendRecordedPayload(
|
||||||
|
buffer,
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
op: "click" as const,
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://example.com/a",
|
||||||
|
pendingNavigation: true,
|
||||||
|
pendingNavigationDeadline: Date.now() + 5_000,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
|
navigatedTo: "https://example.com/b",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
op: "select" as const,
|
||||||
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
|
values: ["tech"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://example.com/list",
|
||||||
|
pendingNavigation: true,
|
||||||
|
pendingNavigationDeadline: Date.now() + 5_000,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "select",
|
||||||
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
|
values: ["tech"],
|
||||||
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits navigate for uncaused URL changes", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
|
||||||
|
};
|
||||||
|
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||||
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
|
expect(buffer.steps[0]).toMatchObject({
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/b",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://passport.example/login",
|
||||||
|
pendingNavigation: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const hop1 = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
"https://passport.example/callback",
|
||||||
|
false,
|
||||||
|
"link",
|
||||||
|
["server_redirect"],
|
||||||
|
);
|
||||||
|
expect(hop1).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://passport.example/callback",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
|
||||||
|
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||||
|
"client_redirect",
|
||||||
|
]);
|
||||||
|
expect(hop2).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://app.example/dashboard",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reuse redirect metadata for a later content-observed URL change", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
navigation: {
|
||||||
|
currentUrl: "https://example.com/redirect",
|
||||||
|
pendingNavigation: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
|
||||||
|
|
||||||
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
|
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
|
||||||
|
|
||||||
|
describe("RecordingTabCoordinator", () => {
|
||||||
|
it("keeps navigation state isolated per tab", () => {
|
||||||
|
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
|
||||||
|
const first = tabs.navigation(4);
|
||||||
|
first.pendingNavigation = true;
|
||||||
|
const second = tabs.navigation(5, "https://example.com/second");
|
||||||
|
|
||||||
|
expect(second).toEqual({
|
||||||
|
currentUrl: "https://example.com/second",
|
||||||
|
pendingNavigation: false,
|
||||||
|
});
|
||||||
|
expect(tabs.navigation(4)).toBe(first);
|
||||||
|
expect(tabs.navigation(4).pendingNavigation).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("invalidates an earlier activation when a newer tab becomes active", () => {
|
||||||
|
const tabs = new RecordingTabCoordinator(4);
|
||||||
|
const second = tabs.noteActivation(5);
|
||||||
|
const third = tabs.noteActivation(6);
|
||||||
|
|
||||||
|
expect(tabs.isLatest(second)).toBe(false);
|
||||||
|
expect(tabs.isLatest(third)).toBe(true);
|
||||||
|
tabs.commit(6);
|
||||||
|
expect(tabs.currentTabId).toBe(6);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("commits the freshly observed URL when returning to an existing tab", () => {
|
||||||
|
const tabs = new RecordingTabCoordinator(4);
|
||||||
|
tabs.navigation(5, "https://example.com/old");
|
||||||
|
|
||||||
|
tabs.commit(5, "https://example.com/new");
|
||||||
|
|
||||||
|
expect(tabs.currentTabId).toBe(5);
|
||||||
|
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { matchObservationTarget } from "../recording/target-matcher";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
frameId,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function observation(
|
||||||
|
matchNodes: CaptureVomMatchNode[],
|
||||||
|
refs: RenderedRef[],
|
||||||
|
): RegisteredObservation {
|
||||||
|
return {
|
||||||
|
stateId: "s1",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
||||||
|
url: "https://example.com",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchObservationTarget", () => {
|
||||||
|
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
||||||
|
const geometry = node(42, "child");
|
||||||
|
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
||||||
|
const ref: RenderedRef = {
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
line: 1,
|
||||||
|
};
|
||||||
|
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
||||||
|
|
||||||
|
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a unique node using canonical top-level viewport geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "left"), node(42, "right")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: "right",
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches iframe capture geometry in the child viewport coordinate space", () => {
|
||||||
|
const child = node(42, "child", 410);
|
||||||
|
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[child],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: "child",
|
||||||
|
geometrySpace: "local",
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restricts a missing frame hint to the root frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "child")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when the source Document has no VOM frame mapping", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: null,
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
fallback: { tag: "button", role: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched for ambiguous geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(41, "root"), node(42, "child")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { frameId: "child" },
|
||||||
|
fallback: { tag: "button", role: "button", name: "保存" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", role: "button", name: "取消" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
});
|
||||||
+50
-34
@@ -1,6 +1,15 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
import { shouldRecordPress } from "../recording/draft-policy";
|
||||||
|
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
||||||
|
|
||||||
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
||||||
|
return buildTraceV2({
|
||||||
|
steps,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
...(startUrl ? { startUrl } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
describe("shouldRecordPress", () => {
|
||||||
it("keeps Enter and Escape", () => {
|
it("keeps Enter and Escape", () => {
|
||||||
@@ -18,36 +27,47 @@ describe("shouldRecordPress", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
describe("reduceTraceSteps", () => {
|
||||||
|
it("does not expose internal tab transitions in trace v2", () => {
|
||||||
|
const trace = reduceTraceSteps([
|
||||||
|
{
|
||||||
|
op: "switch_tab",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s2",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(trace.steps).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
it("builds steps with pages dictionary and page id references", () => {
|
||||||
const drafts: DraftTraceStep[] = [
|
const drafts: RecordingDraftStep[] = [
|
||||||
{
|
{
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
url: "https://example.com/search?q=hello&utm_source=x",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
value: "browser skill",
|
value: "browser skill",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "Enter",
|
key: "Enter",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
navigated_to: "https://example.com/results/42",
|
navigatedTo: "https://example.com/results/42",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
navigated_to: "https://example.com/p/99",
|
navigatedTo: "https://example.com/p/99",
|
||||||
page_url: "https://example.com/results/42",
|
pageUrl: "https://example.com/results/42",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "a",
|
key: "a",
|
||||||
page_url: "https://example.com/p/99",
|
pageUrl: "https://example.com/p/99",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -109,9 +129,9 @@ describe("reduceTraceSteps", () => {
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "Search query" },
|
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
||||||
value: "",
|
value: "",
|
||||||
page_url: "https://example.com/search",
|
pageUrl: "https://example.com/search",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/search",
|
"https://example.com/search",
|
||||||
@@ -125,16 +145,16 @@ describe("reduceTraceSteps", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
const { pages, steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
labels: ["技术"],
|
labels: ["技术"],
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
page_url: "https://example.com/list",
|
pageUrl: "https://example.com/list",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/list",
|
"https://example.com/list",
|
||||||
@@ -151,36 +171,32 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps hover steps before menu clicks", () => {
|
it("drops hover steps, which only exist in trace v3", () => {
|
||||||
const { steps } = reduceTraceSteps(
|
const { steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { tag: "span", role: "button", name: "Account" },
|
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||||
page_url: "https://example.com/app",
|
pageUrl: "https://example.com/app",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "a", role: "link", name: "Profile" },
|
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||||
page_url: "https://example.com/app",
|
pageUrl: "https://example.com/app",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/app",
|
"https://example.com/app",
|
||||||
);
|
);
|
||||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
op: "hover",
|
|
||||||
page: "p1",
|
|
||||||
target: { name: "Account" },
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
buildTraceV2({
|
||||||
[{ op: "navigate", url: "https://example.com/other" }],
|
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
||||||
"https://example.com/start",
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
),
|
startUrl: "https://example.com/start",
|
||||||
|
}).entry.start_url,
|
||||||
).toBe("https://example.com/start");
|
).toBe("https://example.com/start");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
describe("trace reducer v3", () => {
|
||||||
|
it("removes form literals when value redaction is requested", () => {
|
||||||
|
const reduced = reduceTraceStepsV3(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
value: "private@example.com",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "select",
|
||||||
|
values: ["private-account-id"],
|
||||||
|
labels: ["Private account"],
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{ redactValues: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
|
||||||
|
expect(reduced.steps[1]).toMatchObject({ op: "select" });
|
||||||
|
expect(reduced.steps[1]).not.toHaveProperty("selection");
|
||||||
|
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
|
||||||
|
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits tab transitions only for callers that advertised support", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "switch_tab",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s2",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
|
||||||
|
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
|
||||||
|
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not export a tab transition without both observation endpoints", () => {
|
||||||
|
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
|
||||||
|
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
|
||||||
|
|
||||||
|
expect(
|
||||||
|
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
|
||||||
|
).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/final",
|
||||||
|
transitionQualifiers: ["server_redirect"],
|
||||||
|
preStateId: "s2",
|
||||||
|
postStateId: "s3",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const before = structuredClone(drafts);
|
||||||
|
const reduced = reduceTraceStepsV3(drafts);
|
||||||
|
|
||||||
|
expect(drafts).toEqual(before);
|
||||||
|
expect(reduced.steps).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
id: 1,
|
||||||
|
op: "navigate",
|
||||||
|
state: "s1",
|
||||||
|
to: "https://example.com/final",
|
||||||
|
result: { state: "s3" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds the wire model from protocol constants", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||||
|
preStateId: state.id,
|
||||||
|
postStateId: state.id,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(trace.version).toBe(TRACE_VERSION_V3);
|
||||||
|
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
||||||
|
expect(trace.steps[0]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { role: "button", name: "Save", unmatched: true },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
} from "../transport/handshake";
|
} from "../transport/handshake";
|
||||||
import type { Transport } from "../transport/transport";
|
import type { Transport } from "../transport/transport";
|
||||||
import type { ConnectionState, HandshakeResult } from "../transport/types";
|
import type { ConnectionState, HandshakeResult } from "../transport/types";
|
||||||
|
import type { DiagnosticSink } from "./diagnostics";
|
||||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||||
|
|
||||||
@@ -51,6 +52,11 @@ export class ConnectionController {
|
|||||||
private handshakeAbort: AbortController | null = null;
|
private handshakeAbort: AbortController | null = null;
|
||||||
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
private suppressDisconnectRecovery = false;
|
private suppressDisconnectRecovery = false;
|
||||||
|
private readonly diagnostics?: DiagnosticSink;
|
||||||
|
|
||||||
|
constructor(diagnostics?: DiagnosticSink) {
|
||||||
|
this.diagnostics = diagnostics;
|
||||||
|
}
|
||||||
|
|
||||||
get isConnectionEnabled(): boolean {
|
get isConnectionEnabled(): boolean {
|
||||||
return this.connectionEnabled;
|
return this.connectionEnabled;
|
||||||
@@ -89,8 +95,20 @@ export class ConnectionController {
|
|||||||
this.lifecycleHooks = lifecycleHooks;
|
this.lifecycleHooks = lifecycleHooks;
|
||||||
this.instanceId = await getOrCreateInstanceId();
|
this.instanceId = await getOrCreateInstanceId();
|
||||||
this.label = await getLabel();
|
this.label = await getLabel();
|
||||||
|
this.diagnostics?.("controller.attach", {
|
||||||
|
instance_id: this.instanceId,
|
||||||
|
connection_enabled: connectionEnabled,
|
||||||
|
transport_state: transport.state,
|
||||||
|
});
|
||||||
|
|
||||||
transport.onConnectionStateChange((s) => {
|
transport.onConnectionStateChange((s) => {
|
||||||
|
this.diagnostics?.("controller.transport_state", {
|
||||||
|
transport_state: s,
|
||||||
|
controller_state: this.currentState,
|
||||||
|
connection_generation: this.connectionGeneration,
|
||||||
|
suppress_disconnect_recovery: this.suppressDisconnectRecovery,
|
||||||
|
recovery_inflight: this.disconnectRecovery !== null,
|
||||||
|
});
|
||||||
if (s === "disconnected") {
|
if (s === "disconnected") {
|
||||||
// An in-flight handshake belongs to the dead connection — cancel it so
|
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||||
// a late resolution can never clobber the next attempt.
|
// a late resolution can never clobber the next attempt.
|
||||||
@@ -103,6 +121,9 @@ export class ConnectionController {
|
|||||||
// handshake bounce); the caller owns the reconnect policy there.
|
// handshake bounce); the caller owns the reconnect policy there.
|
||||||
this.suppressDisconnectRecovery = false;
|
this.suppressDisconnectRecovery = false;
|
||||||
} else {
|
} else {
|
||||||
|
this.diagnostics?.("controller.disconnect_recovery.requested", {
|
||||||
|
connection_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
void this.recoverFromDisconnect();
|
void this.recoverFromDisconnect();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -161,6 +182,12 @@ export class ConnectionController {
|
|||||||
const generation = ++this.connectionGeneration;
|
const generation = ++this.connectionGeneration;
|
||||||
const abort = new AbortController();
|
const abort = new AbortController();
|
||||||
this.handshakeAbort = abort;
|
this.handshakeAbort = abort;
|
||||||
|
this.diagnostics?.("controller.handshake.started", {
|
||||||
|
connection_generation: generation,
|
||||||
|
browser_name: browser.name,
|
||||||
|
browser_version: browser.version,
|
||||||
|
instance_id: this.instanceId,
|
||||||
|
});
|
||||||
void this.runHandshake(browser, generation, abort.signal);
|
void this.runHandshake(browser, generation, abort.signal);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,11 +224,21 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
this.clearHandshakeRetry();
|
this.clearHandshakeRetry();
|
||||||
this.lastError = null;
|
this.lastError = null;
|
||||||
|
this.diagnostics?.("controller.handshake.completed", {
|
||||||
|
connection_generation: generation,
|
||||||
|
verdict: verdict.kind,
|
||||||
|
daemon_version: outcome.result.version,
|
||||||
|
daemon_protocol: outcome.result.protocol_version,
|
||||||
|
});
|
||||||
this.setState(verdict.kind);
|
this.setState(verdict.kind);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
this.diagnostics?.("controller.handshake.failed", {
|
||||||
|
connection_generation: generation,
|
||||||
|
error: err,
|
||||||
|
});
|
||||||
// Bounce the half-open socket deliberately; the retry timer below owns
|
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||||
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||||
this.suppressDisconnectRecovery = true;
|
this.suppressDisconnectRecovery = true;
|
||||||
@@ -228,23 +265,57 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private recoverFromDisconnect(): Promise<void> {
|
private recoverFromDisconnect(): Promise<void> {
|
||||||
if (this.disconnectRecovery) return this.disconnectRecovery;
|
if (this.disconnectRecovery) {
|
||||||
|
this.diagnostics?.("controller.disconnect_recovery.coalesced", {
|
||||||
|
connection_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
|
return this.disconnectRecovery;
|
||||||
|
}
|
||||||
const recovery = (async () => {
|
const recovery = (async () => {
|
||||||
|
const recoveryGeneration = this.connectionGeneration;
|
||||||
|
this.diagnostics?.("controller.disconnect_recovery.started", {
|
||||||
|
recovery_generation: recoveryGeneration,
|
||||||
|
});
|
||||||
// WSTransport schedules its reconnect timer immediately after notifying
|
// WSTransport schedules its reconnect timer immediately after notifying
|
||||||
// state listeners. Yield once, then disconnect explicitly so that timer
|
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||||
// is cancelled before local session teardown begins.
|
// is cancelled before local session teardown begins.
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
if (!this.connectionEnabled || !this.transport) return;
|
if (!this.connectionEnabled || !this.transport) {
|
||||||
|
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
||||||
|
recovery_generation: recoveryGeneration,
|
||||||
|
reason: "disabled_or_missing_transport",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
// Another path already reconnected while we yielded — nothing to do.
|
// Another path already reconnected while we yielded — nothing to do.
|
||||||
if (this.currentState !== "disconnected") return;
|
if (this.currentState !== "disconnected") {
|
||||||
|
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
||||||
|
recovery_generation: recoveryGeneration,
|
||||||
|
reason: "state_already_changed",
|
||||||
|
current_state: this.currentState,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
await this.transport.disconnect().catch(() => {});
|
await this.transport.disconnect().catch(() => {});
|
||||||
try {
|
try {
|
||||||
|
this.diagnostics?.("controller.disconnect_cleanup.started", {
|
||||||
|
recovery_generation: recoveryGeneration,
|
||||||
|
current_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
await this.lifecycleHooks.onDisconnected?.();
|
await this.lifecycleHooks.onDisconnected?.();
|
||||||
|
this.diagnostics?.("controller.disconnect_cleanup.completed", {
|
||||||
|
recovery_generation: recoveryGeneration,
|
||||||
|
current_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||||
}
|
}
|
||||||
if (!this.connectionEnabled) return;
|
if (!this.connectionEnabled) return;
|
||||||
try {
|
try {
|
||||||
|
this.diagnostics?.("controller.disconnect_reconnect.started", {
|
||||||
|
recovery_generation: recoveryGeneration,
|
||||||
|
current_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
await this.transport.connect();
|
await this.transport.connect();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
@@ -252,22 +323,39 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
this.disconnectRecovery = recovery.finally(() => {
|
this.disconnectRecovery = recovery.finally(() => {
|
||||||
|
this.diagnostics?.("controller.disconnect_recovery.completed", {
|
||||||
|
connection_generation: this.connectionGeneration,
|
||||||
|
current_state: this.currentState,
|
||||||
|
});
|
||||||
this.disconnectRecovery = null;
|
this.disconnectRecovery = null;
|
||||||
});
|
});
|
||||||
return this.disconnectRecovery;
|
return this.disconnectRecovery;
|
||||||
}
|
}
|
||||||
|
|
||||||
private cancelHandshake(): void {
|
private cancelHandshake(): void {
|
||||||
|
const previousGeneration = this.connectionGeneration;
|
||||||
this.connectionGeneration += 1;
|
this.connectionGeneration += 1;
|
||||||
|
this.diagnostics?.("controller.handshake.cancelled", {
|
||||||
|
previous_generation: previousGeneration,
|
||||||
|
next_generation: this.connectionGeneration,
|
||||||
|
had_abort_controller: this.handshakeAbort !== null,
|
||||||
|
});
|
||||||
this.handshakeAbort?.abort();
|
this.handshakeAbort?.abort();
|
||||||
this.handshakeAbort = null;
|
this.handshakeAbort = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||||
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||||
|
this.diagnostics?.("controller.handshake_retry.scheduled", {
|
||||||
|
delay_ms: HANDSHAKE_RETRY_DELAY_MS,
|
||||||
|
connection_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
this.handshakeRetryTimer = setTimeout(() => {
|
this.handshakeRetryTimer = setTimeout(() => {
|
||||||
this.handshakeRetryTimer = null;
|
this.handshakeRetryTimer = null;
|
||||||
if (!this.connectionEnabled || !this.transport) return;
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
|
this.diagnostics?.("controller.handshake_retry.timer_fired", {
|
||||||
|
connection_generation: this.connectionGeneration,
|
||||||
|
});
|
||||||
void this.transport.connect().catch((err) => {
|
void this.transport.connect().catch((err) => {
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
/**
|
/**
|
||||||
* Build a semantic TargetDescriptor for an interacted element.
|
* Build a semantic capture descriptor for an interacted element.
|
||||||
*
|
*
|
||||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||||
* on screen (usually a short visible name). Tag-only noise like
|
* on screen (usually a short visible name). Tag-only noise like
|
||||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
/** Content-script capture descriptor before VOM geometric matching. */
|
||||||
|
export interface CaptureTargetDescriptor {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
|||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
* Recording “点击div” with no name fails this bar.
|
* Recording “点击div” with no name fails this bar.
|
||||||
*/
|
*/
|
||||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
const name = target.name?.trim();
|
const name = target.name?.trim();
|
||||||
if (name && isActionableLabel(name)) return true;
|
if (name && isActionableLabel(name)) return true;
|
||||||
if (
|
if (
|
||||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeTarget(el: Element): TargetDescriptor {
|
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||||
const tag = el.tagName.toLowerCase();
|
const tag = el.tagName.toLowerCase();
|
||||||
const role = inferRole(el);
|
const role = inferRole(el);
|
||||||
const name = accessibleName(el);
|
const name = accessibleName(el);
|
||||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const clickable = resolveClickableElement(target);
|
const clickable = resolveClickableElement(target);
|
||||||
if (!clickable) return null;
|
if (!clickable) return null;
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
import type { ProtocolFrame } from "@/transport/types";
|
||||||
|
|
||||||
|
const STORAGE_KEY = "bsk_diagnostic_events_v1";
|
||||||
|
const MAX_ENTRIES = 500;
|
||||||
|
|
||||||
|
export type DiagnosticFields = Record<string, unknown>;
|
||||||
|
export type DiagnosticSink = (event: string, fields?: DiagnosticFields) => void;
|
||||||
|
|
||||||
|
export interface DiagnosticEntry {
|
||||||
|
id: string;
|
||||||
|
sequence: number;
|
||||||
|
timestamp: string;
|
||||||
|
monotonic_ms: number;
|
||||||
|
worker_boot_id: string;
|
||||||
|
event: string;
|
||||||
|
fields: DiagnosticFields;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DiagnosticStorage {
|
||||||
|
get(key: string): Promise<Record<string, unknown>>;
|
||||||
|
set(items: Record<string, unknown>): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultStorage(): DiagnosticStorage | null {
|
||||||
|
if (typeof chrome === "undefined" || !chrome.storage?.local) return null;
|
||||||
|
return {
|
||||||
|
get: (key) => chrome.storage.local.get(key),
|
||||||
|
set: (items) => chrome.storage.local.set(items),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeBootId(): string {
|
||||||
|
try {
|
||||||
|
return crypto.randomUUID();
|
||||||
|
} catch {
|
||||||
|
return `worker-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalise(value: unknown): unknown {
|
||||||
|
if (value instanceof Error) {
|
||||||
|
return { name: value.name, message: value.message, stack: value.stack };
|
||||||
|
}
|
||||||
|
if (typeof value === "bigint") return value.toString();
|
||||||
|
if (Array.isArray(value)) return value.map(normalise);
|
||||||
|
if (value && typeof value === "object") {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value as Record<string, unknown>).map(([key, child]) => [
|
||||||
|
key,
|
||||||
|
normalise(child),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persistent diagnostics for the MV3 background worker.
|
||||||
|
*
|
||||||
|
* Events are first written to chrome.storage.local so a worker termination or
|
||||||
|
* dead WebSocket does not erase the evidence. Once the post-handshake link is
|
||||||
|
* ready, pending entries are forwarded to the daemon as one
|
||||||
|
* `browser.diagnostic` event and removed from the local queue.
|
||||||
|
*/
|
||||||
|
export class DiagnosticLogger {
|
||||||
|
readonly workerBootId: string;
|
||||||
|
readonly sink: DiagnosticSink;
|
||||||
|
|
||||||
|
private sequence = 0;
|
||||||
|
private storageQueue: Promise<void> = Promise.resolve();
|
||||||
|
private flushPromise: Promise<void> | null = null;
|
||||||
|
private flushRequested = false;
|
||||||
|
private send: ((frame: ProtocolFrame) => void) | null = null;
|
||||||
|
private transportReady = false;
|
||||||
|
private readonly storage: DiagnosticStorage | null;
|
||||||
|
|
||||||
|
constructor(storage: DiagnosticStorage | null = defaultStorage()) {
|
||||||
|
this.workerBootId = makeBootId();
|
||||||
|
this.storage = storage;
|
||||||
|
this.sink = (event, fields = {}) => this.record(event, fields);
|
||||||
|
}
|
||||||
|
|
||||||
|
bindTransport(send: (frame: ProtocolFrame) => void): void {
|
||||||
|
this.send = send;
|
||||||
|
}
|
||||||
|
|
||||||
|
setTransportReady(ready: boolean): void {
|
||||||
|
this.transportReady = ready;
|
||||||
|
if (ready) void this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
record(event: string, fields: DiagnosticFields = {}): void {
|
||||||
|
const sequence = ++this.sequence;
|
||||||
|
const entry: DiagnosticEntry = {
|
||||||
|
id: `${this.workerBootId}:${sequence}`,
|
||||||
|
sequence,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
monotonic_ms: Math.round(performance.now()),
|
||||||
|
worker_boot_id: this.workerBootId,
|
||||||
|
event,
|
||||||
|
fields: normalise(fields) as DiagnosticFields,
|
||||||
|
};
|
||||||
|
console.info("[bsk diagnostic]", entry);
|
||||||
|
this.storageQueue = this.storageQueue
|
||||||
|
.then(async () => {
|
||||||
|
if (!this.storage) return;
|
||||||
|
const stored = await this.storage.get(STORAGE_KEY);
|
||||||
|
const previous = Array.isArray(stored[STORAGE_KEY])
|
||||||
|
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
||||||
|
: [];
|
||||||
|
await this.storage.set({
|
||||||
|
[STORAGE_KEY]: [...previous, entry].slice(-MAX_ENTRIES),
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.warn("[bsk diagnostic] failed to persist event", error);
|
||||||
|
});
|
||||||
|
if (this.transportReady) void this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
flush(): Promise<void> {
|
||||||
|
if (this.flushPromise) {
|
||||||
|
this.flushRequested = true;
|
||||||
|
return this.flushPromise;
|
||||||
|
}
|
||||||
|
const running = (async () => {
|
||||||
|
await this.storageQueue;
|
||||||
|
if (!this.transportReady || !this.send || !this.storage) return;
|
||||||
|
const stored = await this.storage.get(STORAGE_KEY);
|
||||||
|
const entries = Array.isArray(stored[STORAGE_KEY])
|
||||||
|
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
||||||
|
: [];
|
||||||
|
if (entries.length === 0) return;
|
||||||
|
try {
|
||||||
|
this.send({
|
||||||
|
event: "browser.diagnostic",
|
||||||
|
payload: {
|
||||||
|
schema_version: 1,
|
||||||
|
worker_boot_id: this.workerBootId,
|
||||||
|
entries,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[bsk diagnostic] daemon flush failed", error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const sentIds = new Set(entries.map((entry) => entry.id));
|
||||||
|
// Serialize acknowledgement behind every record write that may have
|
||||||
|
// started while send() was in progress. Without this, a concurrent
|
||||||
|
// record can read the pre-ack queue and write an already-sent entry
|
||||||
|
// back into storage, causing the next batch to duplicate it.
|
||||||
|
this.storageQueue = this.storageQueue
|
||||||
|
.then(async () => {
|
||||||
|
const current = await this.storage?.get(STORAGE_KEY);
|
||||||
|
const currentEntries = Array.isArray(current?.[STORAGE_KEY])
|
||||||
|
? (current[STORAGE_KEY] as DiagnosticEntry[])
|
||||||
|
: [];
|
||||||
|
await this.storage?.set({
|
||||||
|
[STORAGE_KEY]: currentEntries.filter((entry) => !sentIds.has(entry.id)),
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.warn("[bsk diagnostic] failed to acknowledge event batch", error);
|
||||||
|
});
|
||||||
|
await this.storageQueue;
|
||||||
|
})().catch((error) => {
|
||||||
|
console.warn("[bsk diagnostic] flush failed", error);
|
||||||
|
});
|
||||||
|
this.flushPromise = running.finally(() => {
|
||||||
|
this.flushPromise = null;
|
||||||
|
if (this.flushRequested && this.transportReady) {
|
||||||
|
this.flushRequested = false;
|
||||||
|
void this.flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return this.flushPromise;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -21,6 +21,7 @@
|
|||||||
* after the handshake has landed.
|
* after the handshake has landed.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import type { ProtocolFrame } from "@/transport/types";
|
import type { ProtocolFrame } from "@/transport/types";
|
||||||
|
|
||||||
export const HEARTBEAT_EVENT = "system.heartbeat";
|
export const HEARTBEAT_EVENT = "system.heartbeat";
|
||||||
@@ -46,6 +47,7 @@ export interface HeartbeatOptions {
|
|||||||
intervalMs?: number;
|
intervalMs?: number;
|
||||||
/** Inject fake timers (tests). */
|
/** Inject fake timers (tests). */
|
||||||
timers?: HeartbeatTimers;
|
timers?: HeartbeatTimers;
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface HeartbeatHandle {
|
export interface HeartbeatHandle {
|
||||||
@@ -74,7 +76,9 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
|||||||
const beat = () => {
|
const beat = () => {
|
||||||
try {
|
try {
|
||||||
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
||||||
|
options.diagnostics?.("heartbeat.sent");
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
options.diagnostics?.("heartbeat.send_failed", { error: err });
|
||||||
// The socket raced a close between the state change and this
|
// The socket raced a close between the state change and this
|
||||||
// tick. The transport's own reconnect path recovers; swallow so
|
// tick. The transport's own reconnect path recovers; swallow so
|
||||||
// the interval keeps running (it is cleared on the disconnect
|
// the interval keeps running (it is cleared on the disconnect
|
||||||
@@ -87,12 +91,14 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
|||||||
if (timer !== null) {
|
if (timer !== null) {
|
||||||
timers.clearInterval(timer);
|
timers.clearInterval(timer);
|
||||||
timer = null;
|
timer = null;
|
||||||
|
options.diagnostics?.("heartbeat.timer.stopped");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const start = () => {
|
const start = () => {
|
||||||
if (timer !== null) return;
|
if (timer !== null) return;
|
||||||
timer = timers.setInterval(beat, intervalMs);
|
timer = timers.setInterval(beat, intervalMs);
|
||||||
|
options.diagnostics?.("heartbeat.timer.started", { interval_ms: intervalMs });
|
||||||
};
|
};
|
||||||
|
|
||||||
const unsubscribe = options.onActiveChange((active) => {
|
const unsubscribe = options.onActiveChange((active) => {
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
* SW idle threshold.
|
* SW idle threshold.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
|
|
||||||
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
|
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
|
||||||
@@ -59,6 +60,7 @@ export interface KeepaliveOptions {
|
|||||||
alarmName?: string;
|
alarmName?: string;
|
||||||
/** Override the alarm period (tests). */
|
/** Override the alarm period (tests). */
|
||||||
periodMin?: number;
|
periodMin?: number;
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface KeepaliveHandle {
|
export interface KeepaliveHandle {
|
||||||
@@ -79,11 +81,24 @@ export function startKeepalive(options: KeepaliveOptions): KeepaliveHandle {
|
|||||||
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
|
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
|
||||||
|
|
||||||
const tick = async () => {
|
const tick = async () => {
|
||||||
if (options.shouldConnect && !options.shouldConnect()) return;
|
if (options.shouldConnect && !options.shouldConnect()) {
|
||||||
if (options.transport.state === "connected") return;
|
options.diagnostics?.("keepalive.tick.skipped", { reason: "connection_disabled" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (options.transport.state === "connected") {
|
||||||
|
options.diagnostics?.("keepalive.tick.healthy", { transport_state: "connected" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
options.diagnostics?.("keepalive.reconnect.started", {
|
||||||
|
transport_state: options.transport.state,
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
await options.transport.connect();
|
await options.transport.connect();
|
||||||
|
options.diagnostics?.("keepalive.reconnect.completed", {
|
||||||
|
transport_state: options.transport.state,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
options.diagnostics?.("keepalive.reconnect.failed", { error: err });
|
||||||
console.debug("[bsk keepalive] connect attempt failed", err);
|
console.debug("[bsk keepalive] connect attempt failed", err);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
* service worker and a tab's content script.
|
* service worker and a tab's content script.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -16,6 +16,10 @@ export interface RecordStartAck {
|
|||||||
ok: true;
|
ok: true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordStepAck =
|
||||||
|
| { ok: true; sequence: number }
|
||||||
|
| { ok: false; expectedSequence: number; error: string };
|
||||||
|
|
||||||
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
||||||
|
|
||||||
export interface RecordQueryMessage {
|
export interface RecordQueryMessage {
|
||||||
@@ -42,7 +46,7 @@ export interface RecordStartMessage {
|
|||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||||
target?: TargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||||
@@ -50,17 +54,30 @@ export interface RecordStepPayload {
|
|||||||
labels?: string[];
|
labels?: string[];
|
||||||
url?: string;
|
url?: string;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
|
commit?: "enter" | "suggestion" | "blur";
|
||||||
/** Page URL when the step was captured. */
|
/** Page URL when the step was captured. */
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
/** Event-target bounds in the source frame's viewport coordinate space. */
|
||||||
|
geometry?: {
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
tag: string;
|
||||||
|
};
|
||||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||||
expects_navigation?: boolean;
|
expects_navigation?: boolean;
|
||||||
/** Whether an observed URL change was synchronously caused by the action. */
|
/** Whether an observed URL change was synchronously caused by the action. */
|
||||||
navigation_caused_by_action?: boolean;
|
navigation_caused_by_action?: boolean;
|
||||||
|
/** Raw webNavigation transition metadata for cause mapping. */
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepMessage {
|
export interface RecordStepMessage {
|
||||||
type: typeof RECORD_STEP;
|
type: typeof RECORD_STEP;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
|
/** Stable for the lifetime of one content-script document. */
|
||||||
|
producerId: string;
|
||||||
|
/** Monotonic within one content-script producer and recording request. */
|
||||||
|
sequence: number;
|
||||||
step: RecordStepPayload;
|
step: RecordStepPayload;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,8 +129,27 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
|||||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
|
if (
|
||||||
|
m.type !== RECORD_STEP ||
|
||||||
|
typeof m.requestId !== "string" ||
|
||||||
|
typeof m.producerId !== "string" ||
|
||||||
|
m.producerId.length === 0 ||
|
||||||
|
typeof m.sequence !== "number" ||
|
||||||
|
!Number.isSafeInteger(m.sequence) ||
|
||||||
|
m.sequence < 1
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
const step = m.step;
|
const step = m.step;
|
||||||
if (typeof step !== "object" || step === null) return false;
|
if (typeof step !== "object" || step === null) return false;
|
||||||
return typeof (step as RecordStepPayload).op === "string";
|
return typeof (step as RecordStepPayload).op === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isAcceptedRecordStepAck(
|
||||||
|
value: unknown,
|
||||||
|
sequence: number,
|
||||||
|
): value is Extract<RecordStepAck, { ok: true }> {
|
||||||
|
if (typeof value !== "object" || value === null) return false;
|
||||||
|
const ack = value as Partial<RecordStepAck>;
|
||||||
|
return ack.ok === true && ack.sequence === sequence;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,136 +0,0 @@
|
|||||||
import type { DraftTraceStep } from "@/transport/types";
|
|
||||||
import type { RecordStepPayload } from "./record-bridge";
|
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
|
||||||
steps: DraftTraceStep[];
|
|
||||||
currentUrl?: string;
|
|
||||||
pendingNavigation: boolean;
|
|
||||||
pendingNavigationDeadline?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
|
||||||
|
|
||||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|
||||||
const pageUrl = payload.page_url;
|
|
||||||
switch (payload.op) {
|
|
||||||
case "click":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "click",
|
|
||||||
target: payload.target,
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "hover":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "hover",
|
|
||||||
target: payload.target,
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "fill":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "fill",
|
|
||||||
target: payload.target,
|
|
||||||
value: payload.value ?? "",
|
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "press":
|
|
||||||
return payload.key
|
|
||||||
? {
|
|
||||||
op: "press",
|
|
||||||
key: payload.key,
|
|
||||||
...(payload.target ? { target: payload.target } : {}),
|
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "select":
|
|
||||||
return payload.target && payload.values
|
|
||||||
? {
|
|
||||||
op: "select",
|
|
||||||
target: payload.target,
|
|
||||||
values: payload.values,
|
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "navigate":
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
|
||||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
|
||||||
const step = buffer.steps[i];
|
|
||||||
if (!step) continue;
|
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
|
||||||
buffer.steps[i] = { ...step, navigated_to: url };
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function observeRecordedNavigation(
|
|
||||||
buffer: RecordingStepBuffer,
|
|
||||||
url: string,
|
|
||||||
causedByAction?: boolean,
|
|
||||||
): void {
|
|
||||||
if (!url || url === buffer.currentUrl) return;
|
|
||||||
buffer.currentUrl = url;
|
|
||||||
const pendingIsCurrent =
|
|
||||||
buffer.pendingNavigation &&
|
|
||||||
(buffer.pendingNavigationDeadline === undefined ||
|
|
||||||
buffer.pendingNavigationDeadline >= Date.now());
|
|
||||||
|
|
||||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
|
||||||
buffer.pendingNavigation = false;
|
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
|
||||||
if (!annotateLastStepNavigation(buffer, url)) {
|
|
||||||
buffer.steps.push({
|
|
||||||
op: "navigate",
|
|
||||||
url,
|
|
||||||
page_url: url,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
buffer.pendingNavigation = false;
|
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
|
||||||
buffer.steps.push({
|
|
||||||
op: "navigate",
|
|
||||||
url,
|
|
||||||
page_url: url,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function appendRecordedPayload(
|
|
||||||
buffer: RecordingStepBuffer,
|
|
||||||
payload: RecordStepPayload,
|
|
||||||
): void {
|
|
||||||
if (payload.op === "navigate") {
|
|
||||||
if (payload.url) {
|
|
||||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const step = toDraftStep({
|
|
||||||
...payload,
|
|
||||||
page_url: payload.page_url ?? buffer.currentUrl,
|
|
||||||
});
|
|
||||||
if (!step) return;
|
|
||||||
buffer.steps.push(step);
|
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
|
||||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
|
||||||
buffer.pendingNavigationDeadline = buffer.pendingNavigation
|
|
||||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
|
||||||
: undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import {
|
||||||
|
RECORD_DOCUMENT_ATTRIBUTE,
|
||||||
|
recordingDocumentMarkerValue,
|
||||||
|
} from "@/shared/recording-document-identity";
|
||||||
|
|
||||||
|
export interface RecordingDocumentMarker {
|
||||||
|
restore(): void;
|
||||||
|
ensure(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function waitForDocumentElement(): Promise<HTMLElement> {
|
||||||
|
if (document.documentElement) return Promise.resolve(document.documentElement);
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
if (!document.documentElement) return;
|
||||||
|
observer.disconnect();
|
||||||
|
resolve(document.documentElement);
|
||||||
|
});
|
||||||
|
observer.observe(document, { childList: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markRecordingDocument(
|
||||||
|
producerId: string,
|
||||||
|
root: HTMLElement = document.documentElement,
|
||||||
|
): RecordingDocumentMarker {
|
||||||
|
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
const markerValue = recordingDocumentMarkerValue(producerId);
|
||||||
|
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
|
||||||
|
ensure();
|
||||||
|
return {
|
||||||
|
ensure,
|
||||||
|
restore() {
|
||||||
|
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
|
||||||
|
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||||
|
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
||||||
|
|
||||||
|
const SETTLE_MIN_MS = 150;
|
||||||
|
const SETTLE_QUIET_MS = 250;
|
||||||
|
const SETTLE_MAX_MS = 2_000;
|
||||||
|
const SETTLE_POLL_MS = 60;
|
||||||
|
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
||||||
|
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export interface DocumentSettleScope {
|
||||||
|
target: CdpTarget;
|
||||||
|
/** Omit only for the target's root document. */
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProbeContext {
|
||||||
|
executionContextId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executionContextForScope(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
): Promise<number | undefined> {
|
||||||
|
if (!scope.frameId) return undefined;
|
||||||
|
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
{
|
||||||
|
frameId: scope.frameId,
|
||||||
|
worldName: SETTLE_WORLD_NAME,
|
||||||
|
grantUniveralAccess: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return result.executionContextId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
context: ProbeContext,
|
||||||
|
): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
if (scope.frameId && context.executionContextId === undefined) {
|
||||||
|
context.executionContextId = await executionContextForScope(cdp, scope);
|
||||||
|
if (context.executionContextId === undefined) return null;
|
||||||
|
}
|
||||||
|
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
...(context.executionContextId !== undefined
|
||||||
|
? { contextId: context.executionContextId }
|
||||||
|
: {}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
context.executionContextId = undefined;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForDocumentSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
options: { signal?: AbortSignal } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
const context: ProbeContext = {};
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS, options.signal);
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, scope, context);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe || probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
|
|
||||||
|
export function shouldRecordPress(
|
||||||
|
key: string,
|
||||||
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||||
|
): boolean {
|
||||||
|
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||||
|
const mods = modifiers ?? [];
|
||||||
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
||||||
|
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveDraftStartUrl(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
explicitStartUrl?: string,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): string {
|
||||||
|
if (explicitStartUrl) return explicitStartUrl;
|
||||||
|
const navigation = drafts.find(
|
||||||
|
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
||||||
|
);
|
||||||
|
if (navigation) return navigation.url;
|
||||||
|
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
export const RECORD_FRAME_PORT = "bsk-record-frame";
|
||||||
|
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
|
||||||
|
export const RECORD_FRAME_START = "bsk-record-frame-start";
|
||||||
|
|
||||||
|
export interface RecordFrameQueryMessage {
|
||||||
|
type: typeof RECORD_FRAME_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordFrameQueryResponse {
|
||||||
|
active: boolean;
|
||||||
|
requestId?: string;
|
||||||
|
startedAtMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordFrameStartMessage {
|
||||||
|
type: typeof RECORD_FRAME_START;
|
||||||
|
requestId: string;
|
||||||
|
startedAtMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordFramePortMessage =
|
||||||
|
| {
|
||||||
|
type: "ready";
|
||||||
|
requestId: string;
|
||||||
|
producerId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "ready_ack";
|
||||||
|
requestId: string;
|
||||||
|
producerId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "stop";
|
||||||
|
requestId: string;
|
||||||
|
commandId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "cancel";
|
||||||
|
requestId: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "stopped";
|
||||||
|
requestId: string;
|
||||||
|
commandId: string;
|
||||||
|
ok: boolean;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
|
||||||
|
return (
|
||||||
|
typeof value === "object" &&
|
||||||
|
value !== null &&
|
||||||
|
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
|
||||||
|
if (typeof value !== "object" || value === null) return false;
|
||||||
|
const message = value as Partial<RecordFrameStartMessage>;
|
||||||
|
return (
|
||||||
|
message.type === RECORD_FRAME_START &&
|
||||||
|
typeof message.requestId === "string" &&
|
||||||
|
typeof message.startedAtMs === "number"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,399 @@
|
|||||||
|
import {
|
||||||
|
isRecordFrameQueryMessage,
|
||||||
|
RECORD_FRAME_PORT,
|
||||||
|
RECORD_FRAME_START,
|
||||||
|
type RecordFramePortMessage,
|
||||||
|
type RecordFrameQueryResponse,
|
||||||
|
type RecordFrameStartMessage,
|
||||||
|
} from "./frame-bridge";
|
||||||
|
|
||||||
|
export interface RecordingCaptureScope {
|
||||||
|
tabId: number;
|
||||||
|
documentId: string;
|
||||||
|
browserFrameId: number;
|
||||||
|
producerId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ArmedRecording {
|
||||||
|
requestId: string;
|
||||||
|
startedAtMs: number;
|
||||||
|
tabIds: Set<number>;
|
||||||
|
agents: Map<string, FrameAgent>;
|
||||||
|
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
|
||||||
|
finishing: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FrameAgent extends RecordingCaptureScope {
|
||||||
|
requestId: string;
|
||||||
|
port: chrome.runtime.Port;
|
||||||
|
stopWaiters: Map<string, (ok: boolean) => void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BrowserFrame {
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BrowserFrameNavigation extends BrowserFrame {
|
||||||
|
tabId: number;
|
||||||
|
lifecycle: "committed" | "completed";
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
|
||||||
|
|
||||||
|
export interface RecordFrameCoordinatorDeps {
|
||||||
|
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
|
||||||
|
sendToDocument(
|
||||||
|
tabId: number,
|
||||||
|
message: RecordFrameStartMessage,
|
||||||
|
target: { documentId?: string; frameId?: number },
|
||||||
|
): Promise<unknown>;
|
||||||
|
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function documentKey(tabId: number, documentId: string): string {
|
||||||
|
return `${tabId}:${documentId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function senderAddress(
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
): { tabId: number; documentId: string; browserFrameId: number } | null {
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
const documentId = sender.documentId;
|
||||||
|
const browserFrameId = sender.frameId;
|
||||||
|
if (
|
||||||
|
typeof tabId !== "number" ||
|
||||||
|
typeof documentId !== "string" ||
|
||||||
|
typeof browserFrameId !== "number"
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { tabId, documentId, browserFrameId };
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultDeps(): RecordFrameCoordinatorDeps {
|
||||||
|
return {
|
||||||
|
async getAllFrames(tabId) {
|
||||||
|
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
|
||||||
|
},
|
||||||
|
sendToDocument(tabId, message, target) {
|
||||||
|
return chrome.tabs.sendMessage(tabId, message, target);
|
||||||
|
},
|
||||||
|
subscribeFrameNavigation(listener) {
|
||||||
|
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
|
||||||
|
listener({ ...details, lifecycle: "committed" });
|
||||||
|
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
|
||||||
|
listener({ ...details, lifecycle: "completed" });
|
||||||
|
chrome.webNavigation.onCommitted.addListener(onCommitted);
|
||||||
|
chrome.webNavigation.onCompleted.addListener(onCompleted);
|
||||||
|
return () => {
|
||||||
|
chrome.webNavigation.onCommitted.removeListener(onCommitted);
|
||||||
|
chrome.webNavigation.onCompleted.removeListener(onCompleted);
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordFrameCoordinator {
|
||||||
|
readonly #deps: RecordFrameCoordinatorDeps;
|
||||||
|
readonly #recordings = new Map<string, ArmedRecording>();
|
||||||
|
#attached = false;
|
||||||
|
#detachFrameNavigation: (() => void) | null = null;
|
||||||
|
|
||||||
|
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
|
||||||
|
this.#deps = deps;
|
||||||
|
}
|
||||||
|
|
||||||
|
attach(): () => void {
|
||||||
|
if (this.#attached) return () => {};
|
||||||
|
this.#attached = true;
|
||||||
|
chrome.runtime.onConnect.addListener(this.#onConnect);
|
||||||
|
chrome.runtime.onMessage.addListener(this.#onMessage);
|
||||||
|
this.#detachFrameNavigation =
|
||||||
|
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
|
||||||
|
return () => {
|
||||||
|
if (!this.#attached) return;
|
||||||
|
this.#attached = false;
|
||||||
|
chrome.runtime.onConnect.removeListener(this.#onConnect);
|
||||||
|
chrome.runtime.onMessage.removeListener(this.#onMessage);
|
||||||
|
this.#detachFrameNavigation?.();
|
||||||
|
this.#detachFrameNavigation = null;
|
||||||
|
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
|
||||||
|
this.#recordings.clear();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
begin(
|
||||||
|
requestId: string,
|
||||||
|
startedAtMs: number,
|
||||||
|
initialTabId: number,
|
||||||
|
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
|
||||||
|
): void {
|
||||||
|
const previous = this.#recordings.get(requestId);
|
||||||
|
if (previous) this.#cancelAgents(previous);
|
||||||
|
this.#recordings.set(requestId, {
|
||||||
|
requestId,
|
||||||
|
startedAtMs,
|
||||||
|
// Register the initial tab before navigation begins. A newly-created
|
||||||
|
// child document can query at document_start before armTab's frame
|
||||||
|
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
|
||||||
|
tabIds: new Set([initialTabId]),
|
||||||
|
agents: new Map(),
|
||||||
|
...(onDocumentReady ? { onDocumentReady } : {}),
|
||||||
|
finishing: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async armTab(requestId: string, tabId: number): Promise<boolean> {
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!recording || recording.finishing) return false;
|
||||||
|
recording.tabIds.add(tabId);
|
||||||
|
|
||||||
|
let frames: BrowserFrame[];
|
||||||
|
try {
|
||||||
|
frames = await this.#deps.getAllFrames(tabId);
|
||||||
|
} catch {
|
||||||
|
frames = [{ frameId: 0 }];
|
||||||
|
}
|
||||||
|
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
|
||||||
|
|
||||||
|
const results = await Promise.all(
|
||||||
|
frames.map(async (frame) => {
|
||||||
|
const started = await this.#startFrame(recording, tabId, frame);
|
||||||
|
return { frameId: frame.frameId, started };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return results.some((result) => result.frameId === 0 && result.started);
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceFor(
|
||||||
|
requestId: string,
|
||||||
|
producerId: string,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
): RecordingCaptureScope | null {
|
||||||
|
const address = senderAddress(sender);
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!address || !recording) return null;
|
||||||
|
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
|
||||||
|
if (!agent || agent.producerId !== producerId) return null;
|
||||||
|
return {
|
||||||
|
tabId: agent.tabId,
|
||||||
|
documentId: agent.documentId,
|
||||||
|
browserFrameId: agent.browserFrameId,
|
||||||
|
producerId: agent.producerId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async stop(requestId: string): Promise<boolean> {
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!recording) return true;
|
||||||
|
recording.finishing = true;
|
||||||
|
const commandId = crypto.randomUUID();
|
||||||
|
const results = await Promise.all(
|
||||||
|
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||||
|
);
|
||||||
|
const succeeded = results.every(Boolean);
|
||||||
|
if (succeeded) this.#recordings.delete(requestId);
|
||||||
|
else recording.finishing = false;
|
||||||
|
return succeeded;
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(requestId: string): void {
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (!recording) return;
|
||||||
|
this.#cancelAgents(recording);
|
||||||
|
this.#recordings.delete(requestId);
|
||||||
|
}
|
||||||
|
|
||||||
|
readonly #onMessage = (
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: RecordFrameQueryResponse) => void,
|
||||||
|
): boolean => {
|
||||||
|
if (!isRecordFrameQueryMessage(message)) return false;
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
const recording =
|
||||||
|
typeof tabId === "number"
|
||||||
|
? [...this.#recordings.values()].find(
|
||||||
|
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
|
||||||
|
)
|
||||||
|
: undefined;
|
||||||
|
sendResponse(
|
||||||
|
recording
|
||||||
|
? {
|
||||||
|
active: true,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
}
|
||||||
|
: { active: false },
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
|
||||||
|
if (frame.frameId === 0) return;
|
||||||
|
for (const recording of this.#recordings.values()) {
|
||||||
|
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
|
||||||
|
// onCommitted can race document_start, while onCompleted runs after the
|
||||||
|
// content script is available. Starting twice is safe and closes both
|
||||||
|
// the early-query and late-SPA-frame gaps.
|
||||||
|
void (async () => {
|
||||||
|
const started = await this.#startFrame(recording, frame.tabId, frame);
|
||||||
|
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
|
||||||
|
const agent = frame.documentId
|
||||||
|
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
|
||||||
|
: [...recording.agents.values()].find(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
|
||||||
|
);
|
||||||
|
if (!agent) return;
|
||||||
|
await recording.onDocumentReady?.({
|
||||||
|
tabId: agent.tabId,
|
||||||
|
documentId: agent.documentId,
|
||||||
|
browserFrameId: agent.browserFrameId,
|
||||||
|
producerId: agent.producerId,
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
readonly #onConnect = (port: chrome.runtime.Port): void => {
|
||||||
|
if (port.name !== RECORD_FRAME_PORT) return;
|
||||||
|
const address = senderAddress(port.sender ?? {});
|
||||||
|
if (!address) {
|
||||||
|
port.disconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let registered: FrameAgent | null = null;
|
||||||
|
port.onMessage.addListener((raw: unknown) => {
|
||||||
|
const message = raw as Partial<RecordFramePortMessage>;
|
||||||
|
if (message.type === "ready") {
|
||||||
|
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
|
||||||
|
port.disconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const requestId = message.requestId;
|
||||||
|
const producerId = message.producerId;
|
||||||
|
if (registered) {
|
||||||
|
if (registered.requestId === requestId && registered.producerId === producerId) {
|
||||||
|
port.postMessage({
|
||||||
|
type: "ready_ack",
|
||||||
|
requestId,
|
||||||
|
producerId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} else {
|
||||||
|
port.disconnect();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const recording = this.#recordings.get(requestId);
|
||||||
|
if (
|
||||||
|
!recording ||
|
||||||
|
recording.finishing ||
|
||||||
|
!recording.tabIds.has(address.tabId) ||
|
||||||
|
producerId.length === 0
|
||||||
|
) {
|
||||||
|
port.disconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const agent: FrameAgent = {
|
||||||
|
...address,
|
||||||
|
requestId,
|
||||||
|
producerId,
|
||||||
|
port,
|
||||||
|
stopWaiters: new Map(),
|
||||||
|
};
|
||||||
|
registered = agent;
|
||||||
|
const key = documentKey(address.tabId, address.documentId);
|
||||||
|
const previous = recording.agents.get(key);
|
||||||
|
if (previous && previous !== agent) previous.port.disconnect();
|
||||||
|
recording.agents.set(key, agent);
|
||||||
|
port.postMessage({
|
||||||
|
type: "ready_ack",
|
||||||
|
requestId,
|
||||||
|
producerId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
message.type === "stopped" &&
|
||||||
|
registered &&
|
||||||
|
message.requestId === registered.requestId &&
|
||||||
|
typeof message.commandId === "string"
|
||||||
|
) {
|
||||||
|
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
|
||||||
|
registered.stopWaiters.delete(message.commandId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
port.onDisconnect.addListener(() => {
|
||||||
|
if (!registered) return;
|
||||||
|
for (const resolve of registered.stopWaiters.values()) resolve(true);
|
||||||
|
const recording = this.#recordings.get(registered.requestId);
|
||||||
|
const key = documentKey(registered.tabId, registered.documentId);
|
||||||
|
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
agent.stopWaiters.delete(commandId);
|
||||||
|
resolve(false);
|
||||||
|
}, RECORD_FRAME_STOP_TIMEOUT_MS);
|
||||||
|
agent.stopWaiters.set(commandId, (ok) => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(ok);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
agent.port.postMessage({
|
||||||
|
type: "stop",
|
||||||
|
requestId,
|
||||||
|
commandId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} catch {
|
||||||
|
clearTimeout(timer);
|
||||||
|
agent.stopWaiters.delete(commandId);
|
||||||
|
resolve(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #startFrame(
|
||||||
|
recording: ArmedRecording,
|
||||||
|
tabId: number,
|
||||||
|
frame: BrowserFrame,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const message: RecordFrameStartMessage = {
|
||||||
|
type: RECORD_FRAME_START,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const response = await this.#deps.sendToDocument(tabId, message, {
|
||||||
|
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
|
||||||
|
});
|
||||||
|
return isStarted(response);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#cancelAgents(recording: ArmedRecording): void {
|
||||||
|
for (const agent of recording.agents.values()) {
|
||||||
|
try {
|
||||||
|
agent.port.postMessage({
|
||||||
|
type: "cancel",
|
||||||
|
requestId: recording.requestId,
|
||||||
|
} satisfies RecordFramePortMessage);
|
||||||
|
} catch {
|
||||||
|
// A destroyed Document has no remaining capture state to cancel.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isStarted(value: unknown): boolean {
|
||||||
|
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const recordFrameCoordinator = new RecordFrameCoordinator();
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
|
||||||
|
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
|
||||||
|
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import {
|
||||||
|
type CaptureVomMatchNode,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
|
||||||
|
export interface IndexedObservationNode {
|
||||||
|
frameId: string;
|
||||||
|
geometry: CaptureVomMatchNode;
|
||||||
|
ref?: RenderedRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CapturedRecordingObservation {
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RegisteredObservation {
|
||||||
|
stateId: string;
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingDocumentScope {
|
||||||
|
frameId: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||||
|
return `${frameId}:${backendNodeId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameTagKey(frameId: string, tag: string): string {
|
||||||
|
return `${frameId}:${tag.toLowerCase()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ObservationNodeIndex {
|
||||||
|
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||||
|
readonly #refById = new Map<string, RenderedRef>();
|
||||||
|
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||||
|
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
|
||||||
|
Partial<Pick<CaptureVomObservationResult, "frames">>,
|
||||||
|
) {
|
||||||
|
const refByNode = new Map<string, RenderedRef>();
|
||||||
|
for (const ref of input.refs) {
|
||||||
|
const frameId = ref.frameId ?? input.rootFrameId;
|
||||||
|
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
||||||
|
this.#refById.set(ref.ref, ref);
|
||||||
|
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
frameRefs.push(ref);
|
||||||
|
this.#refsByFrame.set(frameId, frameRefs);
|
||||||
|
}
|
||||||
|
for (const frame of input.frames ?? []) {
|
||||||
|
const documentId = frame.recordingDocumentId;
|
||||||
|
if (!documentId) continue;
|
||||||
|
this.#scopeByProducer.set(
|
||||||
|
documentId,
|
||||||
|
this.#scopeByProducer.has(documentId)
|
||||||
|
? null
|
||||||
|
: { frameId: frame.frameId, target: frame.target },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const geometry of input.matchNodes) {
|
||||||
|
const { frameId } = geometry;
|
||||||
|
const entry = {
|
||||||
|
frameId,
|
||||||
|
geometry,
|
||||||
|
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
||||||
|
};
|
||||||
|
const key = frameTagKey(frameId, geometry.tag);
|
||||||
|
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
||||||
|
bucket.push(entry);
|
||||||
|
this.#nodesByFrameTag.set(key, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
||||||
|
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
ref(refId: string): RenderedRef | undefined {
|
||||||
|
return this.#refById.get(refId);
|
||||||
|
}
|
||||||
|
|
||||||
|
refs(frameId: string): readonly RenderedRef[] {
|
||||||
|
return this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
documentScope(producerId: string): RecordingDocumentScope | undefined {
|
||||||
|
return this.#scopeByProducer.get(producerId) ?? undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureRecordingObservation(input: {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<CapturedRecordingObservation> {
|
||||||
|
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
||||||
|
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
||||||
|
maxTokens: input.maxTokens,
|
||||||
|
redactValues: input.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
signal: input.signal,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: new ObservationNodeIndex(captured),
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
vomText: captured.text,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
||||||
|
import { RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
||||||
|
|
||||||
|
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
||||||
|
const MIN_CAPTURE_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
export interface TabObservationCursor {
|
||||||
|
lastSettled: RegisteredObservation | null;
|
||||||
|
lastCaptureAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||||
|
return (
|
||||||
|
draft.op === "click" ||
|
||||||
|
draft.op === "hover" ||
|
||||||
|
draft.op === "fill" ||
|
||||||
|
draft.op === "press" ||
|
||||||
|
draft.op === "select"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingObservationSession {
|
||||||
|
readonly registry: RecordingStateRegistry;
|
||||||
|
readonly cursor: TabObservationCursor;
|
||||||
|
readonly annotations: StepAnnotation[];
|
||||||
|
readonly #maxTokens: number;
|
||||||
|
readonly #redactValues: boolean;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
options: {
|
||||||
|
registry?: RecordingStateRegistry;
|
||||||
|
cursor?: TabObservationCursor;
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
maxTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
this.registry = options.registry ?? new RecordingStateRegistry();
|
||||||
|
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
||||||
|
this.annotations = options.annotations ?? [];
|
||||||
|
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
||||||
|
this.#redactValues = options.redactValues ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async capture(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RegisteredObservation> {
|
||||||
|
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
||||||
|
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
||||||
|
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
|
const captured = await captureRecordingObservation({
|
||||||
|
cdp,
|
||||||
|
tabsApi,
|
||||||
|
tabId,
|
||||||
|
maxTokens: this.#maxTokens,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const state = this.registry.register({
|
||||||
|
url: captured.url,
|
||||||
|
title: captured.title,
|
||||||
|
vomText: captured.vomText,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
});
|
||||||
|
const observation: RegisteredObservation = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: captured.index,
|
||||||
|
url: captured.url,
|
||||||
|
};
|
||||||
|
this.cursor.lastSettled = observation;
|
||||||
|
this.cursor.lastCaptureAt = Date.now();
|
||||||
|
return observation;
|
||||||
|
}
|
||||||
|
|
||||||
|
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
||||||
|
const observation = this.cursor.lastSettled;
|
||||||
|
if (isTargeted(draft)) {
|
||||||
|
draft.matchedTarget = observation
|
||||||
|
? matchObservationTarget({
|
||||||
|
observation,
|
||||||
|
hint: draft.targetHint,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: unmatchedTarget(draft.captureTarget);
|
||||||
|
}
|
||||||
|
if (!observation) return;
|
||||||
|
|
||||||
|
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
||||||
|
if (previousActionPending && unmatched) return;
|
||||||
|
draft.preStateId = observation.stateId;
|
||||||
|
this.registry.markStep(observation.stateId, draftId);
|
||||||
|
|
||||||
|
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
||||||
|
const ref = observation.index.ref(draft.matchedTarget.ref);
|
||||||
|
if (!ref) return;
|
||||||
|
this.annotations.push({
|
||||||
|
draftId,
|
||||||
|
op: draft.op,
|
||||||
|
line: ref.line,
|
||||||
|
stateId: observation.stateId,
|
||||||
|
...(draft.op === "fill" && !this.#redactValues
|
||||||
|
? { detail: JSON.stringify(draft.value) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import type { StopReason, TraceV3 } from "@/transport/types";
|
||||||
|
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||||
|
import type { RegisteredObservation } from "./observation-capture";
|
||||||
|
import { RecordingObservationSession } from "./observation-session";
|
||||||
|
import { inferMissingPostStates, SettleController } from "./settle-controller";
|
||||||
|
import { RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { buildTraceV3 } from "./trace-builder-v3";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
interface TabRecordingContext {
|
||||||
|
session: RecordingObservationSession;
|
||||||
|
settle: SettleController;
|
||||||
|
pendingCapture: PendingCapture | null;
|
||||||
|
frameRefresh: Promise<void>;
|
||||||
|
frameRefreshAbort: AbortController | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingCapture {
|
||||||
|
promise: Promise<RegisteredObservation>;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingObservationRuntime {
|
||||||
|
readonly #cdp: CdpRunner;
|
||||||
|
readonly #tabsApi: ChromeTabsApi;
|
||||||
|
readonly #registry = new RecordingStateRegistry();
|
||||||
|
readonly #annotations: StepAnnotation[] = [];
|
||||||
|
readonly #contexts = new Map<number, TabRecordingContext>();
|
||||||
|
readonly #maxTokens?: number;
|
||||||
|
readonly #redactValues: boolean;
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
maxTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
}) {
|
||||||
|
this.#cdp = input.cdp;
|
||||||
|
this.#tabsApi = input.tabsApi;
|
||||||
|
this.#maxTokens = input.maxTokens;
|
||||||
|
this.#redactValues = input.redactValues ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
#context(tabId: number): TabRecordingContext {
|
||||||
|
const existing = this.#contexts.get(tabId);
|
||||||
|
if (existing) return existing;
|
||||||
|
const session = new RecordingObservationSession({
|
||||||
|
registry: this.#registry,
|
||||||
|
annotations: this.#annotations,
|
||||||
|
maxTokens: this.#maxTokens,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
});
|
||||||
|
const context: TabRecordingContext = {
|
||||||
|
session,
|
||||||
|
settle: new SettleController({
|
||||||
|
session,
|
||||||
|
cdp: this.#cdp,
|
||||||
|
tabsApi: this.#tabsApi,
|
||||||
|
tabId,
|
||||||
|
}),
|
||||||
|
pendingCapture: null,
|
||||||
|
frameRefresh: Promise.resolve(),
|
||||||
|
frameRefreshAbort: null,
|
||||||
|
};
|
||||||
|
this.#contexts.set(tabId, context);
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #capture(tabId: number): Promise<RegisteredObservation> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (context.pendingCapture) return context.pendingCapture.promise;
|
||||||
|
|
||||||
|
const abort = new AbortController();
|
||||||
|
let pending!: PendingCapture;
|
||||||
|
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
|
||||||
|
pending = {
|
||||||
|
abort,
|
||||||
|
promise: promise.finally(() => {
|
||||||
|
if (context.pendingCapture === pending) context.pendingCapture = null;
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
context.pendingCapture = pending;
|
||||||
|
return pending.promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async captureInitial(tabId: number): Promise<void> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (context.session.cursor.lastSettled) return;
|
||||||
|
await this.#capture(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Refresh the safe observation after a late child Document becomes usable. */
|
||||||
|
async refreshDocument(tabId: number, producerId: string): Promise<void> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
context.frameRefreshAbort?.abort();
|
||||||
|
const abort = new AbortController();
|
||||||
|
context.frameRefreshAbort = abort;
|
||||||
|
const previous = context.frameRefresh;
|
||||||
|
const refresh = (async () => {
|
||||||
|
await previous;
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
|
||||||
|
// The first capture discovers the safe producer -> CDP Document scope.
|
||||||
|
await this.#capture(tabId);
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
|
||||||
|
if (scope) {
|
||||||
|
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
|
||||||
|
}
|
||||||
|
if (abort.signal.aborted) return;
|
||||||
|
|
||||||
|
// Capture again after the child SPA is quiet so the next action can be
|
||||||
|
// matched against refs that actually exist in its pre-action state.
|
||||||
|
await this.#capture(tabId);
|
||||||
|
})();
|
||||||
|
const tracked = refresh.catch(() => {});
|
||||||
|
context.frameRefresh = tracked;
|
||||||
|
try {
|
||||||
|
await refresh;
|
||||||
|
} finally {
|
||||||
|
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async captureTabTransition(
|
||||||
|
fromTabId: number,
|
||||||
|
toTabId: number,
|
||||||
|
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
|
||||||
|
const from = this.#context(fromTabId);
|
||||||
|
await this.#flushContext(from);
|
||||||
|
if (!from.session.cursor.lastSettled) {
|
||||||
|
try {
|
||||||
|
await this.captureInitial(fromTabId);
|
||||||
|
} catch {
|
||||||
|
// A target-tab observation can still preserve the forward transition.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const to = this.#context(toTabId);
|
||||||
|
let target = to.session.cursor.lastSettled;
|
||||||
|
try {
|
||||||
|
target = await this.#capture(toTabId);
|
||||||
|
} catch {
|
||||||
|
// Reuse the last settled target state when a refresh races navigation.
|
||||||
|
}
|
||||||
|
|
||||||
|
const source = from.session.cursor.lastSettled;
|
||||||
|
return {
|
||||||
|
...(source ? { preStateId: source.stateId } : {}),
|
||||||
|
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
bindTabTransition(
|
||||||
|
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
|
||||||
|
draftId: number,
|
||||||
|
): void {
|
||||||
|
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async processDraft(
|
||||||
|
tabId: number,
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
producerId?: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const draft = drafts[draftIndex];
|
||||||
|
if (!draft) return;
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (!context.session.cursor.lastSettled) {
|
||||||
|
try {
|
||||||
|
await this.captureInitial(tabId);
|
||||||
|
} catch {
|
||||||
|
// Post-action settle can still provide a usable state.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const scope = producerId
|
||||||
|
? context.session.cursor.lastSettled?.index.documentScope(producerId)
|
||||||
|
: undefined;
|
||||||
|
if (
|
||||||
|
producerId &&
|
||||||
|
(draft.op === "click" ||
|
||||||
|
draft.op === "hover" ||
|
||||||
|
draft.op === "fill" ||
|
||||||
|
draft.op === "press" ||
|
||||||
|
draft.op === "select")
|
||||||
|
) {
|
||||||
|
draft.targetHint = {
|
||||||
|
...(draft.targetHint ?? {}),
|
||||||
|
frameId: scope?.frameId ?? null,
|
||||||
|
geometrySpace: "local",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
|
||||||
|
context.settle.schedule(drafts, draftIndex, scope);
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleSettle(
|
||||||
|
tabId: number,
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
scope?: DocumentSettleScope,
|
||||||
|
): void {
|
||||||
|
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
|
||||||
|
}
|
||||||
|
|
||||||
|
clearRedirect(tabId: number): void {
|
||||||
|
this.#contexts.get(tabId)?.settle.clearRedirect();
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
|
||||||
|
this.#context(tabId).settle.scheduleRedirect(drafts, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
async #flushContext(context: TabRecordingContext): Promise<void> {
|
||||||
|
await context.frameRefresh;
|
||||||
|
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||||
|
await context.settle.flushRedirects();
|
||||||
|
await context.settle.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flushRedirects(tabId: number): Promise<void> {
|
||||||
|
await this.#contexts.get(tabId)?.settle.flushRedirects();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
for (const context of this.#contexts.values()) await this.#flushContext(context);
|
||||||
|
}
|
||||||
|
|
||||||
|
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
|
||||||
|
const context = this.#context(tabId);
|
||||||
|
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||||
|
if (!context.session.cursor.lastSettled) {
|
||||||
|
try {
|
||||||
|
await this.captureInitial(tabId);
|
||||||
|
} catch {
|
||||||
|
// The trace may remain observation-free when CDP is unavailable.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await context.settle.settleTrailing(drafts);
|
||||||
|
inferMissingPostStates(drafts);
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
for (const context of this.#contexts.values()) {
|
||||||
|
context.frameRefreshAbort?.abort();
|
||||||
|
context.pendingCapture?.abort.abort();
|
||||||
|
context.settle.cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
buildTrace(input: {
|
||||||
|
drafts: RecordingDraftStep[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
includeTabSwitches?: boolean;
|
||||||
|
}): TraceV3 {
|
||||||
|
return buildTraceV3({
|
||||||
|
registry: this.#registry,
|
||||||
|
drafts: input.drafts,
|
||||||
|
annotations: this.#annotations,
|
||||||
|
startedAt: input.startedAt,
|
||||||
|
purpose: input.purpose,
|
||||||
|
startUrl: input.startUrl,
|
||||||
|
stoppedBy: input.stoppedBy,
|
||||||
|
bskVersion: input.bskVersion,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
includeTabSwitches: input.includeTabSwitches,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||||
|
import { RecordingObservationSession } from "./observation-session";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface PendingSettle {
|
||||||
|
abort: AbortController;
|
||||||
|
scope: DocumentSettleScope;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingRedirect {
|
||||||
|
url: string;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
(error as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
||||||
|
for (let index = 0; index < drafts.length - 1; index += 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
const next = drafts[index + 1];
|
||||||
|
if (draft && next && !draft.postStateId && next.preStateId) {
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SettleController {
|
||||||
|
readonly #session: RecordingObservationSession;
|
||||||
|
readonly #cdp: CdpRunner;
|
||||||
|
readonly #tabsApi: ChromeTabsApi;
|
||||||
|
readonly #tabId: number;
|
||||||
|
readonly #rootScope: DocumentSettleScope;
|
||||||
|
readonly #pending = new Map<number, PendingSettle>();
|
||||||
|
#queue = Promise.resolve();
|
||||||
|
#pendingRedirect: PendingRedirect | null = null;
|
||||||
|
#redirectQueue = Promise.resolve();
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
session: RecordingObservationSession;
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
}) {
|
||||||
|
this.#session = input.session;
|
||||||
|
this.#cdp = input.cdp;
|
||||||
|
this.#tabsApi = input.tabsApi;
|
||||||
|
this.#tabId = input.tabId;
|
||||||
|
this.#rootScope = { target: { tabId: input.tabId } };
|
||||||
|
}
|
||||||
|
|
||||||
|
get hasPending(): boolean {
|
||||||
|
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #captureWithRetry(signal?: AbortSignal) {
|
||||||
|
try {
|
||||||
|
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
||||||
|
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
schedule(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
scope: DocumentSettleScope = this.#rootScope,
|
||||||
|
): void {
|
||||||
|
if (scope.target.tabId !== this.#tabId) {
|
||||||
|
throw new Error(
|
||||||
|
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const landing = drafts[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of this.#pending) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.abort.abort();
|
||||||
|
this.#pending.delete(index);
|
||||||
|
if (landing && drafts[index] && !drafts[index].postStateId) {
|
||||||
|
drafts[index].postStateId = landing;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.#pending.get(draftIndex)?.abort.abort();
|
||||||
|
const pending = { abort: new AbortController(), scope };
|
||||||
|
this.#pending.set(draftIndex, pending);
|
||||||
|
const task = async () => {
|
||||||
|
if (pending.abort.signal.aborted) return;
|
||||||
|
try {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
const observation = await this.#captureWithRetry(pending.abort.signal);
|
||||||
|
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
||||||
|
drafts[draftIndex].postStateId = observation.stateId;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (!isAbortError(error)) {
|
||||||
|
console.warn(
|
||||||
|
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.#queue = this.#queue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
for (const pending of this.#pending.values()) pending.abort.abort();
|
||||||
|
this.#pending.clear();
|
||||||
|
this.clearRedirect();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#queue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#queue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
||||||
|
}
|
||||||
|
|
||||||
|
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
||||||
|
const trailing: RecordingDraftStep[] = [];
|
||||||
|
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
try {
|
||||||
|
const observation = await this.#captureWithRetry();
|
||||||
|
for (const draft of trailing) draft.postStateId = observation.stateId;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
clearRedirect(): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
const pending: PendingRedirect = { url, abort: new AbortController() };
|
||||||
|
this.#pendingRedirect = pending;
|
||||||
|
const task = () => this.#settleRedirect(drafts, pending);
|
||||||
|
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
||||||
|
|
||||||
|
let finalUrl = pending.url;
|
||||||
|
try {
|
||||||
|
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
||||||
|
} catch {
|
||||||
|
// The navigation event URL remains the best available destination.
|
||||||
|
}
|
||||||
|
if (this.#pendingRedirect !== pending) return;
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
if (
|
||||||
|
!finalUrl ||
|
||||||
|
finalUrl === "about:blank" ||
|
||||||
|
this.#session.cursor.lastSettled?.url === finalUrl
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const last = drafts[drafts.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
||||||
|
await this.flush();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
pageUrl: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
transitionQualifiers: ["server_redirect"],
|
||||||
|
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
drafts.push(draft);
|
||||||
|
const draftIndex = drafts.length - 1;
|
||||||
|
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
||||||
|
this.schedule(drafts, draftIndex);
|
||||||
|
await this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flushRedirects(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#redirectQueue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#redirectQueue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
export interface RecordedStateEntry {
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
|
||||||
|
vomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
stepsHere: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function stateIdentity(url: string, body: string): string {
|
||||||
|
return `${url}\0${body}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingStateRegistry {
|
||||||
|
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
||||||
|
readonly #idByIdentity = new Map<string, string>();
|
||||||
|
#nextId = 1;
|
||||||
|
|
||||||
|
register(input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}): RecordedStateEntry {
|
||||||
|
const identity = stateIdentity(input.url, input.vomText);
|
||||||
|
const existingId = this.#idByIdentity.get(identity);
|
||||||
|
if (existingId) {
|
||||||
|
const existing = this.#entriesById.get(existingId)!;
|
||||||
|
if (!existing.title && input.title) existing.title = input.title;
|
||||||
|
if (input.truncated) existing.truncated = true;
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry: RecordedStateEntry = {
|
||||||
|
id: `s${this.#nextId}`,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
vomText: input.vomText,
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
stepsHere: [],
|
||||||
|
};
|
||||||
|
this.#nextId += 1;
|
||||||
|
this.#entriesById.set(entry.id, entry);
|
||||||
|
this.#idByIdentity.set(identity, entry.id);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
values(): RecordedStateEntry[] {
|
||||||
|
return [...this.#entriesById.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
markStep(stateId: string, draftId: number): void {
|
||||||
|
const entry = this.#entriesById.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
import type { RecordStepPayload } from "../record-bridge";
|
||||||
|
import { hasRedirectQualifier } from "./navigation-policy";
|
||||||
|
import type { RecordingDraftStep, TargetMatchHint } from "./types";
|
||||||
|
|
||||||
|
export interface RecordingStepBuffer {
|
||||||
|
steps: RecordingDraftStep[];
|
||||||
|
navigation: RecordingNavigationCursor;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingNavigationCursor {
|
||||||
|
currentUrl?: string;
|
||||||
|
pendingNavigation: boolean;
|
||||||
|
pendingNavigationDeadline?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
|
function toDraftStep(
|
||||||
|
payload: RecordStepPayload,
|
||||||
|
targetHint?: TargetMatchHint,
|
||||||
|
): RecordingDraftStep | null {
|
||||||
|
const pageUrl = payload.page_url;
|
||||||
|
const common = {
|
||||||
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
|
...(targetHint ? { targetHint } : {}),
|
||||||
|
};
|
||||||
|
switch (payload.op) {
|
||||||
|
case "click":
|
||||||
|
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
|
||||||
|
case "hover":
|
||||||
|
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
|
||||||
|
case "fill":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: payload.target,
|
||||||
|
value: payload.value ?? "",
|
||||||
|
...(payload.commit ? { commit: payload.commit } : {}),
|
||||||
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
|
...common,
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "press":
|
||||||
|
return payload.key
|
||||||
|
? {
|
||||||
|
op: "press",
|
||||||
|
key: payload.key,
|
||||||
|
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||||
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
|
...common,
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "select":
|
||||||
|
return payload.target && payload.values
|
||||||
|
? {
|
||||||
|
op: "select",
|
||||||
|
captureTarget: payload.target,
|
||||||
|
values: payload.values,
|
||||||
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
|
...common,
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "navigate":
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||||
|
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
|
||||||
|
const step = buffer.steps[index];
|
||||||
|
if (!step) continue;
|
||||||
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
|
step.navigatedTo = url;
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NavigationObserveResult =
|
||||||
|
| { kind: "noop" }
|
||||||
|
| { kind: "annotated"; index: number }
|
||||||
|
| { kind: "appended"; index: number }
|
||||||
|
| { kind: "coalesce_redirect"; url: string };
|
||||||
|
|
||||||
|
export function observeRecordedNavigation(
|
||||||
|
buffer: RecordingStepBuffer,
|
||||||
|
url: string,
|
||||||
|
causedByAction?: boolean,
|
||||||
|
transitionType?: string,
|
||||||
|
transitionQualifiers?: string[],
|
||||||
|
): NavigationObserveResult {
|
||||||
|
const navigation = buffer.navigation;
|
||||||
|
if (!url || url === navigation.currentUrl) return { kind: "noop" };
|
||||||
|
navigation.currentUrl = url;
|
||||||
|
|
||||||
|
const pendingIsCurrent =
|
||||||
|
navigation.pendingNavigation &&
|
||||||
|
(navigation.pendingNavigationDeadline === undefined ||
|
||||||
|
navigation.pendingNavigationDeadline >= Date.now());
|
||||||
|
|
||||||
|
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||||
|
navigation.pendingNavigation = false;
|
||||||
|
navigation.pendingNavigationDeadline = undefined;
|
||||||
|
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||||
|
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
|
||||||
|
} else {
|
||||||
|
navigation.pendingNavigation = false;
|
||||||
|
navigation.pendingNavigationDeadline = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasRedirectQualifier(transitionQualifiers)) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
|
buffer.steps.push({
|
||||||
|
op: "navigate",
|
||||||
|
url,
|
||||||
|
pageUrl: url,
|
||||||
|
transitionType,
|
||||||
|
transitionQualifiers,
|
||||||
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function appendRecordedPayload(
|
||||||
|
buffer: RecordingStepBuffer,
|
||||||
|
payload: RecordStepPayload,
|
||||||
|
targetHint?: TargetMatchHint,
|
||||||
|
): number | null {
|
||||||
|
if (payload.op === "navigate") {
|
||||||
|
if (!payload.url) return null;
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
payload.url,
|
||||||
|
payload.navigation_caused_by_action,
|
||||||
|
payload.transitionType,
|
||||||
|
payload.transitionQualifiers,
|
||||||
|
);
|
||||||
|
return result.kind === "appended" ? result.index : null;
|
||||||
|
}
|
||||||
|
const step = toDraftStep(
|
||||||
|
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
|
||||||
|
targetHint,
|
||||||
|
);
|
||||||
|
if (!step) return null;
|
||||||
|
buffer.steps.push(step);
|
||||||
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
|
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
|
||||||
|
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
|
||||||
|
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||||
|
: undefined;
|
||||||
|
}
|
||||||
|
return buffer.steps.length - 1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import type { RecordingNavigationCursor } from "./step-buffer";
|
||||||
|
|
||||||
|
export interface TabActivation {
|
||||||
|
tabId: number;
|
||||||
|
revision: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingTabCoordinator {
|
||||||
|
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
|
||||||
|
#activeTabId: number;
|
||||||
|
#currentTabId: number;
|
||||||
|
#activationRevision = 0;
|
||||||
|
|
||||||
|
constructor(initialTabId: number, initialUrl?: string) {
|
||||||
|
this.#activeTabId = initialTabId;
|
||||||
|
this.#currentTabId = initialTabId;
|
||||||
|
this.#navigationByTab.set(initialTabId, {
|
||||||
|
currentUrl: initialUrl,
|
||||||
|
pendingNavigation: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
get activeTabId(): number {
|
||||||
|
return this.#activeTabId;
|
||||||
|
}
|
||||||
|
|
||||||
|
get currentTabId(): number {
|
||||||
|
return this.#currentTabId;
|
||||||
|
}
|
||||||
|
|
||||||
|
noteActivation(tabId: number): TabActivation {
|
||||||
|
this.#activeTabId = tabId;
|
||||||
|
this.#activationRevision += 1;
|
||||||
|
return { tabId, revision: this.#activationRevision };
|
||||||
|
}
|
||||||
|
|
||||||
|
isLatest(activation: TabActivation): boolean {
|
||||||
|
return (
|
||||||
|
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
commit(tabId: number, currentUrl?: string): void {
|
||||||
|
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
|
||||||
|
this.#currentTabId = tabId;
|
||||||
|
}
|
||||||
|
|
||||||
|
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
|
||||||
|
const existing = this.#navigationByTab.get(tabId);
|
||||||
|
if (existing) {
|
||||||
|
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
const created: RecordingNavigationCursor = {
|
||||||
|
currentUrl: fallbackUrl,
|
||||||
|
pendingNavigation: false,
|
||||||
|
};
|
||||||
|
this.#navigationByTab.set(tabId, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { TargetDescriptorV3 } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "../describe-target";
|
||||||
|
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
||||||
|
import type { TargetGeometry, TargetMatchHint } from "./types";
|
||||||
|
|
||||||
|
const MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
function close(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
||||||
|
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidateMatches(
|
||||||
|
candidate: IndexedObservationNode,
|
||||||
|
geometry: TargetGeometry,
|
||||||
|
geometrySpace: "top" | "local",
|
||||||
|
): boolean {
|
||||||
|
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
|
||||||
|
return rect != null && rectMatches(geometry.rect, rect);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
...(fallback?.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback?.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalized(value: string | undefined): string {
|
||||||
|
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
||||||
|
const role = normalized(fallback?.role);
|
||||||
|
const name = normalized(fallback?.name);
|
||||||
|
if (!role && !name) return false;
|
||||||
|
if (role && normalized(ref.role) !== role) return false;
|
||||||
|
if (name && normalized(ref.name) !== name) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
ref: ref.ref,
|
||||||
|
...(ref.role ? { role: ref.role } : {}),
|
||||||
|
...(ref.name ? { name: ref.name } : {}),
|
||||||
|
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function matchObservationTarget(input: {
|
||||||
|
observation: RegisteredObservation;
|
||||||
|
hint?: TargetMatchHint;
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}): TargetDescriptorV3 {
|
||||||
|
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
|
||||||
|
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||||
|
const geometry = input.hint?.geometry;
|
||||||
|
if (geometry) {
|
||||||
|
const matches = input.observation.index
|
||||||
|
.candidates(frameId, geometry.tag)
|
||||||
|
.filter(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.ref &&
|
||||||
|
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
|
||||||
|
);
|
||||||
|
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||||
|
const semanticMatches = matches.filter(
|
||||||
|
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||||
|
);
|
||||||
|
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
||||||
|
return unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const semanticMatches = input.observation.index
|
||||||
|
.refs(frameId)
|
||||||
|
.filter((ref) => matchesSemantics(ref, input.fallback));
|
||||||
|
return semanticMatches.length === 1
|
||||||
|
? descriptor(semanticMatches[0]!)
|
||||||
|
: unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import {
|
||||||
|
type StepV3,
|
||||||
|
type StopReason,
|
||||||
|
TRACE_VERSION_V3,
|
||||||
|
type TraceStateV3,
|
||||||
|
type TraceV3,
|
||||||
|
VOM_FORMAT_VERSION,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl } from "./draft-policy";
|
||||||
|
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
||||||
|
import { formatTraceStateBody } from "./trace-state-body";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
||||||
|
const entries = registry.values();
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
return [
|
||||||
|
...new Set(
|
||||||
|
draftIds.flatMap((id) => {
|
||||||
|
const stepId = stepIdByDraftId.get(id);
|
||||||
|
return stepId === undefined ? [] : [stepId];
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
registry: RecordingStateRegistry;
|
||||||
|
drafts: RecordingDraftStep[];
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
redactValues?: boolean;
|
||||||
|
includeTabSwitches?: boolean;
|
||||||
|
}): TraceV3 {
|
||||||
|
const reduced = reduceTraceStepsV3(input.drafts, {
|
||||||
|
includeTabSwitches: input.includeTabSwitches,
|
||||||
|
redactValues: input.redactValues,
|
||||||
|
});
|
||||||
|
const entries = publishedEntries(input.registry, reduced.steps);
|
||||||
|
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||||
|
for (const annotation of input.annotations ?? []) {
|
||||||
|
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
||||||
|
bucket.push(annotation);
|
||||||
|
annotationsByState.set(annotation.stateId, bucket);
|
||||||
|
}
|
||||||
|
const steps = reduced.steps.map((step) => ({
|
||||||
|
...step,
|
||||||
|
state: publishedId.get(step.state) ?? step.state,
|
||||||
|
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
||||||
|
}));
|
||||||
|
const states: TraceStateV3[] = entries.map((entry) => {
|
||||||
|
const id = publishedId.get(entry.id) ?? entry.id;
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body: formatTraceStateBody({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||||
|
vomText: entry.vomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||||
|
}),
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
version: TRACE_VERSION_V3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
+62
-71
@@ -1,30 +1,10 @@
|
|||||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
import type { RecordingDraftStep } from "./types";
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
|
||||||
|
|
||||||
export function shouldRecordPress(
|
|
||||||
key: string,
|
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
|
||||||
): boolean {
|
|
||||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
|
||||||
const mods = modifiers ?? [];
|
|
||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
|
||||||
// Drop bare character typing — FillSession already records the value.
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
|
||||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
/** Collapse consecutive navigations to the last hop. */
|
||||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
||||||
const out: DraftTraceStep[] = [];
|
const out: RecordingDraftStep[] = [];
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
const prev = out[out.length - 1];
|
const prev = out[out.length - 1];
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||||
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
||||||
const urls: string[] = [];
|
const urls: string[] = [];
|
||||||
if (startUrl) urls.push(startUrl);
|
if (startUrl) urls.push(startUrl);
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
urls.push(step.url);
|
urls.push(step.url);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
if (step.pageUrl) urls.push(step.pageUrl);
|
||||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
||||||
}
|
}
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const unique: string[] = [];
|
const unique: string[] = [];
|
||||||
@@ -58,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildPageRegistry(
|
function buildPageRegistry(
|
||||||
steps: DraftTraceStep[],
|
steps: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
||||||
const urls = collectUrls(steps, startUrl);
|
const urls = collectUrls(steps, startUrl);
|
||||||
const urlToId = new Map<string, string>();
|
const urlToId = new Map<string, string>();
|
||||||
const pages = urls.map((url, index) => {
|
const pages = urls.map((url, index) => {
|
||||||
@@ -81,28 +61,28 @@ function pageIdFor(
|
|||||||
return urlToId.values().next().value ?? "p1";
|
return urlToId.values().next().value ?? "p1";
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
||||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
||||||
if ("page_url" in step && step.page_url) return step.page_url;
|
if (step.pageUrl) return step.pageUrl;
|
||||||
return fallbackUrl;
|
return fallbackUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
function effectForNavigation(
|
function effectForNavigation(
|
||||||
navigatedTo: string | undefined,
|
navigatedTo: string | undefined,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
): Step["effect"] {
|
): StepV2["effect"] {
|
||||||
if (!navigatedTo) return undefined;
|
if (!navigatedTo) return undefined;
|
||||||
const pageId = urlToId.get(navigatedTo);
|
const pageId = urlToId.get(navigatedTo);
|
||||||
if (!pageId) return undefined;
|
if (!pageId) return undefined;
|
||||||
return { navigated_to: pageId };
|
return { navigated_to: pageId };
|
||||||
}
|
}
|
||||||
|
|
||||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
|
||||||
if (!effect) return step;
|
if (!effect) return step;
|
||||||
return { ...step, effect };
|
return { ...step, effect };
|
||||||
}
|
}
|
||||||
|
|
||||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||||
return values.map((value, index) => ({
|
return values.map((value, index) => ({
|
||||||
value,
|
value,
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
@@ -110,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function toV2Step(
|
function toV2Step(
|
||||||
step: DraftTraceStep,
|
step: RecordingDraftStep,
|
||||||
id: number,
|
id: number,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
fallbackUrl?: string,
|
fallbackUrl?: string,
|
||||||
): Step | null {
|
): StepV2 | null {
|
||||||
if (!shouldIncludeDraft(step)) return null;
|
if (!shouldIncludeDraft(step)) return null;
|
||||||
|
|
||||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||||
@@ -129,28 +109,23 @@ function toV2Step(
|
|||||||
to: step.url,
|
to: step.url,
|
||||||
};
|
};
|
||||||
case "click":
|
case "click":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "hover":
|
|
||||||
return {
|
|
||||||
op: "hover",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.target,
|
|
||||||
};
|
|
||||||
case "fill":
|
case "fill":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
value: step.value,
|
value: step.value,
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
...(step.redacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
@@ -161,44 +136,51 @@ function toV2Step(
|
|||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
key: step.key,
|
key: step.key,
|
||||||
...(step.target ? { target: step.target } : {}),
|
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "select":
|
case "select":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
selection: toSelection(step.values, step.labels),
|
selection: toSelection(step.values, step.labels),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
|
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
|
||||||
|
// step variant and fail to decode the whole result if we emit it.
|
||||||
|
case "hover":
|
||||||
|
case "scroll":
|
||||||
|
case "switch_tab":
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReducedTrace {
|
interface ReducedTrace {
|
||||||
pages: PageRef[];
|
pages: PageRefV2[];
|
||||||
steps: Step[];
|
steps: StepV2[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compile capture drafts into record-only trace v2 steps.
|
* Compile capture drafts into record-only trace v2 steps.
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||||
*/
|
*/
|
||||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
||||||
const collapsed = collapseNavigations(steps);
|
const collapsed = collapseNavigations(steps);
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||||
const out: Step[] = [];
|
const out: StepV2[] = [];
|
||||||
let id = 1;
|
let id = 1;
|
||||||
let lastUrl = startUrl;
|
let lastUrl = startUrl;
|
||||||
for (const draft of collapsed) {
|
for (const draft of collapsed) {
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
if (draft.op === "navigate") lastUrl = draft.url;
|
||||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
||||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
out.push(step);
|
out.push(step);
|
||||||
@@ -207,18 +189,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
|||||||
return { pages, steps: out };
|
return { pages, steps: out };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTraceStartUrl(
|
function resolveTraceStartUrl(
|
||||||
drafts: DraftTraceStep[],
|
drafts: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
pages?: PageRef[],
|
pages?: PageRefV2[],
|
||||||
): string {
|
): string {
|
||||||
if (startUrl) return startUrl;
|
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
||||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
}
|
||||||
return step.op === "navigate";
|
|
||||||
});
|
export function buildTraceV2(input: {
|
||||||
if (navigate) return navigate.url;
|
steps: RecordingDraftStep[];
|
||||||
for (const draft of drafts) {
|
startedAt: string;
|
||||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
startUrl?: string;
|
||||||
}
|
purpose?: string;
|
||||||
return pages?.[0]?.url ?? "about:blank";
|
}): TraceV2 {
|
||||||
|
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
||||||
|
return {
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
||||||
|
pages,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||||
|
import { shouldIncludeDraft } from "./draft-policy";
|
||||||
|
import { hasRedirectQualifier } from "./navigation-policy";
|
||||||
|
import { unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface CollapsedDraft {
|
||||||
|
draft: RecordingDraftStep;
|
||||||
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||||
|
return hasRedirectQualifier(step.transitionQualifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||||
|
const output: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const previous = output[output.length - 1];
|
||||||
|
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
||||||
|
previous.draft = {
|
||||||
|
...previous.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? previous.draft.postStateId,
|
||||||
|
};
|
||||||
|
previous.draftIds.push(index + 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
||||||
|
});
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
||||||
|
if (step.cause) return step.cause;
|
||||||
|
const qualifiers = step.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
||||||
|
return values.map((value, index) => ({
|
||||||
|
value,
|
||||||
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ReduceDraftOptions {
|
||||||
|
includeTabSwitches: boolean;
|
||||||
|
redactValues: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function reduceDraft(
|
||||||
|
draft: RecordingDraftStep,
|
||||||
|
id: number,
|
||||||
|
options: ReduceDraftOptions,
|
||||||
|
): StepV3 | null {
|
||||||
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
if (draft.op === "switch_tab") {
|
||||||
|
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
|
||||||
|
return {
|
||||||
|
op: "switch_tab",
|
||||||
|
id,
|
||||||
|
state: draft.preStateId,
|
||||||
|
result: { state: draft.postStateId },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const state = draft.preStateId ?? draft.postStateId;
|
||||||
|
const resultState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!state || !resultState) return null;
|
||||||
|
const common = { id, state, result: { state: resultState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
|
case "navigate":
|
||||||
|
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
||||||
|
case "click":
|
||||||
|
return {
|
||||||
|
op: "click",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "hover":
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "fill":
|
||||||
|
const fillIsRedacted = options.redactValues || draft.redacted === true;
|
||||||
|
return {
|
||||||
|
op: "fill",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
value: fillIsRedacted ? "***" : draft.value,
|
||||||
|
commit: draft.commit ?? "blur",
|
||||||
|
...(fillIsRedacted ? { redacted: true } : {}),
|
||||||
|
};
|
||||||
|
case "press":
|
||||||
|
return {
|
||||||
|
op: "press",
|
||||||
|
...common,
|
||||||
|
key: draft.key,
|
||||||
|
...(draft.captureTarget || draft.matchedTarget
|
||||||
|
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
||||||
|
: {}),
|
||||||
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
|
};
|
||||||
|
case "select":
|
||||||
|
return {
|
||||||
|
op: "select",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
|
||||||
|
};
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...common };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReducedTraceV3 {
|
||||||
|
steps: StepV3[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reduceTraceStepsV3(
|
||||||
|
steps: RecordingDraftStep[],
|
||||||
|
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
|
||||||
|
): ReducedTraceV3 {
|
||||||
|
const output: StepV3[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
|
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||||
|
const step = reduceDraft(draft, output.length + 1, {
|
||||||
|
includeTabSwitches: options.includeTabSwitches === true,
|
||||||
|
redactValues: options.redactValues === true,
|
||||||
|
});
|
||||||
|
if (!step) continue;
|
||||||
|
output.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||||
|
}
|
||||||
|
return { steps: output, stepIdByDraftId };
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import type { StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
||||||
|
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatTraceStateBody(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepIds: number[];
|
||||||
|
vomText: string;
|
||||||
|
annotations: StepAnnotation[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}): string {
|
||||||
|
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
||||||
|
lines.push(`url: ${JSON.stringify(input.url)}`);
|
||||||
|
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
||||||
|
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
||||||
|
for (const annotation of input.annotations) {
|
||||||
|
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byLine.get(annotation.line) ?? [];
|
||||||
|
bucket.push({ annotation, stepId });
|
||||||
|
byLine.set(annotation.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
||||||
|
let line = bodyLine;
|
||||||
|
for (const item of byLine.get(lineIndex) ?? []) {
|
||||||
|
line += annotationText(item.annotation, item.stepId);
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
});
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
import type {
|
||||||
|
FillCommit,
|
||||||
|
KeyModifier,
|
||||||
|
NavigationCause,
|
||||||
|
StepV3,
|
||||||
|
TargetDescriptorV3,
|
||||||
|
} from "@/transport/types";
|
||||||
|
|
||||||
|
export interface TargetGeometry {
|
||||||
|
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
tag: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TargetMatchHint {
|
||||||
|
geometry?: TargetGeometry;
|
||||||
|
/** Missing means top frame; null means the source Document could not be resolved. */
|
||||||
|
frameId?: string | null;
|
||||||
|
geometrySpace?: "top" | "local";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepAnnotation {
|
||||||
|
draftId: number;
|
||||||
|
op: StepV3["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftStateLink {
|
||||||
|
pageUrl?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftTarget {
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
targetHint?: TargetMatchHint;
|
||||||
|
matchedTarget?: TargetDescriptorV3;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftNavigationEffect {
|
||||||
|
navigatedTo?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordingDraftStep =
|
||||||
|
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
||||||
|
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
||||||
|
| ({
|
||||||
|
op: "fill";
|
||||||
|
value: string;
|
||||||
|
commit?: FillCommit;
|
||||||
|
redacted?: boolean;
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "press";
|
||||||
|
key: string;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "select";
|
||||||
|
values: string[];
|
||||||
|
labels?: string[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({ op: "scroll" } & DraftStateLink)
|
||||||
|
| ({ op: "switch_tab" } & DraftStateLink)
|
||||||
|
| ({
|
||||||
|
op: "navigate";
|
||||||
|
url: string;
|
||||||
|
cause?: NavigationCause;
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
} & DraftStateLink);
|
||||||
|
|
||||||
|
export type TargetedRecordingDraft = Extract<
|
||||||
|
RecordingDraftStep,
|
||||||
|
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
||||||
|
>;
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||||
import type { SessionManager } from "./manager";
|
import type { SessionManager } from "./manager";
|
||||||
|
|
||||||
@@ -15,6 +16,7 @@ export interface DisconnectCleanupOptions {
|
|||||||
manager: SessionManager;
|
manager: SessionManager;
|
||||||
sessionStopDeps?: SessionStopDeps;
|
sessionStopDeps?: SessionStopDeps;
|
||||||
onSessionsChanged?: () => void;
|
onSessionsChanged?: () => void;
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -29,7 +31,15 @@ export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
|||||||
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||||
|
|
||||||
return (): Promise<DisconnectCleanupReport> => {
|
return (): Promise<DisconnectCleanupReport> => {
|
||||||
if (inFlight) return inFlight;
|
if (inFlight) {
|
||||||
|
options.diagnostics?.("disconnect_cleanup.coalesced", {
|
||||||
|
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
|
return inFlight;
|
||||||
|
}
|
||||||
|
options.diagnostics?.("disconnect_cleanup.scheduled", {
|
||||||
|
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
inFlight = cleanupSessions(options).finally(() => {
|
inFlight = cleanupSessions(options).finally(() => {
|
||||||
inFlight = null;
|
inFlight = null;
|
||||||
});
|
});
|
||||||
@@ -42,8 +52,17 @@ async function cleanupSessions(
|
|||||||
): Promise<DisconnectCleanupReport> {
|
): Promise<DisconnectCleanupReport> {
|
||||||
const stoppedSessionIds: string[] = [];
|
const stoppedSessionIds: string[] = [];
|
||||||
const failures: DisconnectCleanupFailure[] = [];
|
const failures: DisconnectCleanupFailure[] = [];
|
||||||
|
const capturedSessions = options.manager.list();
|
||||||
|
options.diagnostics?.("disconnect_cleanup.started", {
|
||||||
|
captured_session_ids: capturedSessions.map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
|
|
||||||
for (const ctx of options.manager.list()) {
|
for (const ctx of capturedSessions) {
|
||||||
|
options.diagnostics?.("disconnect_cleanup.session_stop.started", {
|
||||||
|
session_id: ctx.sessionId,
|
||||||
|
agent_window_id: ctx.agentWindowId,
|
||||||
|
still_registered: options.manager.has(ctx.sessionId),
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
const result = await handleSessionStop(
|
const result = await handleSessionStop(
|
||||||
options.manager,
|
options.manager,
|
||||||
@@ -62,14 +81,26 @@ async function cleanupSessions(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
stoppedSessionIds.push(ctx.sessionId);
|
stoppedSessionIds.push(ctx.sessionId);
|
||||||
|
options.diagnostics?.("disconnect_cleanup.session_stop.completed", {
|
||||||
|
session_id: ctx.sessionId,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
failures.push({
|
failures.push({
|
||||||
sessionId: ctx.sessionId,
|
sessionId: ctx.sessionId,
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
});
|
});
|
||||||
|
options.diagnostics?.("disconnect_cleanup.session_stop.failed", {
|
||||||
|
session_id: ctx.sessionId,
|
||||||
|
error: err,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
options.onSessionsChanged?.();
|
options.onSessionsChanged?.();
|
||||||
|
options.diagnostics?.("disconnect_cleanup.completed", {
|
||||||
|
stopped_session_ids: stoppedSessionIds,
|
||||||
|
failures,
|
||||||
|
remaining_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
return { stoppedSessionIds, failures };
|
return { stoppedSessionIds, failures };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
import type { EventFrame } from "@/transport/types";
|
import type { EventFrame } from "@/transport/types";
|
||||||
import type { SessionManager } from "./manager";
|
import type { SessionManager } from "./manager";
|
||||||
@@ -24,6 +25,7 @@ export interface SessionEventHandlerOptions {
|
|||||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||||
*/
|
*/
|
||||||
onSessionsChanged?: () => void;
|
onSessionsChanged?: () => void;
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
function chromeWindowEvents(): WindowRemovedListener {
|
function chromeWindowEvents(): WindowRemovedListener {
|
||||||
@@ -50,6 +52,11 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
|||||||
|
|
||||||
const onRemoved = (windowId: number): void => {
|
const onRemoved = (windowId: number): void => {
|
||||||
const ctx = manager.findByWindowId(windowId);
|
const ctx = manager.findByWindowId(windowId);
|
||||||
|
options.diagnostics?.("chrome.window.removed", {
|
||||||
|
window_id: windowId,
|
||||||
|
matched_session_id: ctx?.sessionId ?? null,
|
||||||
|
live_session_ids: manager.list().map((entry) => entry.sessionId),
|
||||||
|
});
|
||||||
if (!ctx) return;
|
if (!ctx) return;
|
||||||
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
|
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
|
||||||
tab_id: tabId,
|
tab_id: tabId,
|
||||||
@@ -81,7 +88,16 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
|||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
transport.send(event);
|
transport.send(event);
|
||||||
|
options.diagnostics?.("session.window_closed.sent", {
|
||||||
|
session_id: ctx.sessionId,
|
||||||
|
window_id: windowId,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
options.diagnostics?.("session.window_closed.send_failed", {
|
||||||
|
session_id: ctx.sessionId,
|
||||||
|
window_id: windowId,
|
||||||
|
error: err,
|
||||||
|
});
|
||||||
console.warn("[bh] could not push session.window_closed event", err);
|
console.warn("[bh] could not push session.window_closed event", err);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||||
import { RefStore } from "./ref-store";
|
import { RefStore } from "./ref-store";
|
||||||
|
|
||||||
@@ -23,6 +24,7 @@ export interface BorrowReservation {
|
|||||||
export interface SessionManagerOptions {
|
export interface SessionManagerOptions {
|
||||||
agentWindow?: AgentWindowApi;
|
agentWindow?: AgentWindowApi;
|
||||||
now?: () => number;
|
now?: () => number;
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Options for starting a session's Agent Window. */
|
/** Options for starting a session's Agent Window. */
|
||||||
@@ -83,10 +85,12 @@ export class SessionManager {
|
|||||||
private readonly borrowReservations = new Map<number, string>();
|
private readonly borrowReservations = new Map<number, string>();
|
||||||
private readonly agentWindow: AgentWindowApi;
|
private readonly agentWindow: AgentWindowApi;
|
||||||
private readonly now: () => number;
|
private readonly now: () => number;
|
||||||
|
private readonly diagnostics?: DiagnosticSink;
|
||||||
|
|
||||||
constructor(options: SessionManagerOptions = {}) {
|
constructor(options: SessionManagerOptions = {}) {
|
||||||
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
|
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
|
||||||
this.now = options.now ?? Date.now;
|
this.now = options.now ?? Date.now;
|
||||||
|
this.diagnostics = options.diagnostics;
|
||||||
}
|
}
|
||||||
|
|
||||||
has(sessionId: string): boolean {
|
has(sessionId: string): boolean {
|
||||||
@@ -169,6 +173,13 @@ export class SessionManager {
|
|||||||
* daemon in the `tool.session_start` reply.
|
* daemon in the `tool.session_start` reply.
|
||||||
*/
|
*/
|
||||||
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||||
|
this.diagnostics?.("session_manager.start.requested", {
|
||||||
|
session_id: sessionId,
|
||||||
|
existing_session_ids: this.list().map((ctx) => ctx.sessionId),
|
||||||
|
focused: opts.focused ?? null,
|
||||||
|
size: opts.size ?? null,
|
||||||
|
signal_aborted: opts.signal?.aborted ?? false,
|
||||||
|
});
|
||||||
if (this.sessions.has(sessionId)) {
|
if (this.sessions.has(sessionId)) {
|
||||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||||
}
|
}
|
||||||
@@ -178,6 +189,10 @@ export class SessionManager {
|
|||||||
try {
|
try {
|
||||||
const { signal: _signal, ...createOptions } = opts;
|
const { signal: _signal, ...createOptions } = opts;
|
||||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||||
|
this.diagnostics?.("session_manager.window.created", {
|
||||||
|
session_id: sessionId,
|
||||||
|
agent_window_id: windowId,
|
||||||
|
});
|
||||||
throwIfSessionStartAborted(opts.signal);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||||
throwIfSessionStartAborted(opts.signal);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
@@ -191,8 +206,18 @@ export class SessionManager {
|
|||||||
};
|
};
|
||||||
this.sessions.set(sessionId, ctx);
|
this.sessions.set(sessionId, ctx);
|
||||||
this.windowIndex.set(windowId, sessionId);
|
this.windowIndex.set(windowId, sessionId);
|
||||||
|
this.diagnostics?.("session_manager.start.committed", {
|
||||||
|
session_id: sessionId,
|
||||||
|
agent_window_id: windowId,
|
||||||
|
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||||
|
});
|
||||||
return ctx;
|
return ctx;
|
||||||
} catch (startupError) {
|
} catch (startupError) {
|
||||||
|
this.diagnostics?.("session_manager.start.failed", {
|
||||||
|
session_id: sessionId,
|
||||||
|
agent_window_id: windowId,
|
||||||
|
error: startupError,
|
||||||
|
});
|
||||||
if (windowId !== null) {
|
if (windowId !== null) {
|
||||||
try {
|
try {
|
||||||
await this.agentWindow.remove(windowId);
|
await this.agentWindow.remove(windowId);
|
||||||
@@ -216,12 +241,28 @@ export class SessionManager {
|
|||||||
options: { dropOnly?: boolean } = {},
|
options: { dropOnly?: boolean } = {},
|
||||||
): Promise<SessionContext | null> {
|
): Promise<SessionContext | null> {
|
||||||
const ctx = this.sessions.get(sessionId);
|
const ctx = this.sessions.get(sessionId);
|
||||||
|
this.diagnostics?.("session_manager.stop.requested", {
|
||||||
|
session_id: sessionId,
|
||||||
|
drop_only: options.dropOnly ?? false,
|
||||||
|
existed: ctx !== undefined,
|
||||||
|
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||||
|
});
|
||||||
if (!ctx) return null;
|
if (!ctx) return null;
|
||||||
if (!options.dropOnly) {
|
if (!options.dropOnly) {
|
||||||
await this.agentWindow.remove(ctx.agentWindowId);
|
await this.agentWindow.remove(ctx.agentWindowId);
|
||||||
|
this.diagnostics?.("session_manager.window.removed", {
|
||||||
|
session_id: sessionId,
|
||||||
|
agent_window_id: ctx.agentWindowId,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
this.sessions.delete(sessionId);
|
this.sessions.delete(sessionId);
|
||||||
this.windowIndex.delete(ctx.agentWindowId);
|
this.windowIndex.delete(ctx.agentWindowId);
|
||||||
|
this.diagnostics?.("session_manager.stop.committed", {
|
||||||
|
session_id: sessionId,
|
||||||
|
agent_window_id: ctx.agentWindowId,
|
||||||
|
drop_only: options.dropOnly ?? false,
|
||||||
|
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||||
|
});
|
||||||
return ctx;
|
return ctx;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
|
||||||
|
|
||||||
|
const RECORDING_DOCUMENT_ID_PATTERN =
|
||||||
|
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||||
|
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
|
||||||
|
|
||||||
|
export function recordingDocumentMarkerValue(identity: string): string {
|
||||||
|
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
|
||||||
|
throw new TypeError("recording Document identity must be a random UUID");
|
||||||
|
}
|
||||||
|
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
|
||||||
|
export function readRecordingDocumentIdentity(
|
||||||
|
attrs: Readonly<Record<string, string>>,
|
||||||
|
): string | undefined {
|
||||||
|
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
|
||||||
|
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
|
||||||
|
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
|
||||||
|
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import type {
|
|||||||
RequestFrame,
|
RequestFrame,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
|
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
|
||||||
|
|
||||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
@@ -44,9 +45,72 @@ function makeRequest(method: string, params: unknown): RequestFrame {
|
|||||||
|
|
||||||
describe("ToolDispatcher", () => {
|
describe("ToolDispatcher", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
resetBrowserObservationForTests();
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses the configured recording runtime for start and stop", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const tab = {
|
||||||
|
id: 7,
|
||||||
|
windowId: 4242,
|
||||||
|
active: true,
|
||||||
|
status: "complete",
|
||||||
|
url: "https://example.com/start",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const frameCoordinator = {
|
||||||
|
begin: vi.fn(),
|
||||||
|
armTab: vi.fn(async () => true),
|
||||||
|
sourceFor: vi.fn(() => null),
|
||||||
|
stop: vi.fn(async () => true),
|
||||||
|
cancel: vi.fn(),
|
||||||
|
};
|
||||||
|
const sendToTab = vi.fn(async () => ({ ok: true }));
|
||||||
|
setBrowserObservationAttachForTests(
|
||||||
|
() => () => {},
|
||||||
|
() => () => {},
|
||||||
|
);
|
||||||
|
const dispatcher = new ToolDispatcher({
|
||||||
|
transport,
|
||||||
|
sessions,
|
||||||
|
recording: {
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(async () => tab),
|
||||||
|
query: vi.fn(async () => [tab]),
|
||||||
|
},
|
||||||
|
frameCoordinator,
|
||||||
|
sendToTab,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(
|
||||||
|
makeRequest("tool.record_start", {
|
||||||
|
session_id: "aa11",
|
||||||
|
url: "https://example.com/start",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
|
||||||
|
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
|
||||||
|
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
|
||||||
|
|
||||||
|
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(2));
|
||||||
|
|
||||||
|
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
|
||||||
|
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
|
||||||
|
});
|
||||||
|
|
||||||
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -203,6 +267,55 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const remove = vi.fn(async () => {});
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove,
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
let finishDetach: () => void = () => {};
|
||||||
|
const detach = new Promise<void>((resolve) => {
|
||||||
|
finishDetach = resolve;
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(),
|
||||||
|
detachSession: vi.fn(() => detach),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
|
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||||
|
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||||
|
await flushMicrotasks();
|
||||||
|
finishDetach();
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||||
|
expect(sent).toContainEqual({
|
||||||
|
id: "r-1",
|
||||||
|
error: expect.objectContaining({ code: "cancelled" }),
|
||||||
|
});
|
||||||
|
expect(sessions.has("aa11")).toBe(true);
|
||||||
|
expect(remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("returns not_found when stopping an unknown session", async () => {
|
it("returns not_found when stopping an unknown session", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
|
|||||||
@@ -456,6 +456,30 @@ describe("handleClick", () => {
|
|||||||
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
|
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns invalid_selector when Chrome rejects non-CSS selector syntax", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.getDocument": () => ({ root: { nodeId: 1 } }),
|
||||||
|
"DOM.querySelector": () => {
|
||||||
|
throw new Error('{"code":-32000,"message":"DOM Error while querying"}');
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const res = await handleClick(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", selector: "text=下午 12~18点" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
expect(res).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
message: "selector is not valid CSS: text=下午 12~18点",
|
||||||
|
data: {
|
||||||
|
reason: "invalid_selector",
|
||||||
|
selector: "text=下午 12~18点",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("respects the AbortSignal", async () => {
|
it("respects the AbortSignal", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
buildFrameVomScene,
|
buildFrameVomScene,
|
||||||
buildVomScene,
|
buildVomScene,
|
||||||
type CdpAxNode,
|
type CdpAxNode,
|
||||||
|
captureVomObservation,
|
||||||
handleGetHtml,
|
handleGetHtml,
|
||||||
handleObserve,
|
handleObserve,
|
||||||
handleScreenshot,
|
handleScreenshot,
|
||||||
@@ -1329,7 +1330,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||||
const axNodes: CdpAxNode[] = [
|
const axNodes: CdpAxNode[] = [
|
||||||
{
|
{
|
||||||
nodeId: "1",
|
nodeId: "1",
|
||||||
@@ -1364,7 +1365,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
id: 10,
|
id: 10,
|
||||||
sensitive: false,
|
sensitive: true,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -1655,11 +1656,13 @@ describe("buildVomScene", () => {
|
|||||||
|
|
||||||
const scene = buildVomScene(axNodes, captured);
|
const scene = buildVomScene(axNodes, captured);
|
||||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||||
expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
|
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
|
||||||
);
|
);
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "close"');
|
expect(rendered.text).toContain('@e1 button "close"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||||
@@ -1729,7 +1732,7 @@ describe("buildVomScene", () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const scene = buildVomScene(axNodes, captured);
|
const scene = buildVomScene(axNodes, captured);
|
||||||
expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
|
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
|
||||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1798,12 +1801,14 @@ describe("buildVomScene", () => {
|
|||||||
|
|
||||||
const scene = buildVomScene(axNodes, captured);
|
const scene = buildVomScene(axNodes, captured);
|
||||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||||
expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
|
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
|
||||||
);
|
);
|
||||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||||
@@ -2126,7 +2131,7 @@ describe("buildVomScene", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(scene.surfaces).toEqual([
|
expect(scene.surfaces).toEqual([
|
||||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||||
]);
|
]);
|
||||||
expect(renderVom(scene).text).toContain(
|
expect(renderVom(scene).text).toContain(
|
||||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||||
@@ -2904,10 +2909,452 @@ describe("handleSnapshot", () => {
|
|||||||
],
|
],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function secretsSnapshotReply(secrets: {
|
||||||
|
email: string;
|
||||||
|
password: string;
|
||||||
|
otp: string;
|
||||||
|
card: string;
|
||||||
|
}) {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"input",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"type",
|
||||||
|
"text",
|
||||||
|
"password",
|
||||||
|
"autocomplete",
|
||||||
|
"one-time-code",
|
||||||
|
"cc-number",
|
||||||
|
"value",
|
||||||
|
secrets.email,
|
||||||
|
secrets.password,
|
||||||
|
secrets.otp,
|
||||||
|
secrets.card,
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
const style = [i("static"), i("auto"), i("auto")];
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||||
|
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||||
|
[
|
||||||
|
i("type"),
|
||||||
|
i("text"),
|
||||||
|
i("autocomplete"),
|
||||||
|
i("one-time-code"),
|
||||||
|
i("value"),
|
||||||
|
i(secrets.otp),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
i("type"),
|
||||||
|
i("text"),
|
||||||
|
i("autocomplete"),
|
||||||
|
i("cc-number"),
|
||||||
|
i("value"),
|
||||||
|
i(secrets.card),
|
||||||
|
],
|
||||||
|
],
|
||||||
|
inputValue: {
|
||||||
|
index: [2, 3, 4, 5],
|
||||||
|
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3, 4, 5],
|
||||||
|
styles: [style, style, style, style, style],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[10, 10, 200, 32],
|
||||||
|
[10, 50, 200, 32],
|
||||||
|
[10, 90, 200, 32],
|
||||||
|
[10, 130, 200, 32],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2, 3, 4],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const VP_METRICS = {
|
const VP_METRICS = {
|
||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function makeFrameAwareDeps() {
|
||||||
|
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
|
||||||
|
const i = (value: string) => strings.indexOf(value);
|
||||||
|
const styles = [i("static"), i("auto"), i("auto")];
|
||||||
|
const snapshot = {
|
||||||
|
strings,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
frameId: "main",
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("iframe")],
|
||||||
|
backendNodeId: [10, 11, 12],
|
||||||
|
attributes: [[], [], [i("title"), i("Remote")]],
|
||||||
|
contentDocumentIndex: { index: [2], value: [1] },
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1, 2],
|
||||||
|
styles: [styles, styles, styles],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[100, 100, 400, 300],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button")],
|
||||||
|
backendNodeId: [20, 21, 22],
|
||||||
|
attributes: [[], [], []],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1, 2],
|
||||||
|
styles: [styles, styles, styles],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 400, 300],
|
||||||
|
[0, 0, 400, 300],
|
||||||
|
[20, 30, 120, 40],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const mainAx: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "main-root",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
childIds: ["frame-owner"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "frame-owner",
|
||||||
|
parentId: "main-root",
|
||||||
|
role: { type: "role", value: "Iframe" },
|
||||||
|
name: { type: "computedString", value: "Remote" },
|
||||||
|
backendDOMNodeId: 12,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const childAx: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "child-root",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 21,
|
||||||
|
childIds: ["child-button"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "child-button",
|
||||||
|
parentId: "child-root",
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Frame action" },
|
||||||
|
backendDOMNodeId: 22,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") return VP_METRICS;
|
||||||
|
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||||
|
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||||
|
throw new Error(`unexpected root CDP method ${method}`);
|
||||||
|
});
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Accessibility.enable") return {};
|
||||||
|
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||||
|
throw new Error(`unexpected child CDP method ${method}`);
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
trackSessionTab: vi.fn(),
|
||||||
|
} satisfies CdpRunner;
|
||||||
|
return {
|
||||||
|
cdp,
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(
|
||||||
|
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it("exposes the frame-aware observation result for recording", async () => {
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
childIds: ["password"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "password",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Password" },
|
||||||
|
value: { value: "hunter2" },
|
||||||
|
backendDOMNodeId: 13,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||||
|
|
||||||
|
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||||
|
redactValues: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.text).not.toContain("hunter2");
|
||||||
|
expect(result.text).toContain("•••");
|
||||||
|
expect(result.refs[0]).toMatchObject({
|
||||||
|
backendNodeId: 13,
|
||||||
|
role: "textbox",
|
||||||
|
name: "Password",
|
||||||
|
line: expect.any(Number),
|
||||||
|
});
|
||||||
|
expect(result.rootFrameId).toBe("root");
|
||||||
|
expect(result).not.toHaveProperty("frameDocuments");
|
||||||
|
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||||
|
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 13,
|
||||||
|
tag: "input",
|
||||||
|
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||||
|
localRect: { x: 400, y: 300, w: 200, h: 40 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("decorates live observations with active controlled content", async () => {
|
||||||
|
const strings = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"button",
|
||||||
|
"div",
|
||||||
|
"#text",
|
||||||
|
"role",
|
||||||
|
"tab",
|
||||||
|
"aria-selected",
|
||||||
|
"true",
|
||||||
|
"aria-controls",
|
||||||
|
"reviews-panel",
|
||||||
|
"id",
|
||||||
|
"static",
|
||||||
|
"auto",
|
||||||
|
"visible",
|
||||||
|
"1",
|
||||||
|
"A detailed review",
|
||||||
|
];
|
||||||
|
const index = (value: string) => strings.indexOf(value);
|
||||||
|
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
|
||||||
|
const snapshot = {
|
||||||
|
strings,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1, 3],
|
||||||
|
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
index("role"),
|
||||||
|
index("tab"),
|
||||||
|
index("aria-selected"),
|
||||||
|
index("true"),
|
||||||
|
index("aria-controls"),
|
||||||
|
index("reviews-panel"),
|
||||||
|
],
|
||||||
|
[index("id"), index("reviews-panel")],
|
||||||
|
[],
|
||||||
|
],
|
||||||
|
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1, 2, 3, 4],
|
||||||
|
styles: [style, style, style, style, style],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[20, 20, 120, 40],
|
||||||
|
[20, 80, 500, 200],
|
||||||
|
[20, 80, 200, 20],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 0, 1, 1, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["reviews"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "reviews",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 12,
|
||||||
|
role: { type: "role", value: "tab" },
|
||||||
|
name: { type: "computedString", value: "Reviews" },
|
||||||
|
properties: [
|
||||||
|
{ name: "selected", value: { value: true } },
|
||||||
|
{ name: "controls", value: { value: "reviews-panel" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await captureVomObservation(
|
||||||
|
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
|
||||||
|
4,
|
||||||
|
"https://example.com",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.text).toContain('@e1 tab "Reviews"');
|
||||||
|
expect(result.text).toContain("[§ active: Reviews]");
|
||||||
|
expect(result.text).toContain("A detailed review");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||||
|
const secrets = {
|
||||||
|
email: "user@example.com",
|
||||||
|
password: "hunter2",
|
||||||
|
otp: "847291",
|
||||||
|
card: "4111111111111111",
|
||||||
|
};
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
childIds: ["email", "password", "otp", "card"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "email",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Email" },
|
||||||
|
value: { value: secrets.email },
|
||||||
|
backendDOMNodeId: 12,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "password",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Password" },
|
||||||
|
value: { value: secrets.password },
|
||||||
|
backendDOMNodeId: 13,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "otp",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Code" },
|
||||||
|
value: { value: secrets.otp },
|
||||||
|
backendDOMNodeId: 14,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "card",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Card" },
|
||||||
|
value: { value: secrets.card },
|
||||||
|
backendDOMNodeId: 15,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||||
|
|
||||||
|
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||||
|
redactValues: true,
|
||||||
|
});
|
||||||
|
const dumped = JSON.stringify(result);
|
||||||
|
|
||||||
|
expect(result).not.toHaveProperty("frameDocuments");
|
||||||
|
expect(dumped).not.toContain(secrets.email);
|
||||||
|
expect(dumped).not.toContain(secrets.password);
|
||||||
|
expect(dumped).not.toContain(secrets.otp);
|
||||||
|
expect(dumped).not.toContain(secrets.card);
|
||||||
|
expect(dumped).not.toContain("formValue");
|
||||||
|
expect(dumped).not.toContain("formDefaultValue");
|
||||||
|
expect(dumped).not.toContain("axNodes");
|
||||||
|
expect(dumped).not.toContain("domNodes");
|
||||||
|
expect(dumped).not.toContain("attrs");
|
||||||
|
expect(result.text).toContain("•••");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||||
|
const deps = makeFrameAwareDeps();
|
||||||
|
|
||||||
|
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||||
|
|
||||||
|
expect(result.rootFrameId).toBe("main");
|
||||||
|
expect(result.frames).toHaveLength(2);
|
||||||
|
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
});
|
||||||
|
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||||
|
frameId: "child",
|
||||||
|
tag: "button",
|
||||||
|
localRect: { x: 20, y: 30, w: 120, h: 40 },
|
||||||
|
});
|
||||||
|
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||||
|
frameId: "child",
|
||||||
|
name: "Frame action",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stores iframe refs with their owning CDP session", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
const deps = makeFrameAwareDeps();
|
||||||
|
|
||||||
|
const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||||
|
|
||||||
|
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||||
|
expect(result.text).toContain('@e1 button "Frame action"');
|
||||||
|
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
|
||||||
|
expect(frameRef?.[1]).toMatchObject({
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
await sm.start("aa11");
|
await sm.start("aa11");
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -66,6 +66,52 @@ function makeApis(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("handleSessionStop with auto-return", () => {
|
describe("handleSessionStop with auto-return", () => {
|
||||||
|
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort();
|
||||||
|
|
||||||
|
const res = await handleSessionStop(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
{ cdp, signal: controller.signal },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(sm.has("aa11")).toBe(true);
|
||||||
|
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||||
|
expect(aw.remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const controller = new AbortController();
|
||||||
|
let finishDetach: () => void = () => {};
|
||||||
|
const detach = new Promise<void>((resolve) => {
|
||||||
|
finishDetach = resolve;
|
||||||
|
});
|
||||||
|
const cdp = { detachSession: vi.fn(() => detach) };
|
||||||
|
|
||||||
|
const stopping = handleSessionStop(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
{ cdp, signal: controller.signal },
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||||
|
controller.abort();
|
||||||
|
finishDetach();
|
||||||
|
|
||||||
|
const res = await stopping;
|
||||||
|
expect(res).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(sm.has("aa11")).toBe(true);
|
||||||
|
expect(aw.remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||||
const aw = fakeAgentWindow([100]);
|
const aw = fakeAgentWindow([100]);
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
export {
|
||||||
|
type CaptureVomObservationOptions,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "./observation";
|
||||||
|
export {
|
||||||
|
type CaptureVomFrame,
|
||||||
|
type CaptureVomMatchNode,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
} from "./vom/record-safe-observation";
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
|
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
@@ -51,7 +52,12 @@ import {
|
|||||||
handleScreenshot,
|
handleScreenshot,
|
||||||
handleSnapshot,
|
handleSnapshot,
|
||||||
} from "./observation";
|
} from "./observation";
|
||||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
import {
|
||||||
|
handleRecordAwait,
|
||||||
|
handleRecordStart,
|
||||||
|
handleRecordStop,
|
||||||
|
type RecordRuntimeDeps,
|
||||||
|
} from "./record";
|
||||||
import {
|
import {
|
||||||
handleSessionStart,
|
handleSessionStart,
|
||||||
handleSessionStop,
|
handleSessionStop,
|
||||||
@@ -99,6 +105,7 @@ export interface DispatcherDeps {
|
|||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
cdp?: DispatcherCdpRunner;
|
cdp?: DispatcherCdpRunner;
|
||||||
|
recording?: RecordRuntimeDeps;
|
||||||
/**
|
/**
|
||||||
* Invoked whenever a dispatched RPC may have changed the live
|
* Invoked whenever a dispatched RPC may have changed the live
|
||||||
* session set (currently `tool.session_start` and
|
* session set (currently `tool.session_start` and
|
||||||
@@ -112,6 +119,7 @@ export interface DispatcherDeps {
|
|||||||
approveBorrow?: BorrowConfirmationApprover;
|
approveBorrow?: BorrowConfirmationApprover;
|
||||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||||
helpNotificationCopy?: () => { title: string; body: string };
|
helpNotificationCopy?: () => { title: string; body: string };
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -134,10 +142,12 @@ export class ToolDispatcher {
|
|||||||
private readonly transport: Transport;
|
private readonly transport: Transport;
|
||||||
private readonly sessions: SessionManager;
|
private readonly sessions: SessionManager;
|
||||||
private readonly cdp?: DispatcherCdpRunner;
|
private readonly cdp?: DispatcherCdpRunner;
|
||||||
|
private readonly recording?: RecordRuntimeDeps;
|
||||||
private readonly onSessionsChanged?: () => void;
|
private readonly onSessionsChanged?: () => void;
|
||||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||||
|
private readonly diagnostics?: DiagnosticSink;
|
||||||
private subscription: { dispose(): void } | null = null;
|
private subscription: { dispose(): void } | null = null;
|
||||||
private readonly hoverBypassTabs = new Map<number, string>();
|
private readonly hoverBypassTabs = new Map<number, string>();
|
||||||
private readonly hoverLatches = new Map<number, HoverLatch>();
|
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||||
@@ -153,10 +163,12 @@ export class ToolDispatcher {
|
|||||||
this.transport = deps.transport;
|
this.transport = deps.transport;
|
||||||
this.sessions = deps.sessions;
|
this.sessions = deps.sessions;
|
||||||
this.cdp = deps.cdp;
|
this.cdp = deps.cdp;
|
||||||
|
this.recording = deps.recording;
|
||||||
this.onSessionsChanged = deps.onSessionsChanged;
|
this.onSessionsChanged = deps.onSessionsChanged;
|
||||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||||
this.approveBorrow = deps.approveBorrow;
|
this.approveBorrow = deps.approveBorrow;
|
||||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||||
|
this.diagnostics = deps.diagnostics;
|
||||||
}
|
}
|
||||||
|
|
||||||
start(): void {
|
start(): void {
|
||||||
@@ -184,6 +196,16 @@ export class ToolDispatcher {
|
|||||||
private async dispatch(msg: ProtocolFrame): Promise<void> {
|
private async dispatch(msg: ProtocolFrame): Promise<void> {
|
||||||
if (!isRequestFrame(msg)) return;
|
if (!isRequestFrame(msg)) return;
|
||||||
const req = msg as RequestFrame;
|
const req = msg as RequestFrame;
|
||||||
|
const requestSessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||||
|
const startedAt = performance.now();
|
||||||
|
this.diagnostics?.("dispatcher.rpc.received", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
method: req.method,
|
||||||
|
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||||
|
transport_state: this.transport.state,
|
||||||
|
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
||||||
|
});
|
||||||
|
|
||||||
// Cancel frames take a fast path: trip the matching controller
|
// Cancel frames take a fast path: trip the matching controller
|
||||||
// (if any), reply with `{cancelled}` so the daemon can answer
|
// (if any), reply with `{cancelled}` so the daemon can answer
|
||||||
@@ -192,6 +214,12 @@ export class ToolDispatcher {
|
|||||||
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
|
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
|
||||||
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
|
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
|
||||||
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
|
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
|
||||||
|
this.diagnostics?.("dispatcher.cancel.received", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
target_rpc_id: target,
|
||||||
|
target_found: ac !== undefined,
|
||||||
|
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
||||||
|
});
|
||||||
if (ac) {
|
if (ac) {
|
||||||
try {
|
try {
|
||||||
ac.abort();
|
ac.abort();
|
||||||
@@ -205,7 +233,17 @@ export class ToolDispatcher {
|
|||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
this.transport.send(reply);
|
this.transport.send(reply);
|
||||||
|
this.diagnostics?.("dispatcher.cancel.ack_sent", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
target_rpc_id: target,
|
||||||
|
cancelled: ac !== undefined,
|
||||||
|
});
|
||||||
} catch (sendErr) {
|
} catch (sendErr) {
|
||||||
|
this.diagnostics?.("dispatcher.cancel.ack_failed", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
target_rpc_id: target,
|
||||||
|
error: sendErr,
|
||||||
|
});
|
||||||
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
|
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -230,6 +268,13 @@ export class ToolDispatcher {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
this.diagnostics?.("dispatcher.rpc.invoke_failed", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
method: req.method,
|
||||||
|
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||||
|
abort_signal: ac.signal.aborted,
|
||||||
|
error: err,
|
||||||
|
});
|
||||||
if (isAbortLikeError(err)) {
|
if (isAbortLikeError(err)) {
|
||||||
body = {
|
body = {
|
||||||
id: req.id,
|
id: req.id,
|
||||||
@@ -250,8 +295,27 @@ export class ToolDispatcher {
|
|||||||
let sent = true;
|
let sent = true;
|
||||||
try {
|
try {
|
||||||
this.transport.send(body);
|
this.transport.send(body);
|
||||||
|
this.diagnostics?.("dispatcher.rpc.response_sent", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
method: req.method,
|
||||||
|
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||||
|
response_kind: "error" in body ? "error" : "ok",
|
||||||
|
error_code: "error" in body ? body.error.code : null,
|
||||||
|
elapsed_ms: Math.round(performance.now() - startedAt),
|
||||||
|
transport_state: this.transport.state,
|
||||||
|
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
||||||
|
});
|
||||||
} catch (sendErr) {
|
} catch (sendErr) {
|
||||||
sent = false;
|
sent = false;
|
||||||
|
this.diagnostics?.("dispatcher.rpc.response_send_failed", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
method: req.method,
|
||||||
|
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||||
|
started_session_id: startedSession,
|
||||||
|
elapsed_ms: Math.round(performance.now() - startedAt),
|
||||||
|
transport_state: this.transport.state,
|
||||||
|
error: sendErr,
|
||||||
|
});
|
||||||
// Transport is dead by the time we want to reply. Drop the link
|
// Transport is dead by the time we want to reply. Drop the link
|
||||||
// proactively so the alarm-driven keepalive reconnects sooner
|
// proactively so the alarm-driven keepalive reconnects sooner
|
||||||
// and the daemon's pending RPC times out cleanly instead of
|
// and the daemon's pending RPC times out cleanly instead of
|
||||||
@@ -270,12 +334,23 @@ export class ToolDispatcher {
|
|||||||
try {
|
try {
|
||||||
const ctx = await this.sessions.stop(startedSession);
|
const ctx = await this.sessions.stop(startedSession);
|
||||||
if (ctx) {
|
if (ctx) {
|
||||||
|
this.diagnostics?.("dispatcher.session_start.rolled_back", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
session_id: startedSession,
|
||||||
|
agent_window_id: ctx.agentWindowId,
|
||||||
|
reason: "response_send_failed",
|
||||||
|
});
|
||||||
console.warn(
|
console.warn(
|
||||||
"[bsk dispatcher] rolled back orphan session after send failure",
|
"[bsk dispatcher] rolled back orphan session after send failure",
|
||||||
startedSession,
|
startedSession,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} catch (rollbackErr) {
|
} catch (rollbackErr) {
|
||||||
|
this.diagnostics?.("dispatcher.session_start.rollback_failed", {
|
||||||
|
rpc_id: req.id,
|
||||||
|
session_id: startedSession,
|
||||||
|
error: rollbackErr,
|
||||||
|
});
|
||||||
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
|
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -290,6 +365,7 @@ export class ToolDispatcher {
|
|||||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
|
signal,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
case "tool.tab_list":
|
case "tool.tab_list":
|
||||||
@@ -530,44 +606,26 @@ export class ToolDispatcher {
|
|||||||
signal,
|
signal,
|
||||||
});
|
});
|
||||||
case "tool.record_start":
|
case "tool.record_start":
|
||||||
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
return this.recording
|
||||||
tabsApi: chromeTabsApi,
|
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
...this.recording,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
signal,
|
||||||
try {
|
})
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
: recordingRuntimeUnavailable();
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
case "tool.record_stop":
|
case "tool.record_stop":
|
||||||
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
return this.recording
|
||||||
tabsApi: chromeTabsApi,
|
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
...this.recording,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
signal,
|
||||||
try {
|
})
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
: recordingRuntimeUnavailable();
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
case "tool.record_await":
|
case "tool.record_await":
|
||||||
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
return this.recording
|
||||||
tabsApi: chromeTabsApi,
|
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
...this.recording,
|
||||||
signal,
|
signal,
|
||||||
});
|
})
|
||||||
|
: recordingRuntimeUnavailable();
|
||||||
default:
|
default:
|
||||||
return {
|
return {
|
||||||
code: "unknown_method",
|
code: "unknown_method",
|
||||||
@@ -703,6 +761,13 @@ function isRpcError(v: unknown): v is RpcError {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordingRuntimeUnavailable(): RpcError {
|
||||||
|
return {
|
||||||
|
code: "protocol_error",
|
||||||
|
message: "recording runtime is unavailable",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||||
switch (req.method) {
|
switch (req.method) {
|
||||||
case "tool.tab_create":
|
case "tool.tab_create":
|
||||||
|
|||||||
@@ -106,6 +106,14 @@ function isAbortLikeError(err: unknown): boolean {
|
|||||||
return err instanceof DOMException && err.name === "AbortError";
|
return err instanceof DOMException && err.name === "AbortError";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isInvalidCssSelectorError(err: unknown): boolean {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
return (
|
||||||
|
message.includes("DOM Error while querying") ||
|
||||||
|
/invalid selector|not a valid selector|querySelector.*failed/i.test(message)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
@@ -208,6 +216,14 @@ async function resolveBackendNode(
|
|||||||
}
|
}
|
||||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (isInvalidCssSelectorError(err)) {
|
||||||
|
return rpcError(
|
||||||
|
"invalid_params",
|
||||||
|
"invalid_selector",
|
||||||
|
`selector is not valid CSS: ${params.selector}`,
|
||||||
|
{ selector: params.selector },
|
||||||
|
);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -15,22 +15,37 @@ import {
|
|||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
|
type RecordStepAck,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
import {
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
type RecordFrameCoordinator,
|
||||||
|
type RecordingCaptureScope,
|
||||||
|
recordFrameCoordinator,
|
||||||
|
} from "@/lib/recording/frame-coordinator";
|
||||||
|
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
|
||||||
|
import {
|
||||||
|
appendRecordedPayload,
|
||||||
|
observeRecordedNavigation,
|
||||||
|
type RecordingStepBuffer,
|
||||||
|
} from "@/lib/recording/step-buffer";
|
||||||
|
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
|
||||||
|
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
||||||
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import type {
|
import type {
|
||||||
DraftTraceStep,
|
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
RecordAwaitResult,
|
RecordAwaitResult,
|
||||||
|
RecordedTrace,
|
||||||
RecordStartParams,
|
RecordStartParams,
|
||||||
RecordStartResult,
|
RecordStartResult,
|
||||||
RecordStopParams,
|
RecordStopParams,
|
||||||
RecordStopResult,
|
RecordStopResult,
|
||||||
RpcError,
|
RpcError,
|
||||||
Trace,
|
StopReason,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import { TRACE_VERSION_V3 } from "@/transport/types";
|
||||||
import { handleNavigate } from "./navigation";
|
import { handleNavigate } from "./navigation";
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
@@ -43,21 +58,46 @@ import {
|
|||||||
|
|
||||||
interface ActiveRecording {
|
interface ActiveRecording {
|
||||||
requestId: string;
|
requestId: string;
|
||||||
tabId: number;
|
tabs: RecordingTabCoordinator;
|
||||||
agentWindowId: number;
|
agentWindowId: number;
|
||||||
startUrl?: string;
|
startUrl?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
steps: DraftTraceStep[];
|
steps: RecordingDraftStep[];
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
startedAtMs: number;
|
startedAtMs: number;
|
||||||
finishPromise: Promise<Trace>;
|
traceVersion: 2 | 3;
|
||||||
resolveFinish: (trace: Trace) => void;
|
supportsTabSwitchSteps: boolean;
|
||||||
|
finishPromise: Promise<RecordedTrace>;
|
||||||
|
resolveFinish: (trace: RecordedTrace) => void;
|
||||||
rejectFinish: (err: Error) => void;
|
rejectFinish: (err: Error) => void;
|
||||||
settled: boolean;
|
settled: boolean;
|
||||||
finishing: boolean;
|
finishAttempt: Promise<RecordedTrace | null> | null;
|
||||||
currentUrl?: string;
|
observation: RecordingObservationRuntime | null;
|
||||||
pendingNavigation: boolean;
|
stoppedBy: StopReason;
|
||||||
pendingNavigationDeadline?: number;
|
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||||
|
navigationCallbacks: Set<Promise<void>>;
|
||||||
|
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||||
|
acceptingNavigation: boolean;
|
||||||
|
/**
|
||||||
|
* Serializes step appends with navigation observation so a click is always
|
||||||
|
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||||
|
*/
|
||||||
|
actionQueue: Promise<void>;
|
||||||
|
/** Last accepted sequence for each content-script document producer. */
|
||||||
|
lastStepSequenceByProducer: Map<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueRecordingAction(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
task: () => Promise<void>,
|
||||||
|
): Promise<void> {
|
||||||
|
const queued = recording.actionQueue.then(task, task);
|
||||||
|
recording.actionQueue = queued.catch(() => {});
|
||||||
|
return queued;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecordingFinishing(recording: ActiveRecording): boolean {
|
||||||
|
return recording.settled || recording.finishAttempt !== null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const recordings = new Map<string, ActiveRecording>();
|
const recordings = new Map<string, ActiveRecording>();
|
||||||
@@ -78,6 +118,9 @@ function sleep(ms: number): Promise<void> {
|
|||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||||
|
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||||
|
|
||||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||||
|
|
||||||
@@ -137,9 +180,11 @@ async function sendRecordStartWithAck(
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
msg: RecordStartMessage,
|
msg: RecordStartMessage,
|
||||||
sendToTab: RecordDeps["sendToTab"],
|
sendToTab: RecordDeps["sendToTab"],
|
||||||
|
cancelled?: () => boolean,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
let lastError: unknown;
|
let lastError: unknown;
|
||||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||||
|
if (cancelled?.()) throw new Error("record start cancelled");
|
||||||
try {
|
try {
|
||||||
const response = await sendToTab(tabId, msg);
|
const response = await sendToTab(tabId, msg);
|
||||||
if (isRecordStartAck(response)) return;
|
if (isRecordStartAck(response)) return;
|
||||||
@@ -147,6 +192,7 @@ async function sendRecordStartWithAck(
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
lastError = err;
|
lastError = err;
|
||||||
}
|
}
|
||||||
|
if (cancelled?.()) throw new Error("record start cancelled");
|
||||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||||
}
|
}
|
||||||
@@ -154,19 +200,83 @@ async function sendRecordStartWithAck(
|
|||||||
throw lastError ?? new Error("failed to start recording in content script");
|
throw lastError ?? new Error("failed to start recording in content script");
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): Trace {
|
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
|
||||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
if (params.trace_version === undefined) return 2;
|
||||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
if (params.trace_version === TRACE_VERSION_V3) return 3;
|
||||||
return {
|
return {
|
||||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
code: "invalid_params",
|
||||||
recorded_at: new Date().toISOString(),
|
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`,
|
||||||
started_at: recording.startedAt,
|
|
||||||
entry: { start_url: startUrl },
|
|
||||||
pages,
|
|
||||||
steps,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildTrace(recording: ActiveRecording): RecordedTrace {
|
||||||
|
if (recording.traceVersion === 3 && recording.observation) {
|
||||||
|
return recording.observation.buildTrace({
|
||||||
|
drafts: recording.steps,
|
||||||
|
startedAt: recording.startedAt,
|
||||||
|
purpose: recording.purpose,
|
||||||
|
startUrl: recording.startUrl,
|
||||||
|
stoppedBy: recording.stoppedBy,
|
||||||
|
bskVersion: BSK_TRACE_VERSION,
|
||||||
|
includeTabSwitches: recording.supportsTabSwitchSteps,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (recording.traceVersion === 3) {
|
||||||
|
throw new Error("trace v3 observation runtime is unavailable");
|
||||||
|
}
|
||||||
|
return buildTraceV2({
|
||||||
|
steps: recording.steps,
|
||||||
|
startedAt: recording.startedAt,
|
||||||
|
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
|
||||||
|
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function stepBufferFor(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
tabId: number,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): RecordingStepBuffer {
|
||||||
|
return {
|
||||||
|
steps: recording.steps,
|
||||||
|
navigation: recording.tabs.navigation(tabId, fallbackUrl),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function activateRecordingTab(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
targetTabId: number,
|
||||||
|
): Promise<void> {
|
||||||
|
const previousTabId = recording.tabs.currentTabId;
|
||||||
|
if (targetTabId === previousTabId) return;
|
||||||
|
|
||||||
|
const transition = recording.observation
|
||||||
|
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
|
||||||
|
: {};
|
||||||
|
const { targetUrl, ...stateLinks } = transition;
|
||||||
|
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
|
||||||
|
op: "switch_tab",
|
||||||
|
...stateLinks,
|
||||||
|
};
|
||||||
|
recording.steps.push(draft);
|
||||||
|
recording.observation?.bindTabTransition(draft, recording.steps.length);
|
||||||
|
recording.tabs.commit(targetTabId, targetUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processRecordedStep(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
draftIndex: number,
|
||||||
|
tabId: number,
|
||||||
|
producerId?: string,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!recording.observation) return;
|
||||||
|
try {
|
||||||
|
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export interface RecordDeps {
|
export interface RecordDeps {
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
sendToTab(
|
sendToTab(
|
||||||
@@ -174,16 +284,25 @@ export interface RecordDeps {
|
|||||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
): Promise<unknown>;
|
): Promise<unknown>;
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
frameCoordinator?: Pick<
|
||||||
|
RecordFrameCoordinator,
|
||||||
|
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
|
||||||
|
>;
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
|
||||||
|
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
|
||||||
|
};
|
||||||
|
|
||||||
let defaultDeps: RecordDeps | null = null;
|
let defaultDeps: RecordDeps | null = null;
|
||||||
function getDefaultDeps(): RecordDeps {
|
function getDefaultDeps(): RecordDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
defaultDeps = {
|
defaultDeps = {
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
frameCoordinator: recordFrameCoordinator,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return defaultDeps;
|
return defaultDeps;
|
||||||
@@ -241,16 +360,58 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
|||||||
detachBrowserObservation = null;
|
detachBrowserObservation = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function attachRecordStepListener(): () => void {
|
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
_sender: chrome.runtime.MessageSender,
|
sender: chrome.runtime.MessageSender,
|
||||||
_sendResponse: () => void,
|
sendResponse: (response: RecordStepAck) => void,
|
||||||
) => {
|
) => {
|
||||||
if (!isRecordStepMessage(message)) return false;
|
if (!isRecordStepMessage(message)) return false;
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.requestId !== message.requestId) continue;
|
if (recording.requestId !== message.requestId) continue;
|
||||||
appendRecordedPayload(recording, message.step);
|
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
|
||||||
|
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
|
||||||
|
: undefined;
|
||||||
|
if (deps.frameCoordinator && !source) return false;
|
||||||
|
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
|
||||||
|
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
|
||||||
|
const producerKey = source
|
||||||
|
? `${source.tabId}:${source.documentId}:${source.producerId}`
|
||||||
|
: `${sourceTabId}:${message.producerId}`;
|
||||||
|
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
|
||||||
|
if (message.sequence < expectedSequence) {
|
||||||
|
sendResponse({ ok: true, sequence: message.sequence });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (message.sequence > expectedSequence) {
|
||||||
|
sendResponse({
|
||||||
|
ok: false,
|
||||||
|
expectedSequence,
|
||||||
|
error: `out-of-order recorded step ${message.sequence}`,
|
||||||
|
});
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
|
||||||
|
enqueueRecordingAction(recording, async () => {
|
||||||
|
if (sourceTabId !== recording.tabs.currentTabId) {
|
||||||
|
if (!sourceWasActive) return;
|
||||||
|
if (sourceTabId !== recording.tabs.activeTabId)
|
||||||
|
recording.tabs.noteActivation(sourceTabId);
|
||||||
|
await activateRecordingTab(recording, sourceTabId);
|
||||||
|
}
|
||||||
|
await recording.observation?.flushRedirects(sourceTabId);
|
||||||
|
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
|
||||||
|
const draftIndex = appendRecordedPayload(
|
||||||
|
stepBufferFor(recording, sourceTabId, message.step.page_url),
|
||||||
|
message.step,
|
||||||
|
targetHint,
|
||||||
|
);
|
||||||
|
if (draftIndex !== null) {
|
||||||
|
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
sendResponse({ ok: true, sequence: message.sequence });
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@@ -277,7 +438,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
|
|
||||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -314,7 +475,7 @@ async function clearRearmTimersForRecording(
|
|||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
clearRearmTimer(recording.tabId);
|
clearRearmTimer(recording.tabs.currentTabId);
|
||||||
try {
|
try {
|
||||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||||
for (const tab of tabs) {
|
for (const tab of tabs) {
|
||||||
@@ -329,13 +490,16 @@ async function stopRecordingOnAllAgentTabs(
|
|||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
|
||||||
|
throw new Error("failed to flush one or more recording documents");
|
||||||
|
}
|
||||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||||
let tabIds = [recording.tabId];
|
let tabIds = [recording.tabs.currentTabId];
|
||||||
try {
|
try {
|
||||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||||
tabIds = [
|
tabIds = [
|
||||||
...new Set([
|
...new Set([
|
||||||
recording.tabId,
|
recording.tabs.currentTabId,
|
||||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||||
]),
|
]),
|
||||||
];
|
];
|
||||||
@@ -346,11 +510,11 @@ async function stopRecordingOnAllAgentTabs(
|
|||||||
for (const tabId of tabIds) {
|
for (const tabId of tabIds) {
|
||||||
try {
|
try {
|
||||||
const response = await deps.sendToTab(tabId, stopMsg);
|
const response = await deps.sendToTab(tabId, stopMsg);
|
||||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
|
||||||
throw new Error("content script did not confirm recorded steps");
|
throw new Error("content script did not confirm recorded steps");
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
if (tabId === recording.tabId) {
|
if (tabId === recording.tabs.currentTabId) {
|
||||||
throw new Error("failed to flush recorded steps");
|
throw new Error("failed to flush recorded steps");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -368,22 +532,39 @@ async function rearmRecording(
|
|||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
targetTabId: number,
|
targetTabId: number,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
|
activation?: TabActivation,
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||||
// already hides the control chrome while activeRecord is set.
|
// already hides the control chrome while activeRecord is set.
|
||||||
|
const isFinishing = () => isRecordingFinishing(recording);
|
||||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
|
if (isFinishing()) return false;
|
||||||
const startMsg: RecordStartMessage = {
|
const startMsg: RecordStartMessage = {
|
||||||
type: RECORD_START,
|
type: RECORD_START,
|
||||||
requestId: recording.requestId,
|
requestId: recording.requestId,
|
||||||
startedAtMs: recording.startedAtMs,
|
startedAtMs: recording.startedAtMs,
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
if (deps.frameCoordinator) {
|
||||||
recording.tabId = targetTabId;
|
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
|
||||||
|
if (!frameStarted) throw new Error("recording document did not start");
|
||||||
|
}
|
||||||
|
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
|
||||||
|
if (isFinishing()) return false;
|
||||||
|
if (activation) {
|
||||||
|
if (!recording.tabs.isLatest(activation)) return true;
|
||||||
|
await enqueueRecordingAction(recording, async () => {
|
||||||
|
if (isFinishing() || !recording.tabs.isLatest(activation)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await activateRecordingTab(recording, targetTabId);
|
||||||
|
});
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
} catch {
|
} catch {
|
||||||
|
if (isFinishing()) return false;
|
||||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||||
}
|
}
|
||||||
@@ -392,7 +573,7 @@ async function rearmRecording(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
|
||||||
const existing = rearmTimers.get(tabId);
|
const existing = rearmTimers.get(tabId);
|
||||||
if (existing) clearTimeout(existing);
|
if (existing) clearTimeout(existing);
|
||||||
rearmTimers.set(
|
rearmTimers.set(
|
||||||
@@ -401,7 +582,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
|||||||
rearmTimers.delete(tabId);
|
rearmTimers.delete(tabId);
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const current = await findRecordingForTab(tabId, deps);
|
const current = await findRecordingForTab(tabId, deps);
|
||||||
if (current) await rearmRecording(current, tabId, deps);
|
if (current) await rearmRecording(current, tabId, deps, activation);
|
||||||
})();
|
})();
|
||||||
}, RECORD_REARM_DEBOUNCE_MS),
|
}, RECORD_REARM_DEBOUNCE_MS),
|
||||||
);
|
);
|
||||||
@@ -413,14 +594,20 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
|||||||
const windowId = tab.windowId;
|
const windowId = tab.windowId;
|
||||||
if (tabId === undefined || windowId === undefined) return;
|
if (tabId === undefined || windowId === undefined) return;
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.settled || recording.agentWindowId !== windowId) continue;
|
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
|
||||||
scheduleRearmForTab(tabId, deps);
|
scheduleRearmForTab(tabId, deps);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
for (const recording of recordings.values()) {
|
||||||
|
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
|
||||||
|
continue;
|
||||||
|
const activation = recording.tabs.noteActivation(activeInfo.tabId);
|
||||||
|
scheduleRearmForTab(activeInfo.tabId, deps, activation);
|
||||||
|
return;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
chrome.tabs.onCreated.addListener(onCreated);
|
chrome.tabs.onCreated.addListener(onCreated);
|
||||||
@@ -432,13 +619,47 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
const observeMainFrame = (
|
||||||
void (async () => {
|
tabId: number,
|
||||||
const recording = await findRecordingForTab(tabId, deps);
|
url?: string,
|
||||||
if (recording && url) {
|
causedByAction?: boolean,
|
||||||
observeRecordedNavigation(recording, url, causedByAction);
|
transitionType?: string,
|
||||||
}
|
transitionQualifiers?: string[],
|
||||||
})();
|
) => {
|
||||||
|
if (!url) return;
|
||||||
|
const candidates = [...recordings.values()].filter(
|
||||||
|
(recording) =>
|
||||||
|
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
|
||||||
|
);
|
||||||
|
for (const recording of candidates) {
|
||||||
|
const queued = enqueueRecordingAction(recording, async () => {
|
||||||
|
if (tabId !== recording.tabs.currentTabId) {
|
||||||
|
await activateRecordingTab(recording, tabId);
|
||||||
|
}
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
stepBufferFor(recording, tabId, url),
|
||||||
|
url,
|
||||||
|
causedByAction,
|
||||||
|
transitionType,
|
||||||
|
transitionQualifiers,
|
||||||
|
);
|
||||||
|
if (result.kind === "coalesce_redirect") {
|
||||||
|
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.kind === "noop") return;
|
||||||
|
recording.observation?.clearRedirect(tabId);
|
||||||
|
if (result.kind === "appended") {
|
||||||
|
await processRecordedStep(recording, result.index, tabId);
|
||||||
|
} else {
|
||||||
|
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const tracked = queued.catch(() => {});
|
||||||
|
recording.navigationCallbacks.add(tracked);
|
||||||
|
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
const onMainFrameComplete = (tabId: number, url?: string) => {
|
const onMainFrameComplete = (tabId: number, url?: string) => {
|
||||||
void (async () => {
|
void (async () => {
|
||||||
@@ -458,18 +679,13 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||||
) => {
|
) => {
|
||||||
if (details.frameId !== 0) return;
|
if (details.frameId !== 0) return;
|
||||||
const causedByAction =
|
observeMainFrame(
|
||||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
details.tabId,
|
||||||
? true
|
details.url,
|
||||||
: details.transitionType === "typed" ||
|
undefined,
|
||||||
details.transitionType === "auto_bookmark" ||
|
details.transitionType,
|
||||||
details.transitionType === "generated" ||
|
details.transitionQualifiers,
|
||||||
details.transitionType === "keyword" ||
|
);
|
||||||
details.transitionType === "keyword_generated" ||
|
|
||||||
details.transitionType === "reload"
|
|
||||||
? false
|
|
||||||
: undefined;
|
|
||||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
|
||||||
};
|
};
|
||||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||||
@@ -487,6 +703,26 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||||
|
|
||||||
|
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> {
|
||||||
|
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||||
|
await Promise.all([...recording.navigationCallbacks]);
|
||||||
|
const actionTail = recording.actionQueue;
|
||||||
|
await actionTail;
|
||||||
|
await recording.observation?.flush();
|
||||||
|
|
||||||
|
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) {
|
||||||
|
// No event or promise continuation can interleave with this synchronous
|
||||||
|
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||||
|
recording.acceptingNavigation = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
@@ -505,7 +741,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
sendResponse({ active: false });
|
sendResponse({ active: false });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await rearmRecording(recording, tabId, deps);
|
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
|
||||||
|
await rearmRecording(recording, tabId, deps, activation);
|
||||||
sendResponse({
|
sendResponse({
|
||||||
active: true,
|
active: true,
|
||||||
requestId: recording.requestId,
|
requestId: recording.requestId,
|
||||||
@@ -527,23 +764,50 @@ async function finishRecordingByRequest(
|
|||||||
if (recording.requestId !== requestId || recording.settled) continue;
|
if (recording.requestId !== requestId || recording.settled) continue;
|
||||||
const match = await findRecordingForTab(tabId, deps);
|
const match = await findRecordingForTab(tabId, deps);
|
||||||
if (match !== recording) continue;
|
if (match !== recording) continue;
|
||||||
await finishRecording(sessionId, deps);
|
await finishRecording(sessionId, deps, "user_finish");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
async function finishRecording(
|
||||||
|
sessionId: string,
|
||||||
|
deps: RecordDeps,
|
||||||
|
stoppedBy: StopReason,
|
||||||
|
): Promise<RecordedTrace | null> {
|
||||||
const recording = recordings.get(sessionId);
|
const recording = recordings.get(sessionId);
|
||||||
if (!recording || recording.settled || recording.finishing) return null;
|
if (!recording || recording.settled) return null;
|
||||||
recording.finishing = true;
|
if (recording.finishAttempt) return recording.finishAttempt;
|
||||||
|
recording.stoppedBy = stoppedBy;
|
||||||
|
|
||||||
|
const attempt = finishRecordingAttempt(sessionId, recording, deps);
|
||||||
|
recording.finishAttempt = attempt;
|
||||||
|
try {
|
||||||
|
return await attempt;
|
||||||
|
} finally {
|
||||||
|
if (!recording.settled) {
|
||||||
|
recording.finishAttempt = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finishRecordingAttempt(
|
||||||
|
sessionId: string,
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<RecordedTrace | null> {
|
||||||
await clearRearmTimersForRecording(recording, deps);
|
await clearRearmTimersForRecording(recording, deps);
|
||||||
try {
|
try {
|
||||||
|
// Disposing content capture may commit a final dirty fill. Flush content
|
||||||
|
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||||
|
// and the observation queues are drained.
|
||||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||||
} catch {
|
} catch {
|
||||||
recording.finishing = false;
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
if (!(await drainRecordingToStability(recording))) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
|
||||||
|
|
||||||
recording.settled = true;
|
recording.settled = true;
|
||||||
recordings.delete(sessionId);
|
recordings.delete(sessionId);
|
||||||
@@ -558,6 +822,15 @@ export async function handleRecordStart(
|
|||||||
params: RecordStartParams,
|
params: RecordStartParams,
|
||||||
deps: RecordDeps = getDefaultDeps(),
|
deps: RecordDeps = getDefaultDeps(),
|
||||||
): Promise<RecordStartResult | RpcError> {
|
): Promise<RecordStartResult | RpcError> {
|
||||||
|
const traceVersionOrErr = negotiatedTraceVersion(params);
|
||||||
|
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
|
||||||
|
if (traceVersionOrErr === 3 && !deps.cdp) {
|
||||||
|
return {
|
||||||
|
code: "protocol_error",
|
||||||
|
message: "trace v3 recording requires an active CDP connection",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const ctx = ctxOrErr;
|
const ctx = ctxOrErr;
|
||||||
@@ -574,37 +847,64 @@ export async function handleRecordStart(
|
|||||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||||
const requestId = makeRequestId(target.tabId);
|
const requestId = makeRequestId(target.tabId);
|
||||||
let resolveFinish!: (trace: Trace) => void;
|
let resolveFinish!: (trace: RecordedTrace) => void;
|
||||||
let rejectFinish!: (err: Error) => void;
|
let rejectFinish!: (err: Error) => void;
|
||||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
const finishPromise = new Promise<RecordedTrace>((resolve, reject) => {
|
||||||
resolveFinish = resolve;
|
resolveFinish = resolve;
|
||||||
rejectFinish = reject;
|
rejectFinish = reject;
|
||||||
});
|
});
|
||||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||||
const startedAtMs = Date.now();
|
const startedAtMs = Date.now();
|
||||||
|
const maxPageTokens = params.max_page_tokens;
|
||||||
|
const redactValues = params.redact_values ?? false;
|
||||||
recordings.set(params.session_id, {
|
recordings.set(params.session_id, {
|
||||||
requestId,
|
requestId,
|
||||||
tabId: target.tabId,
|
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
|
||||||
agentWindowId: ctx.agentWindowId,
|
agentWindowId: ctx.agentWindowId,
|
||||||
startUrl: navigateUrl,
|
startUrl: navigateUrl,
|
||||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||||
steps: [],
|
steps: [],
|
||||||
startedAt: new Date(startedAtMs).toISOString(),
|
startedAt: new Date(startedAtMs).toISOString(),
|
||||||
startedAtMs,
|
startedAtMs,
|
||||||
|
traceVersion: traceVersionOrErr,
|
||||||
|
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
|
||||||
finishPromise,
|
finishPromise,
|
||||||
resolveFinish,
|
resolveFinish,
|
||||||
rejectFinish,
|
rejectFinish,
|
||||||
settled: false,
|
settled: false,
|
||||||
finishing: false,
|
finishAttempt: null,
|
||||||
currentUrl: navigateUrl,
|
observation:
|
||||||
pendingNavigation: false,
|
traceVersionOrErr === 3 && deps.cdp
|
||||||
pendingNavigationDeadline: undefined,
|
? new RecordingObservationRuntime({
|
||||||
|
cdp: deps.cdp,
|
||||||
|
tabsApi: deps.tabsApi,
|
||||||
|
maxTokens: maxPageTokens,
|
||||||
|
redactValues,
|
||||||
|
})
|
||||||
|
: null,
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
navigationCallbacks: new Set(),
|
||||||
|
acceptingNavigation: true,
|
||||||
|
actionQueue: Promise.resolve(),
|
||||||
|
lastStepSequenceByProducer: new Map(),
|
||||||
|
});
|
||||||
|
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
|
||||||
|
const recording = recordings.get(params.session_id);
|
||||||
|
if (!recording || recording.requestId !== requestId || recording.settled) return;
|
||||||
|
try {
|
||||||
|
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
|
||||||
|
} catch {
|
||||||
|
// The normal action-time capture remains available if a child Document
|
||||||
|
// is replaced while its readiness refresh is in flight.
|
||||||
|
}
|
||||||
});
|
});
|
||||||
// Observe navigations for the whole recording lifetime; attach before
|
// Observe navigations for the whole recording lifetime; attach before
|
||||||
// optional navigate so the destination load can rearm capture.
|
// optional navigate so the destination load can rearm capture.
|
||||||
ensureBrowserObservationListeners(deps);
|
ensureBrowserObservationListeners(deps);
|
||||||
|
|
||||||
const abortPending = async (notifyContent: boolean) => {
|
const abortPending = async (notifyContent: boolean) => {
|
||||||
|
recordings.get(params.session_id)?.observation?.cancel();
|
||||||
|
deps.frameCoordinator?.cancel(requestId);
|
||||||
recordings.delete(params.session_id);
|
recordings.delete(params.session_id);
|
||||||
releaseBrowserObservationListenersIfIdle();
|
releaseBrowserObservationListenersIfIdle();
|
||||||
if (notifyContent) {
|
if (notifyContent) {
|
||||||
@@ -686,7 +986,7 @@ export async function handleRecordStart(
|
|||||||
return cancelledError();
|
return cancelledError();
|
||||||
}
|
}
|
||||||
active.startUrl = startUrl;
|
active.startUrl = startUrl;
|
||||||
active.currentUrl = startUrl;
|
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
|
||||||
|
|
||||||
if (isContentScriptRestrictedUrl(startUrl)) {
|
if (isContentScriptRestrictedUrl(startUrl)) {
|
||||||
await abortPending(false);
|
await abortPending(false);
|
||||||
@@ -694,7 +994,7 @@ export async function handleRecordStart(
|
|||||||
code: "invalid_params",
|
code: "invalid_params",
|
||||||
message: params.url
|
message: params.url
|
||||||
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
||||||
: `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
|
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -721,6 +1021,10 @@ export async function handleRecordStart(
|
|||||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (deps.frameCoordinator) {
|
||||||
|
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
|
||||||
|
if (!frameStarted) throw new Error("top recording document did not start");
|
||||||
|
}
|
||||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||||
} catch {
|
} catch {
|
||||||
await abortPending(true);
|
await abortPending(true);
|
||||||
@@ -731,6 +1035,17 @@ export async function handleRecordStart(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (active.observation) {
|
||||||
|
const activeRecording = recordings.get(params.session_id);
|
||||||
|
if (activeRecording) {
|
||||||
|
try {
|
||||||
|
await activeRecording.observation?.captureInitial(target.tabId);
|
||||||
|
} catch {
|
||||||
|
// Proceed without initial observation; steps may be dropped by reducer.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
const cancelled = await abortIfCancelled(true);
|
const cancelled = await abortIfCancelled(true);
|
||||||
if (cancelled) return cancelled;
|
if (cancelled) return cancelled;
|
||||||
@@ -755,7 +1070,7 @@ export async function handleRecordStop(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const trace = await finishRecording(params.session_id, deps);
|
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||||
if (!trace) {
|
if (!trace) {
|
||||||
return {
|
return {
|
||||||
code: "protocol_error",
|
code: "protocol_error",
|
||||||
@@ -785,9 +1100,9 @@ export async function handleRecordAwait(
|
|||||||
return { code: "cancelled", message: "record_await aborted" };
|
return { code: "cancelled", message: "record_await aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
const outcome = await new Promise<{ trace: RecordedTrace } | { error: RpcError }>((resolve) => {
|
||||||
let settled = false;
|
let settled = false;
|
||||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
const finish = (result: { trace: RecordedTrace } | { error: RpcError }) => {
|
||||||
if (settled) return;
|
if (settled) return;
|
||||||
settled = true;
|
settled = true;
|
||||||
if (timer) clearTimeout(timer);
|
if (timer) clearTimeout(timer);
|
||||||
@@ -829,6 +1144,8 @@ export function clearRecordingForSession(sessionId: string): void {
|
|||||||
}
|
}
|
||||||
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
||||||
if (!recording.settled) {
|
if (!recording.settled) {
|
||||||
|
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
|
||||||
|
recording.observation?.cancel();
|
||||||
recording.settled = true;
|
recording.settled = true;
|
||||||
recording.rejectFinish(new Error("recording cleared"));
|
recording.rejectFinish(new Error("recording cleared"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,6 +76,7 @@ export interface SessionStopResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface SessionStopDeps {
|
export interface SessionStopDeps {
|
||||||
|
signal?: AbortSignal;
|
||||||
cdp?: {
|
cdp?: {
|
||||||
detachSession(sessionId: string): Promise<void>;
|
detachSession(sessionId: string): Promise<void>;
|
||||||
};
|
};
|
||||||
@@ -177,6 +178,9 @@ export async function handleSessionStop(
|
|||||||
message: `session ${params.session_id} unknown`,
|
message: `session ${params.session_id} unknown`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "session_stop aborted before teardown" };
|
||||||
|
}
|
||||||
|
|
||||||
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
||||||
// ids so deletions during iteration do not break the Map iterator.
|
// ids so deletions during iteration do not break the Map iterator.
|
||||||
@@ -187,6 +191,7 @@ export async function handleSessionStop(
|
|||||||
try {
|
try {
|
||||||
const tabManagement = {
|
const tabManagement = {
|
||||||
...(deps.tabManagement ?? {}),
|
...(deps.tabManagement ?? {}),
|
||||||
|
signal: deps.signal,
|
||||||
isAgentWindowId:
|
isAgentWindowId:
|
||||||
deps.tabManagement?.isAgentWindowId ??
|
deps.tabManagement?.isAgentWindowId ??
|
||||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||||
@@ -214,6 +219,17 @@ export async function handleSessionStop(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||||
|
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||||
|
return {
|
||||||
|
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||||
|
return_failures: nonCancelFailures,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { code: "cancelled", message: "session_stop aborted during tab return" };
|
||||||
|
}
|
||||||
|
|
||||||
const result: SessionStopResult = {};
|
const result: SessionStopResult = {};
|
||||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||||
if (returnFailures && returnFailures.length > 0) {
|
if (returnFailures && returnFailures.length > 0) {
|
||||||
@@ -233,6 +249,10 @@ export async function handleSessionStop(
|
|||||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||||
await deps.cdp?.detachSession(params.session_id);
|
await deps.cdp?.detachSession(params.session_id);
|
||||||
|
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||||
|
}
|
||||||
|
|
||||||
// Step 4: close the Agent Window and drop the context.
|
// Step 4: close the Agent Window and drop the context.
|
||||||
await manager.stop(params.session_id);
|
await manager.stop(params.session_id);
|
||||||
|
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ function fakeSnapshotReply() {
|
|||||||
},
|
},
|
||||||
layout: {
|
layout: {
|
||||||
nodeIndex: [1, 2, 3],
|
nodeIndex: [1, 2, 3],
|
||||||
// styles columns follow requested computedStyles order: [position, pointer-events]
|
// Legacy fixture: omitted style columns fall back to visible defaults.
|
||||||
styles: [
|
styles: [
|
||||||
[i("static"), i("auto")],
|
[i("static"), i("auto")],
|
||||||
[i("fixed"), i("auto")],
|
[i("fixed"), i("auto")],
|
||||||
@@ -682,6 +682,46 @@ describe("captureViewModel", () => {
|
|||||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("captures painted visibility as DOM fallback evidence", async () => {
|
||||||
|
const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
|
||||||
|
const i = (value: string) => S.indexOf(value);
|
||||||
|
const snapshot = {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14],
|
||||||
|
attributes: [[], [], [], [], []],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3, 4],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||||
|
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||||
|
[i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
|
||||||
|
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[10, 10, 100, 30],
|
||||||
|
[10, 50, 100, 30],
|
||||||
|
[10, 90, 100, 30],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2, 3],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||||
|
|
||||||
|
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
||||||
|
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
||||||
|
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_t: number, method: string) => {
|
send: vi.fn(async (_t: number, method: string) => {
|
||||||
|
|||||||
@@ -132,4 +132,67 @@ describe("captureFrameData", () => {
|
|||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||||
|
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
||||||
|
const captured: CapturedViewModel = {
|
||||||
|
nodes: [owner],
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
frameNodes: new Map([["main", [owner]]]),
|
||||||
|
rootFrameId: "main",
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
};
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Accessibility.enable") return {};
|
||||||
|
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||||
|
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||||
|
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||||
|
if (method === "Accessibility.getFullAXTree") {
|
||||||
|
return {
|
||||||
|
nodes: [
|
||||||
|
{
|
||||||
|
nodeId: "button",
|
||||||
|
backendDOMNodeId: 101,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Continue" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const documents = await captureFrameData(cdp, 4, captured);
|
||||||
|
const childDocument = documents.find((document) => document.frameId === "child");
|
||||||
|
|
||||||
|
expect(childDocument?.axNodes).toEqual([
|
||||||
|
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||||
|
]);
|
||||||
|
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||||
|
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,183 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import type { CdpRunner } from "../../shared";
|
||||||
|
import type { CapturedNode } from "../capture";
|
||||||
|
import type { FrameDocument, FrameOwnedAxNode } from "../frame-document";
|
||||||
|
import { probeTooltipNames } from "../name-enrichment";
|
||||||
|
import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph";
|
||||||
|
import { stableIdentifierName } from "../semantic-graph/name-evidence";
|
||||||
|
import { frameBackendKey } from "../semantic-graph/types";
|
||||||
|
|
||||||
|
function document(
|
||||||
|
target: CdpTarget,
|
||||||
|
backendNodeId: number,
|
||||||
|
accessibleName?: string,
|
||||||
|
frameId = "child",
|
||||||
|
x = 10,
|
||||||
|
): FrameDocument<FrameOwnedAxNode> {
|
||||||
|
const node: CapturedNode = {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
frameId,
|
||||||
|
tag: "button",
|
||||||
|
attrs: {
|
||||||
|
id: "toolInsertRecord",
|
||||||
|
"data-test-id": "toolInsertRecord",
|
||||||
|
...(accessibleName ? { "aria-label": accessibleName } : {}),
|
||||||
|
},
|
||||||
|
rect: { x, y: 20, w: 40, h: 30 },
|
||||||
|
localRect: { x: 2, y: 3, w: 40, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
cursor: "pointer",
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
frameId,
|
||||||
|
parentFrameId: "main",
|
||||||
|
contextScopeId: "child",
|
||||||
|
target,
|
||||||
|
axNodes: [],
|
||||||
|
domNodes: [node],
|
||||||
|
url: "https://example.com/frame",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function semantics(documents: FrameDocument<FrameOwnedAxNode>[]) {
|
||||||
|
return resolveSemanticGraph(
|
||||||
|
buildSemanticGraph({
|
||||||
|
documents,
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: documents[0]?.frameId,
|
||||||
|
}),
|
||||||
|
{ identifierFallback: false },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("tooltip name enrichment", () => {
|
||||||
|
it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => {
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
let hovered = false;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method, params) => {
|
||||||
|
if (method === "Input.dispatchMouseEvent") {
|
||||||
|
hovered = (params as { x?: number }).x === 30;
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 11 };
|
||||||
|
if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } };
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
return { result: { value: hovered ? ["插入行"] : [] } };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const documents = [document(target, 501)];
|
||||||
|
const graph = semantics(documents);
|
||||||
|
const names = await probeTooltipNames(cdp, 7, documents, graph);
|
||||||
|
|
||||||
|
expect(names.get(frameBackendKey("child", 501))).toBe("插入行");
|
||||||
|
expect(cdp.sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
expect.objectContaining({ frameId: "child" }),
|
||||||
|
);
|
||||||
|
|
||||||
|
const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length;
|
||||||
|
const targetSendsBeforeCache = vi.mocked(
|
||||||
|
cdp.sendToTarget as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
).mock.calls.length;
|
||||||
|
const cached = await probeTooltipNames(cdp, 7, documents, graph);
|
||||||
|
expect(cached.get(frameBackendKey("child", 501))).toBe("插入行");
|
||||||
|
expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache);
|
||||||
|
expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not probe controls that already have a higher-confidence name", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const documents = [document({ tabId: 8 }, 601, "Add a new record")];
|
||||||
|
const names = await probeTooltipNames(cdp, 8, documents, semantics(documents));
|
||||||
|
|
||||||
|
expect(names.size).toBe(0);
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not collect supplemental names from form controls", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
const inputDocument = document({ tabId: 8 }, 602);
|
||||||
|
inputDocument.domNodes[0] = {
|
||||||
|
...inputDocument.domNodes[0],
|
||||||
|
tag: "input",
|
||||||
|
attrs: { type: "email" },
|
||||||
|
formValue: "user@example.com",
|
||||||
|
formDefaultValue: "",
|
||||||
|
formState: "filled",
|
||||||
|
};
|
||||||
|
|
||||||
|
const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
|
||||||
|
|
||||||
|
expect(names.size).toBe(0);
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||||
|
let hoveredX = -10;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method, params) => {
|
||||||
|
if (method === "Input.dispatchMouseEvent") {
|
||||||
|
hoveredX = (params as { x: number }).x;
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (target, method, params) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") {
|
||||||
|
return { executionContextId: target.sessionId === "left-session" ? 21 : 22 };
|
||||||
|
}
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } };
|
||||||
|
}
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const objectId = (params as { objectId: string }).objectId;
|
||||||
|
const value =
|
||||||
|
objectId === "left" && hoveredX === 30
|
||||||
|
? ["Left action"]
|
||||||
|
: objectId === "right" && hoveredX === 130
|
||||||
|
? ["Right action"]
|
||||||
|
: [];
|
||||||
|
return { result: { value } };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
const documents = [
|
||||||
|
document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10),
|
||||||
|
document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110),
|
||||||
|
];
|
||||||
|
|
||||||
|
const names = await probeTooltipNames(cdp, 9, documents, semantics(documents));
|
||||||
|
|
||||||
|
expect(names.get(frameBackendKey("left", 701))).toBe("Left action");
|
||||||
|
expect(names.get(frameBackendKey("right", 701))).toBe("Right action");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("stable identifier evidence", () => {
|
||||||
|
it("accepts readable action identifiers and rejects framework-generated ids", () => {
|
||||||
|
expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record");
|
||||||
|
expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account");
|
||||||
|
expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined();
|
||||||
|
expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
RECORD_DOCUMENT_ATTRIBUTE,
|
||||||
|
recordingDocumentMarkerValue,
|
||||||
|
} from "@/shared/recording-document-identity";
|
||||||
|
import { projectRecordSafeObservation } from "../record-safe-observation";
|
||||||
|
|
||||||
|
describe("projectRecordSafeObservation", () => {
|
||||||
|
it("projects the opaque recording identity without exposing captured attributes", () => {
|
||||||
|
const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000";
|
||||||
|
const result = projectRecordSafeObservation({
|
||||||
|
rootFrameId: "child",
|
||||||
|
frameDocuments: [
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 4, sessionId: "oopif-session" },
|
||||||
|
contextScopeId: "scope-1",
|
||||||
|
axNodes: [],
|
||||||
|
domNodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 1,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "html",
|
||||||
|
attrs: {
|
||||||
|
[RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId),
|
||||||
|
value: "user@example.com",
|
||||||
|
},
|
||||||
|
rect: null,
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.frames).toEqual([
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
target: { tabId: 4, sessionId: "oopif-session" },
|
||||||
|
recordingDocumentId,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
const dumped = JSON.stringify(result);
|
||||||
|
expect(dumped).not.toContain("user@example.com");
|
||||||
|
expect(dumped).not.toContain("attrs");
|
||||||
|
expect(dumped).not.toContain("domNodes");
|
||||||
|
expect(dumped).not.toContain("axNodes");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not project a page-controlled marker value as recording identity", () => {
|
||||||
|
const result = projectRecordSafeObservation({
|
||||||
|
rootFrameId: "root",
|
||||||
|
frameDocuments: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
target: { tabId: 4 },
|
||||||
|
contextScopeId: "scope-1",
|
||||||
|
axNodes: [],
|
||||||
|
domNodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 1,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "html",
|
||||||
|
attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" },
|
||||||
|
rect: null,
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||||
|
expect(JSON.stringify(result)).not.toContain("user@example.com");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,782 @@
|
|||||||
|
import { isVomReferenceNode, renderVom } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "../capture";
|
||||||
|
import type { FrameDocument } from "../frame-document";
|
||||||
|
import { buildSemanticVomScene, type SemanticAxNode } from "../semantic-graph";
|
||||||
|
|
||||||
|
function dom(
|
||||||
|
backendNodeId: number,
|
||||||
|
parentBackendNodeId: number | null,
|
||||||
|
tag: string,
|
||||||
|
attrs: Record<string, string> = {},
|
||||||
|
textContent?: string,
|
||||||
|
): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId,
|
||||||
|
tag,
|
||||||
|
attrs,
|
||||||
|
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||||
|
paintOrder: backendNodeId,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
...(textContent ? { textContent } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function document(
|
||||||
|
frameId: string,
|
||||||
|
axNodes: SemanticAxNode[],
|
||||||
|
domNodes: CapturedNode[],
|
||||||
|
frame?: Pick<FrameDocument<SemanticAxNode>, "parentFrameId" | "ownerBackendNodeId">,
|
||||||
|
): FrameDocument<SemanticAxNode> {
|
||||||
|
return {
|
||||||
|
frameId,
|
||||||
|
contextScopeId: frameId,
|
||||||
|
target: { tabId: 1 },
|
||||||
|
axNodes,
|
||||||
|
domNodes,
|
||||||
|
...frame,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("semantic VOM graph", () => {
|
||||||
|
it("preserves backend-less AX structure without making it referenceable", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["toolbar"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "toolbar",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "toolbar" },
|
||||||
|
name: { type: "computedString", value: "Formatting" },
|
||||||
|
childIds: ["bold", "italic"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "bold",
|
||||||
|
parentId: "toolbar",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Bold" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "italic",
|
||||||
|
parentId: "toolbar",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Italic" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), dom(2, 1, "button"), dom(3, 1, "button")],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const toolbar = scene.nodes.find((node) => node.role === "toolbar");
|
||||||
|
const buttons = scene.nodes.filter((node) => node.role === "button");
|
||||||
|
expect(toolbar).toEqual(expect.objectContaining({ name: "Formatting", referenceable: false }));
|
||||||
|
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
||||||
|
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
||||||
|
expect(renderVom(scene).refs.map((ref) => ref.backendNodeId)).toEqual([2, 3]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves missing AX names through one DOM naming pipeline", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["labelled", "titled", "text"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "labelled",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "titled",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "text",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
dom(1, null, "body"),
|
||||||
|
dom(2, 1, "button", { "aria-label": "Add row" }),
|
||||||
|
dom(3, 1, "button", { title: "Settings" }),
|
||||||
|
dom(4, 1, "button"),
|
||||||
|
dom(5, 4, "span", {}, "Save"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.nodes.filter((node) => node.role === "button").map((node) => node.name)).toEqual([
|
||||||
|
"Add row",
|
||||||
|
"Settings",
|
||||||
|
"Save",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("aggregates AX text at the semantic text frontier", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["covered", "inline-only", "repeated", "labelled"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "covered",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "paragraph" },
|
||||||
|
childIds: ["covered-static"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "covered-static",
|
||||||
|
parentId: "covered",
|
||||||
|
role: { type: "role", value: "StaticText" },
|
||||||
|
name: { type: "computedString", value: "Covered text" },
|
||||||
|
childIds: ["covered-inline"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "covered-inline",
|
||||||
|
parentId: "covered-static",
|
||||||
|
role: { type: "role", value: "InlineTextBox" },
|
||||||
|
name: { type: "computedString", value: "Covered text" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "inline-only",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "paragraph" },
|
||||||
|
childIds: ["inline-leaf"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "inline-leaf",
|
||||||
|
parentId: "inline-only",
|
||||||
|
role: { type: "role", value: "InlineTextBox" },
|
||||||
|
name: { type: "computedString", value: "Inline fallback" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "repeated",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "paragraph" },
|
||||||
|
childIds: ["first-go", "second-go"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "first-go",
|
||||||
|
parentId: "repeated",
|
||||||
|
role: { type: "role", value: "StaticText" },
|
||||||
|
name: { type: "computedString", value: "Go" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "second-go",
|
||||||
|
parentId: "repeated",
|
||||||
|
role: { type: "role", value: "StaticText" },
|
||||||
|
name: { type: "computedString", value: "Go" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "labelled",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 5,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
childIds: ["visible-label"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "visible-label",
|
||||||
|
parentId: "labelled",
|
||||||
|
role: { type: "role", value: "StaticText" },
|
||||||
|
name: { type: "computedString", value: "Visible text" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
dom(1, null, "body"),
|
||||||
|
dom(2, 1, "p"),
|
||||||
|
dom(3, 1, "p"),
|
||||||
|
dom(4, 1, "p"),
|
||||||
|
dom(5, 1, "button", { "aria-label": "Explicit action" }),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Covered text");
|
||||||
|
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Inline fallback");
|
||||||
|
expect(scene.nodes.find((node) => node.backendNodeId === 4)?.name).toBe("Go Go");
|
||||||
|
expect(scene.nodes.find((node) => node.backendNodeId === 5)?.name).toBe("Explicit action");
|
||||||
|
expect(renderVom(scene).text).not.toContain("Covered text Covered text");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses stable identifiers only as a final name fallback", () => {
|
||||||
|
const base = document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["fallback", "labelled", "container"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "fallback",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "labelled",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Localized name" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "container",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
dom(1, null, "body"),
|
||||||
|
dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }),
|
||||||
|
dom(3, 1, "button", { id: "toolIgnoredFallback" }),
|
||||||
|
dom(4, 1, "div", { id: "toolbarContainer" }),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
const staticScene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [base],
|
||||||
|
});
|
||||||
|
const enrichedScene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [base],
|
||||||
|
supplementalNames: new Map([["main\u00002", "插入行"]]),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record");
|
||||||
|
expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name");
|
||||||
|
expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined();
|
||||||
|
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行");
|
||||||
|
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe(
|
||||||
|
"Localized name",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("isolates equal backend node ids in sibling frames", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["owner-a", "owner-b"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "owner-a",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 10,
|
||||||
|
role: { type: "role", value: "Iframe" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "owner-b",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 20,
|
||||||
|
role: { type: "role", value: "Iframe" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), dom(10, 1, "iframe"), dom(20, 1, "iframe")],
|
||||||
|
),
|
||||||
|
document(
|
||||||
|
"frame-a",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "button-a",
|
||||||
|
backendDOMNodeId: 5,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
childIds: ["text-a"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "text-a",
|
||||||
|
parentId: "button-a",
|
||||||
|
role: { type: "role", value: "StaticText" },
|
||||||
|
name: { type: "computedString", value: "Accept" },
|
||||||
|
childIds: ["inline-a"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "inline-a",
|
||||||
|
parentId: "text-a",
|
||||||
|
role: { type: "role", value: "InlineTextBox" },
|
||||||
|
name: { type: "computedString", value: "Accept" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(5, null, "button")],
|
||||||
|
{ parentFrameId: "main", ownerBackendNodeId: 10 },
|
||||||
|
),
|
||||||
|
document(
|
||||||
|
"frame-b",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "button-b",
|
||||||
|
backendDOMNodeId: 5,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
childIds: ["text-b"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "text-b",
|
||||||
|
parentId: "button-b",
|
||||||
|
role: { type: "role", value: "StaticText" },
|
||||||
|
name: { type: "computedString", value: "Decline" },
|
||||||
|
childIds: ["inline-b"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "inline-b",
|
||||||
|
parentId: "text-b",
|
||||||
|
role: { type: "role", value: "InlineTextBox" },
|
||||||
|
name: { type: "computedString", value: "Decline" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(5, null, "button")],
|
||||||
|
{ parentFrameId: "main", ownerBackendNodeId: 20 },
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const frameButtons = scene.nodes.filter((node) => node.backendNodeId === 5);
|
||||||
|
const owners = new Map(
|
||||||
|
scene.nodes
|
||||||
|
.filter((node) => node.backendNodeId === 10 || node.backendNodeId === 20)
|
||||||
|
.map((node) => [node.backendNodeId, node.id]),
|
||||||
|
);
|
||||||
|
expect(frameButtons.map((node) => node.frameId)).toEqual(["frame-a", "frame-b"]);
|
||||||
|
expect(frameButtons.map((node) => node.name)).toEqual(["Accept", "Decline"]);
|
||||||
|
expect(frameButtons.map((node) => node.parentId)).toEqual([owners.get(10), owners.get(20)]);
|
||||||
|
expect(renderVom(scene).refs.map((ref) => [ref.frameId, ref.backendNodeId])).toEqual([
|
||||||
|
["frame-a", 5],
|
||||||
|
["frame-b", 5],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires render evidence before promoting DOM fallback semantics", () => {
|
||||||
|
const hiddenOwner = {
|
||||||
|
...dom(10, 1, "iframe", { hidden: "" }),
|
||||||
|
rect: null,
|
||||||
|
localRect: null,
|
||||||
|
rendered: false,
|
||||||
|
};
|
||||||
|
const hiddenButton = {
|
||||||
|
...dom(2, 1, "button", { "aria-label": "Hidden action" }),
|
||||||
|
rendered: false,
|
||||||
|
};
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), hiddenButton, hiddenOwner],
|
||||||
|
),
|
||||||
|
document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
|
||||||
|
expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
|
||||||
|
expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
|
||||||
|
expect(renderVom(scene).refs).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[],
|
||||||
|
[dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(renderVom(scene).refs).toEqual([
|
||||||
|
expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets explicit DOM structure override ignored AX metadata", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["toolbar"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "toolbar",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
ignored: true,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["wrapper"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "wrapper",
|
||||||
|
parentId: "toolbar",
|
||||||
|
backendDOMNodeId: 5,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["bold", "italic"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "bold",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Bold" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "italic",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Italic" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
dom(1, null, "body"),
|
||||||
|
dom(2, 1, "div", { role: "toolbar" }),
|
||||||
|
dom(5, 2, "div"),
|
||||||
|
dom(3, 5, "button"),
|
||||||
|
dom(4, 5, "button"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const toolbar = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||||
|
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
||||||
|
expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" }));
|
||||||
|
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
||||||
|
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||||
|
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("infers an unnamed group only from independent interaction branches", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["wrapper"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "wrapper",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["first", "second"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "first",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "First" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "second",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Second" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const group = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||||
|
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
||||||
|
expect(group).toEqual(expect.objectContaining({ role: "group" }));
|
||||||
|
expect(group?.name).toBeUndefined();
|
||||||
|
expect(group && isVomReferenceNode(group)).toBe(false);
|
||||||
|
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
||||||
|
|
||||||
|
const rendered = renderVom(scene);
|
||||||
|
const lines = rendered.text.split("\n");
|
||||||
|
const groupLine = lines.findIndex((line) => line.trim() === "group");
|
||||||
|
expect(groupLine).toBeGreaterThanOrEqual(0);
|
||||||
|
for (const ref of rendered.refs) {
|
||||||
|
expect(ref.line).toBeGreaterThan(groupLine);
|
||||||
|
expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never promotes a form value into its semantic name through supplemental evidence", () => {
|
||||||
|
const secret = "user@example.com";
|
||||||
|
const input = {
|
||||||
|
...dom(2, 1, "input", { type: "email" }),
|
||||||
|
formValue: secret,
|
||||||
|
formDefaultValue: "",
|
||||||
|
formState: "filled" as const,
|
||||||
|
};
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
supplementalNames: new Map([["main\u00002", secret]]),
|
||||||
|
identifierFallback: false,
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["email"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "email",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
value: { value: secret },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), input],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const field = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||||
|
expect(field?.name).toBeUndefined();
|
||||||
|
const rendered = renderVom(scene, { redactValues: true });
|
||||||
|
expect(rendered.text).toContain("•••");
|
||||||
|
expect(JSON.stringify(rendered)).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps uncertain single-branch wrappers flat", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["wrapper"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "wrapper",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["button"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "button",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Only action" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||||
|
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe(
|
||||||
|
scene.nodes.find((node) => node.backendNodeId === 1)?.id,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not infer across mixed content boundaries", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["wrapper"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "wrapper",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["first", "second", "paragraph"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "first",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "First" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "second",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Second" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "paragraph",
|
||||||
|
parentId: "wrapper",
|
||||||
|
backendDOMNodeId: 5,
|
||||||
|
role: { type: "role", value: "paragraph" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
dom(1, null, "body"),
|
||||||
|
dom(2, 1, "div"),
|
||||||
|
dom(3, 2, "button"),
|
||||||
|
dom(4, 2, "button"),
|
||||||
|
dom(5, 2, "p", {}, "Description"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps only the innermost reliable inferred group", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["outer"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "outer",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["inner", "third"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "inner",
|
||||||
|
parentId: "outer",
|
||||||
|
backendDOMNodeId: 3,
|
||||||
|
role: { type: "role", value: "generic" },
|
||||||
|
childIds: ["first", "second"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "first",
|
||||||
|
parentId: "inner",
|
||||||
|
backendDOMNodeId: 4,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "First" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "second",
|
||||||
|
parentId: "inner",
|
||||||
|
backendDOMNodeId: 5,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Second" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "third",
|
||||||
|
parentId: "outer",
|
||||||
|
backendDOMNodeId: 6,
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Third" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
dom(1, null, "body"),
|
||||||
|
dom(2, 1, "div"),
|
||||||
|
dom(3, 2, "div"),
|
||||||
|
dom(4, 3, "button"),
|
||||||
|
dom(5, 3, "button"),
|
||||||
|
dom(6, 2, "button"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const groups = scene.nodes.filter((node) => node.role === "group");
|
||||||
|
expect(groups.map((node) => node.backendNodeId)).toEqual([3]);
|
||||||
|
expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,16 +1,17 @@
|
|||||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||||
// shared string table); we request exactly three computed styles so the
|
// shared string table); requested computed styles are decoded through
|
||||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||||
|
|
||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
import { clearHover, waitForHover } from "./hover-perception";
|
||||||
|
|
||||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||||
const STYLE_COL = Object.fromEntries(
|
const STYLE_COL = Object.fromEntries(
|
||||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||||
@@ -38,6 +39,12 @@ export interface CapturedNode {
|
|||||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||||
*/
|
*/
|
||||||
cursor?: string;
|
cursor?: string;
|
||||||
|
/**
|
||||||
|
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||||
|
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||||
|
* nodes remain authoritative even when they are outside the viewport.
|
||||||
|
*/
|
||||||
|
rendered?: boolean;
|
||||||
textContent?: string;
|
textContent?: string;
|
||||||
formState?: "empty" | "filled" | "default";
|
formState?: "empty" | "filled" | "default";
|
||||||
formValue?: string;
|
formValue?: string;
|
||||||
@@ -460,36 +467,6 @@ function hoverStateExpression(): string {
|
|||||||
})()`;
|
})()`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
|
||||||
throwIfAborted(signal);
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
const timer = setTimeout(() => {
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
resolve();
|
|
||||||
}, ms);
|
|
||||||
const onAbort = () => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
reject(captureAbortError());
|
|
||||||
};
|
|
||||||
signal?.addEventListener("abort", onAbort, { once: true });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
|
||||||
await cdp
|
|
||||||
.send(tabId, "Input.dispatchMouseEvent", {
|
|
||||||
type: "mouseMoved",
|
|
||||||
x: -10,
|
|
||||||
y: -10,
|
|
||||||
})
|
|
||||||
.catch(() =>
|
|
||||||
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
|
||||||
// best effort
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function capturedText(node: CapturedNode): string | undefined {
|
function capturedText(node: CapturedNode): string | undefined {
|
||||||
const value =
|
const value =
|
||||||
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||||
@@ -669,7 +646,7 @@ async function probeHoverSurfaces(
|
|||||||
try {
|
try {
|
||||||
await clearHover(cdp, tabId);
|
await clearHover(cdp, tabId);
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await wait(HOVER_SETTLE_MS, options.signal);
|
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: hoverStateExpression(),
|
expression: hoverStateExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
@@ -684,7 +661,7 @@ async function probeHoverSurfaces(
|
|||||||
y: candidate.y,
|
y: candidate.y,
|
||||||
});
|
});
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await wait(HOVER_SETTLE_MS, options.signal);
|
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: hoverStateExpression(),
|
expression: hoverStateExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
@@ -817,6 +794,8 @@ function parseDocumentNodes(
|
|||||||
const posCol = STYLE_COL.position;
|
const posCol = STYLE_COL.position;
|
||||||
const peCol = STYLE_COL["pointer-events"];
|
const peCol = STYLE_COL["pointer-events"];
|
||||||
const cursorCol = STYLE_COL.cursor;
|
const cursorCol = STYLE_COL.cursor;
|
||||||
|
const visibilityCol = STYLE_COL.visibility;
|
||||||
|
const opacityCol = STYLE_COL.opacity;
|
||||||
const inputValues = sparseIndexMap(dn.inputValue);
|
const inputValues = sparseIndexMap(dn.inputValue);
|
||||||
const textValues = sparseIndexMap(dn.textValue);
|
const textValues = sparseIndexMap(dn.textValue);
|
||||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||||
@@ -861,6 +840,7 @@ function parseDocumentNodes(
|
|||||||
let position = "static";
|
let position = "static";
|
||||||
let pointerEvents = "auto";
|
let pointerEvents = "auto";
|
||||||
let cursor = "auto";
|
let cursor = "auto";
|
||||||
|
let rendered = false;
|
||||||
const li = layoutByNode.get(n);
|
const li = layoutByNode.get(n);
|
||||||
if (li !== undefined) {
|
if (li !== undefined) {
|
||||||
const b = dl?.bounds?.[li];
|
const b = dl?.bounds?.[li];
|
||||||
@@ -881,6 +861,13 @@ function parseDocumentNodes(
|
|||||||
position = str(strings, styleRow[posCol]) || "static";
|
position = str(strings, styleRow[posCol]) || "static";
|
||||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||||
|
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||||
|
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||||
|
rendered =
|
||||||
|
localRect !== null &&
|
||||||
|
visibility !== "hidden" &&
|
||||||
|
visibility !== "collapse" &&
|
||||||
|
(Number.parseFloat(opacity) || 0) > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||||
@@ -916,6 +903,7 @@ function parseDocumentNodes(
|
|||||||
position,
|
position,
|
||||||
pointerEvents,
|
pointerEvents,
|
||||||
cursor,
|
cursor,
|
||||||
|
rendered,
|
||||||
textContent,
|
textContent,
|
||||||
...(formValue !== undefined ? { formValue } : {}),
|
...(formValue !== undefined ? { formValue } : {}),
|
||||||
...(tag === "input" || tag === "textarea"
|
...(tag === "input" || tag === "textarea"
|
||||||
@@ -1020,7 +1008,13 @@ export async function captureViewModel(
|
|||||||
options: CaptureViewModelOptions = {},
|
options: CaptureViewModelOptions = {},
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
let metrics: LayoutMetricsReply = {};
|
||||||
|
try {
|
||||||
|
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
|
} catch (error) {
|
||||||
|
if (isAbortError(error)) throw error;
|
||||||
|
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||||
|
}
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const dpr = devicePixelRatio(metrics);
|
const dpr = devicePixelRatio(metrics);
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
|||||||
@@ -33,25 +33,19 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
|
|||||||
|
|
||||||
function transformFrameNodes(
|
function transformFrameNodes(
|
||||||
nodes: CapturedNode[],
|
nodes: CapturedNode[],
|
||||||
projection: GeometryProjection,
|
projection: GeometryProjection | null,
|
||||||
): CapturedNode[] {
|
): CapturedNode[] {
|
||||||
return nodes.map((node) => ({
|
return nodes.map((node) => ({
|
||||||
...node,
|
...node,
|
||||||
rect: (() => {
|
rect: projection
|
||||||
const bounds = projectRectToViewport(node.rect, projection);
|
? (() => {
|
||||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
const bounds = projectRectToViewport(node.rect, projection);
|
||||||
})(),
|
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||||
|
})()
|
||||||
|
: null,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
|
||||||
if (frame.ownerBackendNodeId === undefined) return undefined;
|
|
||||||
const parentNodes = frame.parentFrameId
|
|
||||||
? captured.frameNodes?.get(frame.parentFrameId)
|
|
||||||
: captured.nodes;
|
|
||||||
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function captureMissingFrameDocuments(
|
async function captureMissingFrameDocuments(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -65,14 +59,20 @@ async function captureMissingFrameDocuments(
|
|||||||
|
|
||||||
for (const frame of graph.frames) {
|
for (const frame of graph.frames) {
|
||||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||||
const owner = frameOwnerNode(captured, frame);
|
|
||||||
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
|
||||||
try {
|
try {
|
||||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||||
signal,
|
signal,
|
||||||
});
|
});
|
||||||
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
let projection: GeometryProjection | null = null;
|
||||||
if (!projection) continue;
|
try {
|
||||||
|
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) throw err;
|
||||||
|
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
err,
|
||||||
|
});
|
||||||
|
}
|
||||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||||
@@ -85,11 +85,13 @@ async function captureMissingFrameDocuments(
|
|||||||
for (const [childFrameId, nodes] of childFrames) {
|
for (const [childFrameId, nodes] of childFrames) {
|
||||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||||
}
|
}
|
||||||
captured.iframeNodes.set(
|
if (frame.ownerBackendNodeId !== undefined) {
|
||||||
frame.ownerBackendNodeId,
|
captured.iframeNodes.set(
|
||||||
captured.frameNodes.get(frame.frameId) ?? [],
|
frame.ownerBackendNodeId,
|
||||||
);
|
captured.frameNodes.get(frame.frameId) ?? [],
|
||||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
);
|
||||||
|
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||||
|
}
|
||||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||||
|
|||||||
@@ -77,9 +77,12 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
|||||||
): FrameDocument<T>[] {
|
): FrameDocument<T>[] {
|
||||||
const frames = frameList(graph, batches, captured);
|
const frames = frameList(graph, batches, captured);
|
||||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||||
|
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||||
|
captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
|
||||||
const backendOwner = new Map<string, string>();
|
const backendOwner = new Map<string, string>();
|
||||||
for (const frame of frames) {
|
for (const frame of frames) {
|
||||||
for (const node of captured.frameNodes?.get(frame.frameId) ?? []) {
|
for (const node of domNodesForFrame(frame.frameId)) {
|
||||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -165,6 +168,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
|||||||
...frame,
|
...frame,
|
||||||
contextScopeId: frame.frameId,
|
contextScopeId: frame.frameId,
|
||||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||||
domNodes: captured.frameNodes?.get(frame.frameId) ?? [],
|
domNodes: domNodesForFrame(frame.frameId),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
function abortError(): Error {
|
||||||
|
const error = new Error("hover perception aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForHover(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
if (signal?.aborted) throw abortError();
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
reject(abortError());
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||||
|
await cdp
|
||||||
|
.send(tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: -10,
|
||||||
|
y: -10,
|
||||||
|
})
|
||||||
|
.catch(() =>
|
||||||
|
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
||||||
|
// Best effort cleanup.
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,296 @@
|
|||||||
|
import { isVomReferenceNode } from "@browser-skill/vom";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||||
|
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
||||||
|
import { clearHover, waitForHover } from "./hover-perception";
|
||||||
|
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
||||||
|
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
||||||
|
|
||||||
|
const MAX_TOOLTIP_CANDIDATES = 8;
|
||||||
|
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||||
|
const TOOLTIP_SETTLE_MS = 250;
|
||||||
|
const CACHE_TTL_MS = 5 * 60_000;
|
||||||
|
const MAX_CACHE_ENTRIES = 500;
|
||||||
|
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||||
|
|
||||||
|
interface TooltipCacheEntry {
|
||||||
|
name: string;
|
||||||
|
expiresAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
||||||
|
document: FrameDocument<T>;
|
||||||
|
backendNodeId: number;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
stableIdentifier: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TooltipNameProbeOptions {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
||||||
|
|
||||||
|
function cacheKey<T extends FrameOwnedAxNode>(
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
backendNodeId: number,
|
||||||
|
): string {
|
||||||
|
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cachedName<T extends FrameOwnedAxNode>(
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
backendNodeId: number,
|
||||||
|
): string | undefined {
|
||||||
|
const key = cacheKey(document, backendNodeId);
|
||||||
|
const entry = tooltipCache.get(key);
|
||||||
|
if (!entry) return undefined;
|
||||||
|
if (entry.expiresAt <= Date.now()) {
|
||||||
|
tooltipCache.delete(key);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return entry.name;
|
||||||
|
}
|
||||||
|
|
||||||
|
function storeName<T extends FrameOwnedAxNode>(
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
backendNodeId: number,
|
||||||
|
name: string,
|
||||||
|
): void {
|
||||||
|
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
||||||
|
const oldest = tooltipCache.keys().next().value;
|
||||||
|
if (oldest !== undefined) tooltipCache.delete(oldest);
|
||||||
|
}
|
||||||
|
tooltipCache.set(cacheKey(document, backendNodeId), {
|
||||||
|
name,
|
||||||
|
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function tooltipCollectorFunction(): string {
|
||||||
|
return `function() {
|
||||||
|
const ownRoot = this && typeof this.getRootNode === "function" ? this.getRootNode() : null;
|
||||||
|
const roots = ownRoot && ownRoot !== document ? [ownRoot, document] : [document];
|
||||||
|
const items = [];
|
||||||
|
const seen = new Set();
|
||||||
|
for (const root of roots) {
|
||||||
|
for (const el of Array.from(root.querySelectorAll("[role='tooltip']"))) {
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 || rect.height <= 0) continue;
|
||||||
|
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
||||||
|
const text = String(el.getAttribute("aria-label") || el.getAttribute("title") || el.textContent || "")
|
||||||
|
.replace(/\\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
if (!text || text.length > 160 || seen.has(text)) continue;
|
||||||
|
seen.add(text);
|
||||||
|
items.push(text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return items.slice(0, 20);
|
||||||
|
}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanTooltipDiff(before: string[], after: string[]): string | undefined {
|
||||||
|
const existing = new Set(before.map((value) => value.toLowerCase()));
|
||||||
|
const added = [...new Set(after.map((value) => value.replace(/\s+/g, " ").trim()))].filter(
|
||||||
|
(value) => value && !existing.has(value.toLowerCase()),
|
||||||
|
);
|
||||||
|
return added.length === 1 ? added[0] : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidates<T extends FrameOwnedAxNode>(
|
||||||
|
documents: FrameDocument<T>[],
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
): TooltipCandidate<T>[] {
|
||||||
|
const documentByFrame = new Map(documents.map((document) => [document.frameId, document]));
|
||||||
|
const byFrame = new Map<string, TooltipCandidate<T>[]>();
|
||||||
|
for (const node of graph.nodes.values()) {
|
||||||
|
if (
|
||||||
|
node.vom.name ||
|
||||||
|
FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") ||
|
||||||
|
!node.referenceable ||
|
||||||
|
node.backendNodeId === undefined ||
|
||||||
|
!isVomReferenceNode({
|
||||||
|
...node.vom,
|
||||||
|
id: 0,
|
||||||
|
parentId: null,
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
frameId: node.frameId,
|
||||||
|
referenceable: true,
|
||||||
|
})
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const document = documentByFrame.get(node.frameId);
|
||||||
|
const rect = node.dom?.rect;
|
||||||
|
if (!document || !rect || rect.w <= 0 || rect.h <= 0) continue;
|
||||||
|
if (
|
||||||
|
rect.x + rect.w <= 0 ||
|
||||||
|
rect.y + rect.h <= 0 ||
|
||||||
|
rect.x >= graph.viewport.width ||
|
||||||
|
rect.y >= graph.viewport.height
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
node.vom.disabled ||
|
||||||
|
node.vom.inert ||
|
||||||
|
node.vom.pointerEvents === "none" ||
|
||||||
|
node.vom.sensitive
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const frameCandidates = byFrame.get(node.frameId) ?? [];
|
||||||
|
frameCandidates.push({
|
||||||
|
document,
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
x: (Math.max(0, rect.x) + Math.min(graph.viewport.width, rect.x + rect.w)) / 2,
|
||||||
|
y: (Math.max(0, rect.y) + Math.min(graph.viewport.height, rect.y + rect.h)) / 2,
|
||||||
|
stableIdentifier: stableIdentifierName(node.dom?.attrs ?? {}) !== undefined,
|
||||||
|
});
|
||||||
|
byFrame.set(node.frameId, frameCandidates);
|
||||||
|
}
|
||||||
|
|
||||||
|
const out: TooltipCandidate<T>[] = [];
|
||||||
|
for (const stableIdentifier of [true, false]) {
|
||||||
|
const queues = documents.map((document) =>
|
||||||
|
(byFrame.get(document.frameId) ?? []).filter(
|
||||||
|
(candidate) => candidate.stableIdentifier === stableIdentifier,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
for (let index = 0; queues.some((queue) => index < queue.length); index += 1) {
|
||||||
|
for (const queue of queues) {
|
||||||
|
const candidate = queue[index];
|
||||||
|
if (candidate) out.push(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executionContextId<T extends FrameOwnedAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
): Promise<number | undefined> {
|
||||||
|
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||||
|
cdp,
|
||||||
|
document.target,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
{
|
||||||
|
frameId: document.frameId,
|
||||||
|
worldName: "bsk-vom-tooltip",
|
||||||
|
grantUniveralAccess: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return result.executionContextId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolvedObjectId<T extends FrameOwnedAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
contextId: number,
|
||||||
|
backendNodeId: number,
|
||||||
|
): Promise<string | undefined> {
|
||||||
|
const result = await sendToCdpTarget<{
|
||||||
|
object?: { objectId?: string };
|
||||||
|
}>(cdp, document.target, "DOM.resolveNode", {
|
||||||
|
backendNodeId,
|
||||||
|
executionContextId: contextId,
|
||||||
|
});
|
||||||
|
return result.object?.objectId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function tooltipTexts<T extends FrameOwnedAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
objectId: string,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const result = await sendToCdpTarget<{
|
||||||
|
result?: { value?: string[] };
|
||||||
|
}>(cdp, document.target, "Runtime.callFunctionOn", {
|
||||||
|
functionDeclaration: tooltipCollectorFunction(),
|
||||||
|
objectId,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
return result.result?.value ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function releaseObject<T extends FrameOwnedAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
document: FrameDocument<T>,
|
||||||
|
objectId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObject", { objectId }).catch(
|
||||||
|
() => undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
documents: FrameDocument<T>[],
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
options: TooltipNameProbeOptions = {},
|
||||||
|
): Promise<Map<string, string>> {
|
||||||
|
const names = new Map<string, string>();
|
||||||
|
const pending: TooltipCandidate<T>[] = [];
|
||||||
|
for (const candidate of candidates(documents, graph)) {
|
||||||
|
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
||||||
|
if (cached) {
|
||||||
|
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
||||||
|
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
||||||
|
pending.push(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (pending.length === 0) return names;
|
||||||
|
|
||||||
|
const started = Date.now();
|
||||||
|
const contexts = new Map<string, number>();
|
||||||
|
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||||
|
try {
|
||||||
|
for (const candidate of pending) {
|
||||||
|
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
|
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
||||||
|
let objectId: string | undefined;
|
||||||
|
try {
|
||||||
|
let contextId = contexts.get(candidate.document.frameId);
|
||||||
|
if (contextId === undefined) {
|
||||||
|
contextId = await executionContextId(cdp, candidate.document);
|
||||||
|
if (contextId === undefined) continue;
|
||||||
|
contexts.set(candidate.document.frameId, contextId);
|
||||||
|
}
|
||||||
|
objectId = await resolvedObjectId(
|
||||||
|
cdp,
|
||||||
|
candidate.document,
|
||||||
|
contextId,
|
||||||
|
candidate.backendNodeId,
|
||||||
|
);
|
||||||
|
if (!objectId) continue;
|
||||||
|
await clearHover(cdp, tabId);
|
||||||
|
const before = await tooltipTexts(cdp, candidate.document, objectId);
|
||||||
|
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: candidate.x,
|
||||||
|
y: candidate.y,
|
||||||
|
});
|
||||||
|
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
||||||
|
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
||||||
|
const name = cleanTooltipDiff(before, after);
|
||||||
|
if (!name) continue;
|
||||||
|
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
||||||
|
storeName(candidate.document, candidate.backendNodeId, name);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||||
|
} finally {
|
||||||
|
if (objectId) await releaseObject(cdp, candidate.document, objectId);
|
||||||
|
await clearHover(cdp, tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await clearHover(cdp, tabId);
|
||||||
|
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||||
|
}
|
||||||
|
return names;
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
|
||||||
|
import type { CapturedSurfaceProbe } from "./capture";
|
||||||
|
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||||
|
|
||||||
|
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||||
|
export interface CaptureVomFrame {
|
||||||
|
frameId: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
parentFrameId?: string;
|
||||||
|
ownerBackendNodeId?: number;
|
||||||
|
/** BrowserSkill-generated identity for the recording agent in this Document. */
|
||||||
|
recordingDocumentId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
|
||||||
|
export interface CaptureVomMatchNode {
|
||||||
|
frameId: string;
|
||||||
|
backendNodeId: number;
|
||||||
|
tag: string;
|
||||||
|
/** Top-level viewport-relative geometry. */
|
||||||
|
rect: Rect | null;
|
||||||
|
/** Frame-local viewport-relative geometry, when available. */
|
||||||
|
localRect?: Rect | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
|
||||||
|
* caller, form values in `text` are masked as well.
|
||||||
|
*/
|
||||||
|
export interface CaptureVomObservationResult {
|
||||||
|
text: string;
|
||||||
|
refs: RenderedRef[];
|
||||||
|
truncated: boolean;
|
||||||
|
rootFrameId: string;
|
||||||
|
frames: CaptureVomFrame[];
|
||||||
|
matchNodes: CaptureVomMatchNode[];
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||||
|
return documents.map((document) => {
|
||||||
|
let recordingDocumentId: string | undefined;
|
||||||
|
for (const node of document.domNodes) {
|
||||||
|
recordingDocumentId = readRecordingDocumentIdentity(node.attrs);
|
||||||
|
if (recordingDocumentId) break;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
frameId: document.frameId,
|
||||||
|
target: document.target,
|
||||||
|
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||||
|
...(document.ownerBackendNodeId !== undefined
|
||||||
|
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||||
|
: {}),
|
||||||
|
...(recordingDocumentId ? { recordingDocumentId } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
|
||||||
|
return documents.flatMap((document) =>
|
||||||
|
document.domNodes.map((node) => ({
|
||||||
|
frameId: document.frameId,
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
tag: node.tag,
|
||||||
|
rect: node.rect,
|
||||||
|
...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectRecordSafeObservation(input: {
|
||||||
|
rootFrameId: string;
|
||||||
|
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||||
|
rendered: VomResult;
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
}): CaptureVomObservationResult {
|
||||||
|
return {
|
||||||
|
text: input.rendered.text,
|
||||||
|
refs: input.rendered.refs,
|
||||||
|
truncated: input.rendered.truncated,
|
||||||
|
rootFrameId: input.rootFrameId,
|
||||||
|
frames: projectFrames(input.frameDocuments),
|
||||||
|
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||||
|
surfaceProbes: input.surfaceProbes,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
import type { Viewport } from "@browser-skill/vom";
|
||||||
|
import type { FrameDocument } from "../frame-document";
|
||||||
|
import {
|
||||||
|
frameAxKey,
|
||||||
|
frameBackendKey,
|
||||||
|
type SemanticAxNode,
|
||||||
|
type SemanticFrame,
|
||||||
|
type SemanticGraph,
|
||||||
|
type SemanticGraphNode,
|
||||||
|
type SemanticNodeId,
|
||||||
|
} from "./types";
|
||||||
|
|
||||||
|
export interface BuildSemanticGraphInput {
|
||||||
|
documents: FrameDocument<SemanticAxNode>[];
|
||||||
|
viewport: Viewport;
|
||||||
|
rootFrameId?: string;
|
||||||
|
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function domNodeId(frameId: string, backendNodeId: number): SemanticNodeId {
|
||||||
|
return JSON.stringify([frameId, "dom", backendNodeId]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function axNodeId(frameId: string, nodeId: string): SemanticNodeId {
|
||||||
|
return JSON.stringify([frameId, "ax", nodeId]);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGraph {
|
||||||
|
const rootFrameId =
|
||||||
|
input.rootFrameId ??
|
||||||
|
input.documents.find((document) => !document.parentFrameId)?.frameId ??
|
||||||
|
input.documents[0]?.frameId ??
|
||||||
|
"root";
|
||||||
|
const frames = new Map<string, SemanticFrame>();
|
||||||
|
const nodes = new Map<SemanticNodeId, SemanticGraphNode>();
|
||||||
|
const nodeIdsByFrame = new Map<string, SemanticNodeId[]>();
|
||||||
|
const nodeByFrameBackend = new Map<string, SemanticNodeId>();
|
||||||
|
const nodeByFrameAx = new Map<string, SemanticNodeId>();
|
||||||
|
|
||||||
|
for (const [frameOrder, document] of input.documents.entries()) {
|
||||||
|
frames.set(document.frameId, {
|
||||||
|
frameId: document.frameId,
|
||||||
|
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||||
|
...(document.ownerBackendNodeId !== undefined
|
||||||
|
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||||
|
: {}),
|
||||||
|
contextScopeId: document.contextScopeId,
|
||||||
|
target: document.target,
|
||||||
|
...(document.url ? { url: document.url } : {}),
|
||||||
|
order: frameOrder,
|
||||||
|
});
|
||||||
|
|
||||||
|
const frameNodeIds: SemanticNodeId[] = [];
|
||||||
|
nodeIdsByFrame.set(document.frameId, frameNodeIds);
|
||||||
|
for (const [index, dom] of document.domNodes.entries()) {
|
||||||
|
const id = domNodeId(document.frameId, dom.backendNodeId);
|
||||||
|
const node: SemanticGraphNode = {
|
||||||
|
id,
|
||||||
|
frameId: document.frameId,
|
||||||
|
backendNodeId: dom.backendNodeId,
|
||||||
|
dom,
|
||||||
|
sourceOrder: index,
|
||||||
|
excluded:
|
||||||
|
document.frameId === rootFrameId &&
|
||||||
|
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
|
||||||
|
};
|
||||||
|
nodes.set(id, node);
|
||||||
|
frameNodeIds.push(id);
|
||||||
|
nodeByFrameBackend.set(frameBackendKey(document.frameId, dom.backendNodeId), id);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [index, ax] of document.axNodes.entries()) {
|
||||||
|
const joinedId =
|
||||||
|
typeof ax.backendDOMNodeId === "number"
|
||||||
|
? nodeByFrameBackend.get(frameBackendKey(document.frameId, ax.backendDOMNodeId))
|
||||||
|
: undefined;
|
||||||
|
const id = joinedId ?? axNodeId(document.frameId, ax.nodeId);
|
||||||
|
const existing = nodes.get(id);
|
||||||
|
if (existing) {
|
||||||
|
existing.ax = ax;
|
||||||
|
existing.axNodeId = ax.nodeId;
|
||||||
|
} else {
|
||||||
|
nodes.set(id, {
|
||||||
|
id,
|
||||||
|
frameId: document.frameId,
|
||||||
|
...(typeof ax.backendDOMNodeId === "number"
|
||||||
|
? { backendNodeId: ax.backendDOMNodeId }
|
||||||
|
: {}),
|
||||||
|
axNodeId: ax.nodeId,
|
||||||
|
ax,
|
||||||
|
sourceOrder: document.domNodes.length + index,
|
||||||
|
excluded:
|
||||||
|
document.frameId === rootFrameId &&
|
||||||
|
typeof ax.backendDOMNodeId === "number" &&
|
||||||
|
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
|
||||||
|
});
|
||||||
|
frameNodeIds.push(id);
|
||||||
|
}
|
||||||
|
nodeByFrameAx.set(frameAxKey(document.frameId, ax.nodeId), id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const document of input.documents) {
|
||||||
|
for (const dom of document.domNodes) {
|
||||||
|
const id = nodeByFrameBackend.get(frameBackendKey(document.frameId, dom.backendNodeId));
|
||||||
|
const node = id ? nodes.get(id) : undefined;
|
||||||
|
if (!node || dom.parentBackendNodeId === null) continue;
|
||||||
|
node.domParentId = nodeByFrameBackend.get(
|
||||||
|
frameBackendKey(document.frameId, dom.parentBackendNodeId),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const ax of document.axNodes) {
|
||||||
|
const id = nodeByFrameAx.get(frameAxKey(document.frameId, ax.nodeId));
|
||||||
|
const node = id ? nodes.get(id) : undefined;
|
||||||
|
if (!node || !ax.parentId) continue;
|
||||||
|
node.axParentId = nodeByFrameAx.get(frameAxKey(document.frameId, ax.parentId));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||||
|
const excludedQueue: SemanticNodeId[] = [];
|
||||||
|
for (const node of nodes.values()) {
|
||||||
|
if (node.axParentId) {
|
||||||
|
const children = axChildren.get(node.axParentId) ?? [];
|
||||||
|
children.push(node.id);
|
||||||
|
axChildren.set(node.axParentId, children);
|
||||||
|
}
|
||||||
|
if (node.excluded) excludedQueue.push(node.id);
|
||||||
|
}
|
||||||
|
for (let index = 0; index < excludedQueue.length; index += 1) {
|
||||||
|
for (const childId of axChildren.get(excludedQueue[index]) ?? []) {
|
||||||
|
const child = nodes.get(childId);
|
||||||
|
if (!child || child.excluded) continue;
|
||||||
|
child.excluded = true;
|
||||||
|
excludedQueue.push(childId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const frame of frames.values()) {
|
||||||
|
if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) continue;
|
||||||
|
frame.ownerNodeId = nodeByFrameBackend.get(
|
||||||
|
frameBackendKey(frame.parentFrameId, frame.ownerBackendNodeId),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
viewport: input.viewport,
|
||||||
|
rootFrameId,
|
||||||
|
frames,
|
||||||
|
nodes,
|
||||||
|
nodeIdsByFrame,
|
||||||
|
nodeByFrameBackend,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import type { Viewport, VomScene } from "@browser-skill/vom";
|
||||||
|
import type { FrameDocument } from "../frame-document";
|
||||||
|
import { buildSemanticGraph as buildGraph } from "./build";
|
||||||
|
import { projectSemanticGraph as projectGraph } from "./project";
|
||||||
|
import { resolveSemanticGraph as resolveGraph } from "./resolve";
|
||||||
|
import { normalizeSemanticStructure as normalizeStructure } from "./structure";
|
||||||
|
import type {
|
||||||
|
ResolvedSemanticGraph,
|
||||||
|
SemanticAxNode,
|
||||||
|
SemanticGraph,
|
||||||
|
StructuredSemanticGraph,
|
||||||
|
} from "./types";
|
||||||
|
|
||||||
|
export type { SemanticAxNode } from "./types";
|
||||||
|
|
||||||
|
export interface SemanticVomInput {
|
||||||
|
documents: FrameDocument<SemanticAxNode>[];
|
||||||
|
viewport: Viewport;
|
||||||
|
rootFrameId?: string;
|
||||||
|
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph {
|
||||||
|
return buildGraph(input);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveSemanticGraph(
|
||||||
|
graph: SemanticGraph,
|
||||||
|
options: {
|
||||||
|
supplementalNames?: ReadonlyMap<string, string>;
|
||||||
|
identifierFallback?: boolean;
|
||||||
|
} = {},
|
||||||
|
): ResolvedSemanticGraph {
|
||||||
|
return resolveGraph(graph, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
|
||||||
|
return normalizeStructure(graph);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
|
||||||
|
return projectGraph(graph);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSemanticVomScene(
|
||||||
|
input: SemanticVomInput & {
|
||||||
|
supplementalNames?: ReadonlyMap<string, string>;
|
||||||
|
identifierFallback?: boolean;
|
||||||
|
},
|
||||||
|
): VomScene {
|
||||||
|
return projectSemanticGraph(
|
||||||
|
normalizeSemanticStructure(
|
||||||
|
resolveSemanticGraph(buildSemanticGraph(input), {
|
||||||
|
supplementalNames: input.supplementalNames,
|
||||||
|
identifierFallback: input.identifierFallback,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const;
|
||||||
|
const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]);
|
||||||
|
const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]);
|
||||||
|
|
||||||
|
function identifierTokens(value: string): string[] {
|
||||||
|
return value
|
||||||
|
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
||||||
|
.split(/[^a-zA-Z0-9]+/)
|
||||||
|
.map((token) => token.toLowerCase())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function readableIdentifier(value: string | undefined): string | undefined {
|
||||||
|
const raw = value?.trim();
|
||||||
|
if (!raw || raw.length < 3 || raw.length > 80) return undefined;
|
||||||
|
if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined;
|
||||||
|
|
||||||
|
const tokens = identifierTokens(raw);
|
||||||
|
if (GENERATED_PREFIXES.has(tokens[0])) return undefined;
|
||||||
|
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift();
|
||||||
|
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
|
||||||
|
if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined;
|
||||||
|
return tokens
|
||||||
|
.map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token))
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stableIdentifierName(attrs: Readonly<Record<string, string>>): string | undefined {
|
||||||
|
for (const name of IDENTIFIER_ATTRS) {
|
||||||
|
const label = readableIdentifier(attrs[name]);
|
||||||
|
if (label) return label;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import type { VomNode, VomScene } from "@browser-skill/vom";
|
||||||
|
import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
|
||||||
|
|
||||||
|
function retained(node: StructuredSemanticNode): boolean {
|
||||||
|
return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
|
||||||
|
}
|
||||||
|
|
||||||
|
function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
|
||||||
|
const connected = new Set<string>([graph.rootFrameId]);
|
||||||
|
const childrenByParent = new Map<string, string[]>();
|
||||||
|
for (const frame of graph.frames.values()) {
|
||||||
|
if (!frame.parentFrameId) continue;
|
||||||
|
const children = childrenByParent.get(frame.parentFrameId) ?? [];
|
||||||
|
children.push(frame.frameId);
|
||||||
|
childrenByParent.set(frame.parentFrameId, children);
|
||||||
|
}
|
||||||
|
const queue = [graph.rootFrameId];
|
||||||
|
for (let index = 0; index < queue.length; index += 1) {
|
||||||
|
for (const frameId of childrenByParent.get(queue[index]) ?? []) {
|
||||||
|
if (connected.has(frameId)) continue;
|
||||||
|
const frame = graph.frames.get(frameId);
|
||||||
|
if (!frame?.ownerNodeId) continue;
|
||||||
|
const owner = graph.nodes.get(frame.ownerNodeId);
|
||||||
|
if (!owner || !retained(owner)) continue;
|
||||||
|
connected.add(frame.frameId);
|
||||||
|
queue.push(frame.frameId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return connected;
|
||||||
|
}
|
||||||
|
|
||||||
|
function domAncestors(
|
||||||
|
graph: StructuredSemanticGraph,
|
||||||
|
node: StructuredSemanticNode,
|
||||||
|
numericId: ReadonlyMap<SemanticNodeId, number>,
|
||||||
|
): number[] {
|
||||||
|
const ancestors: number[] = [];
|
||||||
|
const seen = new Set<SemanticNodeId>();
|
||||||
|
let current = node.domParentId;
|
||||||
|
while (current && !seen.has(current)) {
|
||||||
|
seen.add(current);
|
||||||
|
const id = numericId.get(current);
|
||||||
|
if (id !== undefined) ancestors.push(id);
|
||||||
|
current = graph.nodes.get(current)?.domParentId;
|
||||||
|
}
|
||||||
|
return ancestors;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
|
||||||
|
const connectedFrames = connectedFrameIds(graph);
|
||||||
|
const orderedNodes = [...graph.frames.values()]
|
||||||
|
.filter((frame) => connectedFrames.has(frame.frameId))
|
||||||
|
.sort((a, b) => a.order - b.order)
|
||||||
|
.flatMap((frame) =>
|
||||||
|
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
||||||
|
.map((id) => graph.nodes.get(id))
|
||||||
|
.filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
|
||||||
|
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
||||||
|
);
|
||||||
|
|
||||||
|
const numericId = new Map<SemanticNodeId, number>();
|
||||||
|
orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
|
||||||
|
const nodes: VomNode[] = orderedNodes.map((node) => ({
|
||||||
|
...node.vom,
|
||||||
|
id: numericId.get(node.id) as number,
|
||||||
|
parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
|
||||||
|
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||||
|
frameId: node.frameId,
|
||||||
|
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
||||||
|
referenceable: node.referenceable,
|
||||||
|
...(node.dom
|
||||||
|
? {
|
||||||
|
domParentId:
|
||||||
|
node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
|
||||||
|
domAncestorIds: domAncestors(graph, node, numericId),
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
viewport: graph.viewport,
|
||||||
|
nodes,
|
||||||
|
rootFrameId: graph.rootFrameId,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,521 @@
|
|||||||
|
import type { VomNode } from "@browser-skill/vom";
|
||||||
|
import { stableIdentifierName } from "./name-evidence";
|
||||||
|
import type {
|
||||||
|
ResolvedSemanticGraph,
|
||||||
|
ResolvedSemanticNode,
|
||||||
|
SemanticAxNode,
|
||||||
|
SemanticGraph,
|
||||||
|
SemanticGraphNode,
|
||||||
|
SemanticNodeId,
|
||||||
|
} from "./types";
|
||||||
|
import { frameBackendKey } from "./types";
|
||||||
|
|
||||||
|
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
||||||
|
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||||
|
const TEXT_AX_ROLES = new Set(["statictext", "inlinetextbox", "labeltext", "text"]);
|
||||||
|
const INTERACTIVE_ROLES = new Set([
|
||||||
|
"button",
|
||||||
|
"checkbox",
|
||||||
|
"combobox",
|
||||||
|
"link",
|
||||||
|
"listbox",
|
||||||
|
"menuitem",
|
||||||
|
"menuitemcheckbox",
|
||||||
|
"menuitemradio",
|
||||||
|
"option",
|
||||||
|
"radio",
|
||||||
|
"searchbox",
|
||||||
|
"slider",
|
||||||
|
"spinbutton",
|
||||||
|
"switch",
|
||||||
|
"tab",
|
||||||
|
"textbox",
|
||||||
|
"treeitem",
|
||||||
|
]);
|
||||||
|
const SENSITIVE_INPUT_TYPES = new Set([
|
||||||
|
"password",
|
||||||
|
"credit-card",
|
||||||
|
"one-time-code",
|
||||||
|
"current-password",
|
||||||
|
"new-password",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function clean(value: string | undefined): string | undefined {
|
||||||
|
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||||
|
return normalized || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function axValue(field?: { value?: string | number | boolean }): string | undefined {
|
||||||
|
return field?.value === undefined ? undefined : clean(String(field.value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function axProperty(node: SemanticAxNode | undefined, name: string): string | undefined {
|
||||||
|
return axValue(node?.properties?.find((property) => property.name === name)?.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRole(value: string | undefined): string | undefined {
|
||||||
|
const role = clean(value)?.toLowerCase();
|
||||||
|
return role || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
|
||||||
|
if (tag === "button") return "button";
|
||||||
|
if (tag === "textarea") return "textbox";
|
||||||
|
if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
|
||||||
|
if (tag === "a" && attrs.href) return "link";
|
||||||
|
if (tag === "iframe") return "Iframe";
|
||||||
|
if (tag === "dialog") return "dialog";
|
||||||
|
if (tag === "img") return "img";
|
||||||
|
if (tag === "table") return "table";
|
||||||
|
if (tag === "tr") return "row";
|
||||||
|
if (tag === "th") return "columnheader";
|
||||||
|
if (tag === "td") return "cell";
|
||||||
|
if (tag === "ul" || tag === "ol") return "list";
|
||||||
|
if (tag === "li") return "listitem";
|
||||||
|
if (/^h[1-6]$/.test(tag)) return "heading";
|
||||||
|
if (tag === "nav") return "navigation";
|
||||||
|
if (tag === "main") return "main";
|
||||||
|
if (tag === "form") return "form";
|
||||||
|
if (tag === "input") {
|
||||||
|
const type = (attrs.type ?? "text").toLowerCase();
|
||||||
|
if (type === "hidden") return undefined;
|
||||||
|
if (type === "checkbox") return "checkbox";
|
||||||
|
if (type === "radio") return "radio";
|
||||||
|
if (type === "range") return "slider";
|
||||||
|
if (type === "number") return "spinbutton";
|
||||||
|
if (type === "search") return "searchbox";
|
||||||
|
if (["button", "submit", "reset", "image"].includes(type)) return "button";
|
||||||
|
return "textbox";
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function domRole(tag: string, attrs: Record<string, string>): string | undefined {
|
||||||
|
return normalizeRole(attrs.role) ?? nativeRole(tag, attrs);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvedRole(node: SemanticGraphNode): {
|
||||||
|
value?: string;
|
||||||
|
source: ResolvedSemanticNode["roleSource"];
|
||||||
|
} {
|
||||||
|
const rawAxRole = clean(axValue(node.ax?.role));
|
||||||
|
const axRole = normalizeRole(rawAxRole);
|
||||||
|
const attrs = node.dom?.attrs ?? {};
|
||||||
|
const explicitDomRole = normalizeRole(attrs.role);
|
||||||
|
const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs);
|
||||||
|
const usableAxRole = node.ax?.ignored !== true;
|
||||||
|
if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) {
|
||||||
|
return { value: rawAxRole, source: "ax" };
|
||||||
|
}
|
||||||
|
if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" };
|
||||||
|
if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" };
|
||||||
|
if (rawAxRole) {
|
||||||
|
return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" };
|
||||||
|
}
|
||||||
|
return { source: "none" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function sensitive(node: SemanticGraphNode): boolean {
|
||||||
|
const attrs = node.dom?.attrs ?? {};
|
||||||
|
const type = (attrs.type ?? "").toLowerCase();
|
||||||
|
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||||
|
return (
|
||||||
|
type === "password" ||
|
||||||
|
autocomplete.startsWith("cc-") ||
|
||||||
|
SENSITIVE_INPUT_TYPES.has(autocomplete) ||
|
||||||
|
SENSITIVE_INPUT_TYPES.has(axProperty(node.ax, "inputType") ?? "")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function inputState(
|
||||||
|
node: SemanticGraphNode,
|
||||||
|
role: string | undefined,
|
||||||
|
value: string | undefined,
|
||||||
|
isSensitive: boolean,
|
||||||
|
): VomNode["inputState"] {
|
||||||
|
if (node.dom?.formState) return node.dom.formState;
|
||||||
|
const tag = node.dom?.tag.toLowerCase() ?? "";
|
||||||
|
if (!FORM_TAGS.has(tag) && !["textbox", "searchbox"].includes(role ?? "")) return undefined;
|
||||||
|
if (isSensitive) return value ? "filled" : "empty";
|
||||||
|
if (node.dom?.formValue !== undefined) {
|
||||||
|
if (node.dom.formValue === "") return "empty";
|
||||||
|
return node.dom.formValue === (node.dom.formDefaultValue ?? "") ? "default" : "filled";
|
||||||
|
}
|
||||||
|
return value ? "filled" : "empty";
|
||||||
|
}
|
||||||
|
|
||||||
|
function externalHrefHost(
|
||||||
|
href: string | undefined,
|
||||||
|
pageUrl: string | undefined,
|
||||||
|
): string | undefined {
|
||||||
|
if (!href || !pageUrl) return undefined;
|
||||||
|
try {
|
||||||
|
const page = new URL(pageUrl);
|
||||||
|
const target = new URL(href, page);
|
||||||
|
return target.origin === page.origin ? undefined : target.hostname;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ResolveIndexes {
|
||||||
|
domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
||||||
|
axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
||||||
|
domIdIndex: Map<string, SemanticNodeId>;
|
||||||
|
labelForIndex: Map<string, SemanticNodeId[]>;
|
||||||
|
domText: (nodeId: SemanticNodeId) => string | undefined;
|
||||||
|
axText: (nodeId: SemanticNodeId) => string | undefined;
|
||||||
|
hasNativeDescendant: (nodeId: SemanticNodeId) => boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildIndexes(graph: SemanticGraph): ResolveIndexes {
|
||||||
|
const domChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||||
|
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||||
|
const domIdIndex = new Map<string, SemanticNodeId>();
|
||||||
|
const labelForIndex = new Map<string, SemanticNodeId[]>();
|
||||||
|
for (const node of graph.nodes.values()) {
|
||||||
|
if (node.domParentId) {
|
||||||
|
const children = domChildren.get(node.domParentId) ?? [];
|
||||||
|
children.push(node.id);
|
||||||
|
domChildren.set(node.domParentId, children);
|
||||||
|
}
|
||||||
|
if (node.axParentId) {
|
||||||
|
const children = axChildren.get(node.axParentId) ?? [];
|
||||||
|
children.push(node.id);
|
||||||
|
axChildren.set(node.axParentId, children);
|
||||||
|
}
|
||||||
|
const id = clean(node.dom?.attrs.id);
|
||||||
|
if (id) domIdIndex.set(`${node.frameId}\u0000${id}`, node.id);
|
||||||
|
const targetId = clean(node.dom?.attrs.for);
|
||||||
|
if (node.dom?.tag.toLowerCase() === "label" && targetId) {
|
||||||
|
const key = `${node.frameId}\u0000${targetId}`;
|
||||||
|
const labels = labelForIndex.get(key) ?? [];
|
||||||
|
labels.push(node.id);
|
||||||
|
labelForIndex.set(key, labels);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [parentId, children] of axChildren) {
|
||||||
|
const order = graph.nodes.get(parentId)?.ax?.childIds ?? [];
|
||||||
|
const orderByAxId = new Map(order.map((nodeId, index) => [nodeId, index]));
|
||||||
|
children.sort((a, b) => {
|
||||||
|
const aIndex = orderByAxId.get(graph.nodes.get(a)?.axNodeId ?? "") ?? -1;
|
||||||
|
const bIndex = orderByAxId.get(graph.nodes.get(b)?.axNodeId ?? "") ?? -1;
|
||||||
|
return (
|
||||||
|
(aIndex < 0 ? Number.MAX_SAFE_INTEGER : aIndex) -
|
||||||
|
(bIndex < 0 ? Number.MAX_SAFE_INTEGER : bIndex)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const textMemo = new Map<SemanticNodeId, string | undefined>();
|
||||||
|
const resolvingText = new Set<SemanticNodeId>();
|
||||||
|
const domText = (nodeId: SemanticNodeId): string | undefined => {
|
||||||
|
if (textMemo.has(nodeId)) return textMemo.get(nodeId);
|
||||||
|
if (resolvingText.has(nodeId)) return undefined;
|
||||||
|
resolvingText.add(nodeId);
|
||||||
|
const node = graph.nodes.get(nodeId);
|
||||||
|
const parts: string[] = [];
|
||||||
|
const own = clean(node?.dom?.textContent);
|
||||||
|
if (own) parts.push(own);
|
||||||
|
for (const childId of domChildren.get(nodeId) ?? []) {
|
||||||
|
const child = graph.nodes.get(childId);
|
||||||
|
const childRole = child
|
||||||
|
? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
||||||
|
: undefined;
|
||||||
|
if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
|
||||||
|
const childText = domText(childId);
|
||||||
|
if (childText) parts.push(childText);
|
||||||
|
if (parts.join(" ").length >= 240) break;
|
||||||
|
}
|
||||||
|
resolvingText.delete(nodeId);
|
||||||
|
const result = clean(parts.join(" ").slice(0, 240));
|
||||||
|
textMemo.set(nodeId, result);
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const axTextMemo = new Map<SemanticNodeId, string | undefined>();
|
||||||
|
const resolvingAxText = new Set<SemanticNodeId>();
|
||||||
|
const axText = (nodeId: SemanticNodeId): string | undefined => {
|
||||||
|
if (axTextMemo.has(nodeId)) return axTextMemo.get(nodeId);
|
||||||
|
if (resolvingAxText.has(nodeId)) return undefined;
|
||||||
|
resolvingAxText.add(nodeId);
|
||||||
|
const complete = (value: string | undefined): string | undefined => {
|
||||||
|
resolvingAxText.delete(nodeId);
|
||||||
|
axTextMemo.set(nodeId, value);
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
const node = graph.nodes.get(nodeId);
|
||||||
|
const role = normalizeRole(axValue(node?.ax?.role)) ?? "";
|
||||||
|
if (TEXT_AX_ROLES.has(role)) {
|
||||||
|
const own = clean(axValue(node?.ax?.name) ?? axValue(node?.ax?.value));
|
||||||
|
if (own) return complete(own);
|
||||||
|
}
|
||||||
|
const parts: string[] = [];
|
||||||
|
for (const childId of axChildren.get(nodeId) ?? []) {
|
||||||
|
const child = graph.nodes.get(childId);
|
||||||
|
const childRole = normalizeRole(axValue(child?.ax?.role)) ?? "";
|
||||||
|
if (INTERACTIVE_ROLES.has(childRole)) continue;
|
||||||
|
const childText = axText(childId);
|
||||||
|
if (childText) parts.push(childText);
|
||||||
|
if (parts.join(" ").length >= 240) break;
|
||||||
|
}
|
||||||
|
return complete(clean(parts.join(" ").slice(0, 240)));
|
||||||
|
};
|
||||||
|
|
||||||
|
const nativeMemo = new Map<SemanticNodeId, boolean>();
|
||||||
|
const hasNativeDescendant = (nodeId: SemanticNodeId): boolean => {
|
||||||
|
const cached = nativeMemo.get(nodeId);
|
||||||
|
if (cached !== undefined) return cached;
|
||||||
|
nativeMemo.set(nodeId, false);
|
||||||
|
for (const childId of domChildren.get(nodeId) ?? []) {
|
||||||
|
const child = graph.nodes.get(childId);
|
||||||
|
const childRole =
|
||||||
|
(child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ??
|
||||||
|
(child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined);
|
||||||
|
if (
|
||||||
|
(child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
|
||||||
|
INTERACTIVE_ROLES.has(childRole ?? "") ||
|
||||||
|
hasNativeDescendant(childId)
|
||||||
|
) {
|
||||||
|
nativeMemo.set(nodeId, true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
domChildren,
|
||||||
|
axChildren,
|
||||||
|
domIdIndex,
|
||||||
|
labelForIndex,
|
||||||
|
domText,
|
||||||
|
axText,
|
||||||
|
hasNativeDescendant,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function ariaLabelledText(node: SemanticGraphNode, indexes: ResolveIndexes): string | undefined {
|
||||||
|
const ids = (node.dom?.attrs["aria-labelledby"] ?? "").split(/\s+/).filter(Boolean);
|
||||||
|
const parts = ids.flatMap((id) => {
|
||||||
|
const target = indexes.domIdIndex.get(`${node.frameId}\u0000${id}`);
|
||||||
|
return target ? (indexes.domText(target) ?? []) : [];
|
||||||
|
});
|
||||||
|
return clean(parts.join(" "));
|
||||||
|
}
|
||||||
|
|
||||||
|
function nativeLabelText(
|
||||||
|
node: SemanticGraphNode,
|
||||||
|
graph: SemanticGraph,
|
||||||
|
indexes: ResolveIndexes,
|
||||||
|
): string | undefined {
|
||||||
|
const id = clean(node.dom?.attrs.id);
|
||||||
|
if (id) {
|
||||||
|
const labels = indexes.labelForIndex.get(`${node.frameId}\u0000${id}`) ?? [];
|
||||||
|
const explicit = clean(
|
||||||
|
labels
|
||||||
|
.map((labelId) => indexes.domText(labelId))
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" "),
|
||||||
|
);
|
||||||
|
if (explicit) return explicit;
|
||||||
|
}
|
||||||
|
let parentId = node.domParentId;
|
||||||
|
let guard = 0;
|
||||||
|
while (parentId && guard <= graph.nodes.size) {
|
||||||
|
const parent = graph.nodes.get(parentId);
|
||||||
|
if (!parent) break;
|
||||||
|
if (parent.dom?.tag.toLowerCase() === "label") return indexes.domText(parentId);
|
||||||
|
parentId = parent.domParentId;
|
||||||
|
guard += 1;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function precedingAxText(
|
||||||
|
node: SemanticGraphNode,
|
||||||
|
graph: SemanticGraph,
|
||||||
|
indexes: ResolveIndexes,
|
||||||
|
): string | undefined {
|
||||||
|
if (!node.axParentId) return undefined;
|
||||||
|
const siblings = indexes.axChildren.get(node.axParentId) ?? [];
|
||||||
|
const index = siblings.indexOf(node.id);
|
||||||
|
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
||||||
|
const sibling = graph.nodes.get(siblings[index - offset]);
|
||||||
|
const role = normalizeRole(axValue(sibling?.ax?.role)) ?? "";
|
||||||
|
if (INTERACTIVE_ROLES.has(role)) break;
|
||||||
|
if (!TEXT_AX_ROLES.has(role) && role !== "generic") continue;
|
||||||
|
const text = clean(axValue(sibling?.ax?.name) ?? axValue(sibling?.ax?.value))?.replace(
|
||||||
|
/[::]\s*$/,
|
||||||
|
"",
|
||||||
|
);
|
||||||
|
if (text && text.length <= 80) return text;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function precedingDomText(
|
||||||
|
node: SemanticGraphNode,
|
||||||
|
graph: SemanticGraph,
|
||||||
|
indexes: ResolveIndexes,
|
||||||
|
): string | undefined {
|
||||||
|
if (!node.domParentId) return undefined;
|
||||||
|
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
|
||||||
|
const index = siblings.indexOf(node.id);
|
||||||
|
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
||||||
|
const sibling = graph.nodes.get(siblings[index - offset]);
|
||||||
|
const siblingRole = sibling
|
||||||
|
? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
|
||||||
|
: undefined;
|
||||||
|
if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
|
||||||
|
const text = indexes.domText(siblings[index - offset])?.replace(/[::]\s*$/, "");
|
||||||
|
if (text && text.length <= 80) return text;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvedName(
|
||||||
|
node: SemanticGraphNode,
|
||||||
|
role: string | undefined,
|
||||||
|
graph: SemanticGraph,
|
||||||
|
indexes: ResolveIndexes,
|
||||||
|
supplementalNames: ReadonlyMap<string, string>,
|
||||||
|
identifierFallback: boolean,
|
||||||
|
): string | undefined {
|
||||||
|
const attrs = node.dom?.attrs ?? {};
|
||||||
|
const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
||||||
|
const descendantText = indexes.domText(node.id);
|
||||||
|
const interactive =
|
||||||
|
INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
|
||||||
|
NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
||||||
|
const candidates = [
|
||||||
|
node.ax?.ignored === true ? undefined : axValue(node.ax?.name),
|
||||||
|
ariaLabelledText(node, indexes),
|
||||||
|
clean(attrs["aria-label"]),
|
||||||
|
nativeLabelText(node, graph, indexes),
|
||||||
|
clean(attrs.title),
|
||||||
|
clean(attrs.alt),
|
||||||
|
node.ax?.ignored === true ? undefined : indexes.axText(node.id),
|
||||||
|
descendantText,
|
||||||
|
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
||||||
|
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
||||||
|
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
||||||
|
formControl || !interactive || node.backendNodeId === undefined
|
||||||
|
? undefined
|
||||||
|
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
|
||||||
|
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
|
||||||
|
];
|
||||||
|
let name = candidates.find((candidate) => candidate !== undefined);
|
||||||
|
const hasPopup = axProperty(node.ax, "hasPopup");
|
||||||
|
if (name && hasPopup && hasPopup !== "false") {
|
||||||
|
name =
|
||||||
|
axProperty(node.ax, "expanded") === "true" ? `${name} [expanded]` : `${name} [has-submenu]`;
|
||||||
|
}
|
||||||
|
if (!name && role?.toLowerCase() === "iframe") return clean(attrs.id);
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
function insideNative(node: SemanticGraphNode, graph: SemanticGraph): boolean {
|
||||||
|
let parentId = node.domParentId;
|
||||||
|
let guard = 0;
|
||||||
|
while (parentId && guard <= graph.nodes.size) {
|
||||||
|
const parent = graph.nodes.get(parentId);
|
||||||
|
if (!parent) break;
|
||||||
|
if (parent.dom && NATIVE_CONTROL_TAGS.has(parent.dom.tag.toLowerCase())) return true;
|
||||||
|
parentId = parent.domParentId;
|
||||||
|
guard += 1;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nearbyText(
|
||||||
|
node: SemanticGraphNode,
|
||||||
|
graph: SemanticGraph,
|
||||||
|
indexes: ResolveIndexes,
|
||||||
|
): string | undefined {
|
||||||
|
if (!node.domParentId) return undefined;
|
||||||
|
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
|
||||||
|
const index = siblings.indexOf(node.id);
|
||||||
|
if (index < 0) return undefined;
|
||||||
|
const texts = siblings
|
||||||
|
.slice(Math.max(0, index - 3), index + 4)
|
||||||
|
.filter((id) => id !== node.id)
|
||||||
|
.flatMap((id) => indexes.domText(id) ?? [])
|
||||||
|
.slice(0, 3);
|
||||||
|
return clean(texts.join(" "));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveSemanticGraph(
|
||||||
|
graph: SemanticGraph,
|
||||||
|
options: {
|
||||||
|
supplementalNames?: ReadonlyMap<string, string>;
|
||||||
|
identifierFallback?: boolean;
|
||||||
|
} = {},
|
||||||
|
): ResolvedSemanticGraph {
|
||||||
|
const indexes = buildIndexes(graph);
|
||||||
|
const supplementalNames = options.supplementalNames ?? new Map<string, string>();
|
||||||
|
const identifierFallback = options.identifierFallback ?? true;
|
||||||
|
const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
|
||||||
|
for (const node of graph.nodes.values()) {
|
||||||
|
const roleResolution = resolvedRole(node);
|
||||||
|
const role = roleResolution.value;
|
||||||
|
const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback);
|
||||||
|
const isSensitive = sensitive(node);
|
||||||
|
const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
|
||||||
|
const value = isSensitive ? undefined : sourceValue;
|
||||||
|
const attrs = { ...(node.dom?.attrs ?? {}) };
|
||||||
|
if (isSensitive) delete attrs.value;
|
||||||
|
const frame = graph.frames.get(node.frameId);
|
||||||
|
const tag = node.dom?.tag.toLowerCase() ?? "";
|
||||||
|
const vom: ResolvedSemanticNode["vom"] = {
|
||||||
|
tag,
|
||||||
|
rect: node.dom?.rect ?? null,
|
||||||
|
paintOrder: node.dom?.paintOrder ?? 0,
|
||||||
|
position: node.dom?.position || "static",
|
||||||
|
pointerEvents: node.dom?.pointerEvents || "auto",
|
||||||
|
...(node.dom?.cursor ? { cursor: node.dom.cursor } : {}),
|
||||||
|
...(role ? { role } : {}),
|
||||||
|
...(name ? { name } : {}),
|
||||||
|
...(value !== undefined ? { value } : {}),
|
||||||
|
...(clean(node.dom?.formPlaceholder ?? attrs.placeholder)
|
||||||
|
? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) }
|
||||||
|
: {}),
|
||||||
|
attrs,
|
||||||
|
...(clean(node.dom?.textContent) ? { text: clean(node.dom?.textContent) } : {}),
|
||||||
|
...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}),
|
||||||
|
inputState: inputState(node, role, sourceValue, isSensitive),
|
||||||
|
sensitive: isSensitive,
|
||||||
|
modal:
|
||||||
|
["dialog", "alertdialog"].includes(role?.toLowerCase() ?? "") ||
|
||||||
|
tag === "dialog" ||
|
||||||
|
(attrs["aria-modal"] ?? "").toLowerCase() === "true",
|
||||||
|
disabled:
|
||||||
|
Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
|
||||||
|
(attrs["aria-disabled"] ?? "").toLowerCase() === "true",
|
||||||
|
inert: Object.prototype.hasOwnProperty.call(attrs, "inert"),
|
||||||
|
hasNativeDescendant: indexes.hasNativeDescendant(node.id),
|
||||||
|
insideNative: insideNative(node, graph),
|
||||||
|
...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
|
||||||
|
};
|
||||||
|
const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected");
|
||||||
|
const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded");
|
||||||
|
const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls");
|
||||||
|
if (selected === "true" && !vom.attrs?.["aria-selected"]) {
|
||||||
|
vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
|
||||||
|
}
|
||||||
|
if (expanded === "true" && !vom.attrs?.["aria-expanded"]) {
|
||||||
|
vom.attrs = { ...(vom.attrs ?? {}), "aria-expanded": "true" };
|
||||||
|
}
|
||||||
|
if (controls && !vom.attrs?.["aria-controls"]) {
|
||||||
|
vom.attrs = { ...(vom.attrs ?? {}), "aria-controls": controls };
|
||||||
|
}
|
||||||
|
nodes.set(node.id, {
|
||||||
|
...node,
|
||||||
|
vom,
|
||||||
|
referenceable: node.backendNodeId !== undefined,
|
||||||
|
roleSource: roleResolution.source,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return { ...graph, nodes };
|
||||||
|
}
|
||||||
@@ -0,0 +1,456 @@
|
|||||||
|
import {
|
||||||
|
applyVomInteractionRecovery,
|
||||||
|
isVomReferenceNode,
|
||||||
|
isVomStructuralRole,
|
||||||
|
type VomNode,
|
||||||
|
} from "@browser-skill/vom";
|
||||||
|
import type {
|
||||||
|
ResolvedSemanticGraph,
|
||||||
|
ResolvedSemanticNode,
|
||||||
|
SemanticNodeId,
|
||||||
|
StructureDecision,
|
||||||
|
StructuredSemanticGraph,
|
||||||
|
StructuredSemanticNode,
|
||||||
|
} from "./types";
|
||||||
|
|
||||||
|
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
|
||||||
|
const ROOT_ROLES = new Set(["rootwebarea", "webarea"]);
|
||||||
|
const CONTENT_BOUNDARY_ROLES = new Set([
|
||||||
|
"article",
|
||||||
|
"dialog",
|
||||||
|
"document",
|
||||||
|
"feed",
|
||||||
|
"form",
|
||||||
|
"grid",
|
||||||
|
"heading",
|
||||||
|
"iframe",
|
||||||
|
"list",
|
||||||
|
"main",
|
||||||
|
"navigation",
|
||||||
|
"paragraph",
|
||||||
|
"region",
|
||||||
|
"section",
|
||||||
|
"table",
|
||||||
|
"tree",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
|
||||||
|
const attrs = node.vom.attrs ?? {};
|
||||||
|
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
|
||||||
|
attrs[name]?.trim(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function explicitStructural(node: ResolvedSemanticNode): boolean {
|
||||||
|
return (
|
||||||
|
isVomStructuralRole(node.vom.role) &&
|
||||||
|
node.roleSource !== "none" &&
|
||||||
|
node.roleSource !== "ax-ignored"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean {
|
||||||
|
return node.ax !== undefined && node.ax.ignored !== true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRenderedDomFallback(node: ResolvedSemanticNode): boolean {
|
||||||
|
if (!node.dom) return false;
|
||||||
|
const attrs = node.dom.attrs;
|
||||||
|
if (
|
||||||
|
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
||||||
|
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
||||||
|
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null);
|
||||||
|
}
|
||||||
|
|
||||||
|
function initialParentId(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
node: ResolvedSemanticNode,
|
||||||
|
): SemanticNodeId | undefined {
|
||||||
|
let parentId = node.axParentId ?? node.domParentId;
|
||||||
|
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
|
||||||
|
const axParent = graph.nodes.get(node.axParentId);
|
||||||
|
const axRole = axParent?.vom.role?.toLowerCase() ?? "";
|
||||||
|
const meaningfulAxParent =
|
||||||
|
axParent !== undefined &&
|
||||||
|
!ROOT_ROLES.has(axRole) &&
|
||||||
|
(explicitStructural(axParent) ||
|
||||||
|
!TRANSPARENT_ROLES.has(axRole) ||
|
||||||
|
Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent)));
|
||||||
|
parentId = meaningfulAxParent ? node.axParentId : node.domParentId;
|
||||||
|
}
|
||||||
|
if (parentId) return parentId;
|
||||||
|
if (node.frameId === graph.rootFrameId) return undefined;
|
||||||
|
return graph.frames.get(node.frameId)?.ownerNodeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] {
|
||||||
|
return [...graph.frames.values()]
|
||||||
|
.sort((a, b) => a.order - b.order)
|
||||||
|
.flatMap((frame) =>
|
||||||
|
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
||||||
|
.map((id) => graph.nodes.get(id))
|
||||||
|
.filter((node): node is ResolvedSemanticNode => node !== undefined)
|
||||||
|
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function recoveredNodes(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
nodes: ResolvedSemanticNode[],
|
||||||
|
): Map<SemanticNodeId, VomNode> {
|
||||||
|
const numericId = new Map(nodes.map((node, index) => [node.id, index + 1]));
|
||||||
|
const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id]));
|
||||||
|
const provisional = nodes.map((node) => ({
|
||||||
|
...node.vom,
|
||||||
|
id: numericId.get(node.id) as number,
|
||||||
|
parentId: (() => {
|
||||||
|
const parent = initialParentId(graph, node);
|
||||||
|
return parent ? (numericId.get(parent) ?? null) : null;
|
||||||
|
})(),
|
||||||
|
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||||
|
frameId: node.frameId,
|
||||||
|
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
||||||
|
referenceable: node.referenceable,
|
||||||
|
}));
|
||||||
|
return new Map(
|
||||||
|
applyVomInteractionRecovery(provisional).flatMap((node) => {
|
||||||
|
const id = semanticId.get(node.id);
|
||||||
|
return id ? [[id, node] as const] : [];
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function redundantSourceNodes(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
nodes: ResolvedSemanticNode[],
|
||||||
|
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
|
||||||
|
): Set<SemanticNodeId> {
|
||||||
|
const byKey = new Map<string, ResolvedSemanticNode[]>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
const output = recovered.get(node.id) as VomNode;
|
||||||
|
const name = output.name?.trim().toLowerCase() ?? "";
|
||||||
|
if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue;
|
||||||
|
const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`;
|
||||||
|
const matches = byKey.get(key) ?? [];
|
||||||
|
matches.push(node);
|
||||||
|
byKey.set(key, matches);
|
||||||
|
}
|
||||||
|
const redundant = new Set<SemanticNodeId>();
|
||||||
|
for (const matches of byKey.values()) {
|
||||||
|
const axOnly = matches.filter((node) => node.ax && !node.dom);
|
||||||
|
const domOnly = matches.filter((node) => node.dom && !node.ax);
|
||||||
|
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
|
||||||
|
redundant.add(domOnly[0].id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return redundant;
|
||||||
|
}
|
||||||
|
|
||||||
|
function baseDecision(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
node: ResolvedSemanticNode,
|
||||||
|
output: VomNode,
|
||||||
|
frameOwnerIds: ReadonlySet<SemanticNodeId>,
|
||||||
|
redundant: ReadonlySet<SemanticNodeId>,
|
||||||
|
): StructureDecision {
|
||||||
|
if (node.excluded) return { kind: "excluded", reason: "excluded" };
|
||||||
|
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
|
||||||
|
if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) {
|
||||||
|
return { kind: "excluded", reason: "non-rendered-dom-fallback" };
|
||||||
|
}
|
||||||
|
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
|
||||||
|
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
|
||||||
|
return { kind: "transparent", reason: "child-document-root" };
|
||||||
|
}
|
||||||
|
if (node.ax?.ignored === true && !node.dom) {
|
||||||
|
return { kind: "transparent", reason: "ax-ignored" };
|
||||||
|
}
|
||||||
|
const role = output.role?.toLowerCase() ?? "";
|
||||||
|
if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" };
|
||||||
|
if (
|
||||||
|
output.name ||
|
||||||
|
output.text ||
|
||||||
|
output.value ||
|
||||||
|
output.placeholder ||
|
||||||
|
hasMeaningfulRelations(node) ||
|
||||||
|
(output.rect && ["fixed", "sticky"].includes(output.position))
|
||||||
|
) {
|
||||||
|
return { kind: "keep", reason: "semantic" };
|
||||||
|
}
|
||||||
|
return { kind: "transparent", reason: "wrapper" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildChildren(
|
||||||
|
nodes: ResolvedSemanticNode[],
|
||||||
|
relation: "axParentId" | "domParentId",
|
||||||
|
): Map<SemanticNodeId, SemanticNodeId[]> {
|
||||||
|
const children = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
const parentId = node[relation];
|
||||||
|
if (!parentId) continue;
|
||||||
|
const siblings = children.get(parentId) ?? [];
|
||||||
|
siblings.push(node.id);
|
||||||
|
children.set(parentId, siblings);
|
||||||
|
}
|
||||||
|
return children;
|
||||||
|
}
|
||||||
|
|
||||||
|
function explicitDomAncestors(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
nodes: ResolvedSemanticNode[],
|
||||||
|
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
||||||
|
): Set<SemanticNodeId> {
|
||||||
|
const inside = new Set<SemanticNodeId>();
|
||||||
|
const roots = nodes.filter((node) => !node.domParentId);
|
||||||
|
const queue = roots.map((node) => node.id);
|
||||||
|
for (let index = 0; index < queue.length; index += 1) {
|
||||||
|
const parentId = queue[index];
|
||||||
|
const parent = graph.nodes.get(parentId);
|
||||||
|
const parentIsBoundary =
|
||||||
|
parent !== undefined &&
|
||||||
|
explicitStructural(parent) &&
|
||||||
|
!ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? "");
|
||||||
|
for (const childId of domChildren.get(parentId) ?? []) {
|
||||||
|
if (inside.has(parentId) || parentIsBoundary) inside.add(childId);
|
||||||
|
queue.push(childId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inside;
|
||||||
|
}
|
||||||
|
|
||||||
|
function reliableGroupCandidates(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
nodes: ResolvedSemanticNode[],
|
||||||
|
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
|
||||||
|
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
|
||||||
|
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
||||||
|
): Set<SemanticNodeId> {
|
||||||
|
const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren);
|
||||||
|
const remainingChildren = new Map(
|
||||||
|
nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]),
|
||||||
|
);
|
||||||
|
const summaries = new Map<SemanticNodeId, { blocked: boolean; interactions: number }>();
|
||||||
|
const queue = nodes
|
||||||
|
.filter((node) => (remainingChildren.get(node.id) ?? 0) === 0)
|
||||||
|
.map((node) => node.id);
|
||||||
|
for (let index = 0; index < queue.length; index += 1) {
|
||||||
|
const nodeId = queue[index];
|
||||||
|
const node = graph.nodes.get(nodeId);
|
||||||
|
const output = recovered.get(nodeId);
|
||||||
|
if (!node || !output) continue;
|
||||||
|
const role = output.role?.toLowerCase() ?? "";
|
||||||
|
const excluded = decisions.get(nodeId)?.kind === "excluded";
|
||||||
|
const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role);
|
||||||
|
const summary = excluded
|
||||||
|
? { blocked: true, interactions: 0 }
|
||||||
|
: isVomReferenceNode(output)
|
||||||
|
? { blocked: false, interactions: 1 }
|
||||||
|
: blocked
|
||||||
|
? { blocked: true, interactions: 0 }
|
||||||
|
: (domChildren.get(nodeId) ?? []).reduce(
|
||||||
|
(combined, childId) => {
|
||||||
|
const child = summaries.get(childId);
|
||||||
|
return {
|
||||||
|
blocked: combined.blocked || !child || child.blocked,
|
||||||
|
interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
{ blocked: false, interactions: 0 },
|
||||||
|
);
|
||||||
|
summaries.set(nodeId, summary);
|
||||||
|
const parentId = node.domParentId;
|
||||||
|
if (!parentId) continue;
|
||||||
|
const remaining = (remainingChildren.get(parentId) ?? 1) - 1;
|
||||||
|
remainingChildren.set(parentId, remaining);
|
||||||
|
if (remaining === 0) queue.push(parentId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates = new Set<SemanticNodeId>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
const output = recovered.get(node.id) as VomNode;
|
||||||
|
const role = output.role?.toLowerCase() ?? "";
|
||||||
|
if (
|
||||||
|
!node.dom ||
|
||||||
|
node.dom.tag.toLowerCase() !== "div" ||
|
||||||
|
!TRANSPARENT_ROLES.has(role) ||
|
||||||
|
insideExplicitStructure.has(node.id) ||
|
||||||
|
decisions.get(node.id)?.kind === "excluded" ||
|
||||||
|
node.dom.attrs.hidden !== undefined ||
|
||||||
|
(node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" ||
|
||||||
|
node.dom.attrs.inert !== undefined
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id));
|
||||||
|
const blocked = branches.some((summary) => !summary || summary.blocked);
|
||||||
|
const contributingBranches = branches.filter(
|
||||||
|
(summary) => (summary?.interactions ?? 0) > 0,
|
||||||
|
).length;
|
||||||
|
const interactions = branches.reduce(
|
||||||
|
(count, summary) => Math.min(2, count + (summary?.interactions ?? 0)),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
if (!blocked && contributingBranches >= 2 && interactions >= 2) {
|
||||||
|
candidates.add(node.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return candidates;
|
||||||
|
}
|
||||||
|
|
||||||
|
function leafCandidates(
|
||||||
|
nodes: ResolvedSemanticNode[],
|
||||||
|
candidates: ReadonlySet<SemanticNodeId>,
|
||||||
|
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
||||||
|
): Set<SemanticNodeId> {
|
||||||
|
const parentById = new Map(
|
||||||
|
nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])),
|
||||||
|
);
|
||||||
|
const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]));
|
||||||
|
const containsCandidate = new Set<SemanticNodeId>();
|
||||||
|
const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id);
|
||||||
|
const selected = new Set<SemanticNodeId>();
|
||||||
|
for (let index = 0; index < queue.length; index += 1) {
|
||||||
|
const nodeId = queue[index];
|
||||||
|
if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId);
|
||||||
|
const parentId = parentById.get(nodeId);
|
||||||
|
if (!parentId) continue;
|
||||||
|
if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId);
|
||||||
|
const remaining = (childCount.get(parentId) ?? 1) - 1;
|
||||||
|
childCount.set(parentId, remaining);
|
||||||
|
if (remaining === 0) queue.push(parentId);
|
||||||
|
}
|
||||||
|
return selected;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nearestKept(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
startId: SemanticNodeId | undefined,
|
||||||
|
relation: "axParentId" | "domParentId",
|
||||||
|
kept: ReadonlySet<SemanticNodeId>,
|
||||||
|
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||||
|
): SemanticNodeId | undefined {
|
||||||
|
let currentId = startId;
|
||||||
|
const path: SemanticNodeId[] = [];
|
||||||
|
const seen = new Set<SemanticNodeId>();
|
||||||
|
let result: SemanticNodeId | undefined;
|
||||||
|
while (currentId && !seen.has(currentId)) {
|
||||||
|
if (kept.has(currentId)) {
|
||||||
|
result = currentId;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (memo.has(currentId)) {
|
||||||
|
result = memo.get(currentId);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
seen.add(currentId);
|
||||||
|
path.push(currentId);
|
||||||
|
currentId = graph.nodes.get(currentId)?.[relation];
|
||||||
|
}
|
||||||
|
for (const id of path) memo.set(id, result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalParentId(
|
||||||
|
graph: ResolvedSemanticGraph,
|
||||||
|
node: ResolvedSemanticNode,
|
||||||
|
kept: ReadonlySet<SemanticNodeId>,
|
||||||
|
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
|
||||||
|
axParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||||
|
domParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||||
|
): SemanticNodeId | undefined {
|
||||||
|
const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo);
|
||||||
|
const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo);
|
||||||
|
const axNode = axParent ? graph.nodes.get(axParent) : undefined;
|
||||||
|
const domNode = domParent ? graph.nodes.get(domParent) : undefined;
|
||||||
|
const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? "");
|
||||||
|
const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false;
|
||||||
|
const domIsStructural =
|
||||||
|
domNode !== undefined &&
|
||||||
|
(explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group");
|
||||||
|
if (axIsExplicit) return axParent;
|
||||||
|
if (domIsStructural) return domParent;
|
||||||
|
if (axParent && !axIsRoot) return axParent;
|
||||||
|
if (domParent) return domParent;
|
||||||
|
if (axParent) return axParent;
|
||||||
|
if (node.frameId === graph.rootFrameId) return undefined;
|
||||||
|
return graph.frames.get(node.frameId)?.ownerNodeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] {
|
||||||
|
const {
|
||||||
|
id: _id,
|
||||||
|
parentId: _parentId,
|
||||||
|
backendNodeId: _backendNodeId,
|
||||||
|
frameId: _frameId,
|
||||||
|
contextScopeId: _contextScopeId,
|
||||||
|
referenceable: _referenceable,
|
||||||
|
...vom
|
||||||
|
} = node;
|
||||||
|
if (!inferredGroup) return vom;
|
||||||
|
const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom;
|
||||||
|
return { ...groupVom, role: "group" };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
|
||||||
|
const nodes = orderedNodes(graph);
|
||||||
|
const recovered = recoveredNodes(graph, nodes);
|
||||||
|
const redundant = redundantSourceNodes(graph, nodes, recovered);
|
||||||
|
const frameOwnerIds = new Set(
|
||||||
|
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
|
||||||
|
);
|
||||||
|
const decisions = new Map<SemanticNodeId, StructureDecision>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
decisions.set(
|
||||||
|
node.id,
|
||||||
|
baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const domChildren = buildChildren(nodes, "domParentId");
|
||||||
|
const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren);
|
||||||
|
for (const nodeId of leafCandidates(nodes, candidates, domChildren)) {
|
||||||
|
decisions.set(nodeId, {
|
||||||
|
kind: "inferred-group",
|
||||||
|
reason: "independent-interaction-branches",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const kept = new Set(
|
||||||
|
nodes
|
||||||
|
.filter((node) => {
|
||||||
|
const kind = decisions.get(node.id)?.kind;
|
||||||
|
return kind === "keep" || kind === "inferred-group";
|
||||||
|
})
|
||||||
|
.map((node) => node.id),
|
||||||
|
);
|
||||||
|
const axParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
||||||
|
const domParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
||||||
|
const structuredNodes = new Map<SemanticNodeId, StructuredSemanticNode>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
const structure = decisions.get(node.id) as StructureDecision;
|
||||||
|
const inferredGroup = structure.kind === "inferred-group";
|
||||||
|
structuredNodes.set(node.id, {
|
||||||
|
...node,
|
||||||
|
vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup),
|
||||||
|
structure,
|
||||||
|
...(kept.has(node.id)
|
||||||
|
? {
|
||||||
|
semanticParentId: finalParentId(
|
||||||
|
graph,
|
||||||
|
node,
|
||||||
|
kept,
|
||||||
|
decisions,
|
||||||
|
axParentMemo,
|
||||||
|
domParentMemo,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return { ...graph, nodes: structuredNodes };
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import type { Viewport, VomNode } from "@browser-skill/vom";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import type { CapturedNode } from "../capture";
|
||||||
|
|
||||||
|
export interface SemanticAxNode {
|
||||||
|
nodeId: string;
|
||||||
|
frameId?: string;
|
||||||
|
parentId?: string;
|
||||||
|
backendDOMNodeId?: number;
|
||||||
|
ignored?: boolean;
|
||||||
|
role?: { type: string; value?: string };
|
||||||
|
name?: { type: string; value?: string };
|
||||||
|
description?: { value?: string };
|
||||||
|
value?: { value?: string | number | boolean };
|
||||||
|
properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
|
||||||
|
childIds?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SemanticNodeId = string;
|
||||||
|
|
||||||
|
export interface SemanticFrame {
|
||||||
|
frameId: string;
|
||||||
|
parentFrameId?: string;
|
||||||
|
ownerBackendNodeId?: number;
|
||||||
|
ownerNodeId?: SemanticNodeId;
|
||||||
|
contextScopeId: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
url?: string;
|
||||||
|
order: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SemanticGraphNode {
|
||||||
|
id: SemanticNodeId;
|
||||||
|
frameId: string;
|
||||||
|
backendNodeId?: number;
|
||||||
|
axNodeId?: string;
|
||||||
|
ax?: SemanticAxNode;
|
||||||
|
dom?: CapturedNode;
|
||||||
|
axParentId?: SemanticNodeId;
|
||||||
|
domParentId?: SemanticNodeId;
|
||||||
|
sourceOrder: number;
|
||||||
|
excluded: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SemanticGraph {
|
||||||
|
viewport: Viewport;
|
||||||
|
rootFrameId: string;
|
||||||
|
frames: Map<string, SemanticFrame>;
|
||||||
|
nodes: Map<SemanticNodeId, SemanticGraphNode>;
|
||||||
|
nodeIdsByFrame: Map<string, SemanticNodeId[]>;
|
||||||
|
nodeByFrameBackend: Map<string, SemanticNodeId>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResolvedSemanticNode extends SemanticGraphNode {
|
||||||
|
vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
|
||||||
|
referenceable: boolean;
|
||||||
|
roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
|
||||||
|
nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StructureDecision =
|
||||||
|
| { kind: "keep"; reason: "semantic" | "frame-owner" }
|
||||||
|
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
|
||||||
|
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
|
||||||
|
| {
|
||||||
|
kind: "excluded";
|
||||||
|
reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback";
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface StructuredSemanticNode extends ResolvedSemanticNode {
|
||||||
|
structure: StructureDecision;
|
||||||
|
semanticParentId?: SemanticNodeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StructuredSemanticGraph extends Omit<ResolvedSemanticGraph, "nodes"> {
|
||||||
|
nodes: Map<SemanticNodeId, StructuredSemanticNode>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||||
|
return `${frameId}\u0000${backendNodeId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function frameAxKey(frameId: string, axNodeId: string): string {
|
||||||
|
return `${frameId}\u0000${axNodeId}`;
|
||||||
|
}
|
||||||
@@ -65,6 +65,11 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("performHandshake", () => {
|
describe("performHandshake", () => {
|
||||||
|
it("advertises the protocol compatibility boundary", () => {
|
||||||
|
expect(PROTOCOL_VERSION).toBe("1.1");
|
||||||
|
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.0");
|
||||||
|
});
|
||||||
|
|
||||||
it("sends system.handshake with identity and both compat fields", async () => {
|
it("sends system.handshake with identity and both compat fields", async () => {
|
||||||
let sentFrame: ProtocolFrame | null = null;
|
let sentFrame: ProtocolFrame | null = null;
|
||||||
const transport = fakeTransport((req) => {
|
const transport = fakeTransport((req) => {
|
||||||
@@ -74,7 +79,7 @@ describe("performHandshake", () => {
|
|||||||
result: {
|
result: {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.0.0",
|
min_compatible_peer: "0.0.0",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.0",
|
||||||
},
|
},
|
||||||
@@ -129,8 +134,8 @@ describe("performHandshake", () => {
|
|||||||
const response = {
|
const response = {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
} satisfies HandshakeResult;
|
} satisfies HandshakeResult;
|
||||||
const transport = fakeTransport((req) => ({
|
const transport = fakeTransport((req) => ({
|
||||||
id: (req as { id: string }).id,
|
id: (req as { id: string }).id,
|
||||||
@@ -145,7 +150,7 @@ describe("performHandshake", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(outcome.result.min_compatible_peer).toBeUndefined();
|
expect(outcome.result.min_compatible_peer).toBeUndefined();
|
||||||
expect(outcome.result.min_compatible_protocol).toBe("1.0");
|
expect(outcome.result.min_compatible_protocol).toBe("1.1");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects when the daemon responds with an error", async () => {
|
it("rejects when the daemon responds with an error", async () => {
|
||||||
@@ -178,9 +183,9 @@ describe("performHandshake", () => {
|
|||||||
result: {
|
result: {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.0.0",
|
min_compatible_peer: "0.0.0",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
emit({
|
emit({
|
||||||
@@ -188,14 +193,14 @@ describe("performHandshake", () => {
|
|||||||
result: {
|
result: {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.0.0",
|
min_compatible_peer: "0.0.0",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(pending).resolves.toMatchObject({
|
await expect(pending).resolves.toMatchObject({
|
||||||
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
|
result: { server: "browser-skill-daemon", protocol_version: "1.1" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import type {
|
|||||||
ResponseFrame,
|
ResponseFrame,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
export const PROTOCOL_VERSION = "1.0";
|
export const PROTOCOL_VERSION = "1.1";
|
||||||
/**
|
/**
|
||||||
* Extension semver, injected at build time from `package.json` via
|
* Extension semver, injected at build time from `package.json` via
|
||||||
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
|
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ export type RpcErrorReason =
|
|||||||
| "agent_window_scope"
|
| "agent_window_scope"
|
||||||
| "element_not_visible"
|
| "element_not_visible"
|
||||||
| "ref_not_found"
|
| "ref_not_found"
|
||||||
|
| "invalid_selector"
|
||||||
| "selector_not_found"
|
| "selector_not_found"
|
||||||
| "target_not_fillable"
|
| "target_not_fillable"
|
||||||
| "target_not_select"
|
| "target_not_select"
|
||||||
@@ -657,10 +658,60 @@ export interface EmulateResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
// Semantic record payloads mirror the versioned Rust protocol models.
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
export const TRACE_VERSION_V3 = 3;
|
||||||
|
/** Logical v2 identifier. Not a wire field — v2 envelopes omit `version`. */
|
||||||
|
export const TRACE_VERSION_V2 = 2;
|
||||||
|
export const VOM_FORMAT_VERSION = 1;
|
||||||
|
|
||||||
|
export interface TargetDescriptorV3 {
|
||||||
|
ref?: string;
|
||||||
|
role?: string;
|
||||||
|
name?: string;
|
||||||
|
ctx?: string;
|
||||||
|
unmatched?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecorderInfo {
|
||||||
|
bsk: string;
|
||||||
|
vom: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StopReason = "user_finish" | "cli_stop";
|
||||||
|
|
||||||
|
export interface TraceStateV3 {
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
body: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepResultV3 {
|
||||||
|
state: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepCommonV3 {
|
||||||
|
id: number;
|
||||||
|
state: string;
|
||||||
|
result: StepResultV3;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NavigationCause =
|
||||||
|
| "user_typed"
|
||||||
|
| "link"
|
||||||
|
| "form_submit"
|
||||||
|
| "reload"
|
||||||
|
| "history"
|
||||||
|
| "script"
|
||||||
|
| "browser";
|
||||||
|
|
||||||
|
export type FillCommit = "enter" | "suggestion" | "blur";
|
||||||
|
|
||||||
|
/** Legacy v2 target shape retained for existing record producers. */
|
||||||
|
export interface TargetDescriptorV2 {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -673,103 +724,109 @@ export interface TraceEntry {
|
|||||||
start_url: string;
|
start_url: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PageRef {
|
export interface PageRefV2 {
|
||||||
id: string;
|
id: string;
|
||||||
url: string;
|
url: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SelectedOption {
|
export interface SelectedOptionV2 {
|
||||||
value: string;
|
value: string;
|
||||||
label?: string;
|
label?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepEffect {
|
export interface StepEffectV2 {
|
||||||
navigated_to: string;
|
navigated_to: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepCommon {
|
export interface StepCommonV2 {
|
||||||
id: number;
|
id: number;
|
||||||
page: string;
|
page: string;
|
||||||
effect?: StepEffect;
|
effect?: StepEffectV2;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Capture/buffer draft before v2 reduction. */
|
|
||||||
export type DraftTraceStep =
|
|
||||||
| {
|
|
||||||
op: "click";
|
|
||||||
target: TargetDescriptor;
|
|
||||||
navigated_to?: string;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "hover";
|
|
||||||
target: TargetDescriptor;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "fill";
|
|
||||||
target: TargetDescriptor;
|
|
||||||
value: string;
|
|
||||||
redacted?: boolean;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "press";
|
|
||||||
key: string;
|
|
||||||
target?: TargetDescriptor;
|
|
||||||
modifiers?: KeyModifier[];
|
|
||||||
navigated_to?: string;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "select";
|
|
||||||
target: TargetDescriptor;
|
|
||||||
values: string[];
|
|
||||||
labels?: string[];
|
|
||||||
navigated_to?: string;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "navigate";
|
|
||||||
url: string;
|
|
||||||
page_url?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Exported record-only step (trace v2). */
|
/** Exported record-only step (trace v2). */
|
||||||
export type Step =
|
export type StepV2 =
|
||||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
| ({ op: "navigate" } & StepCommonV2 & { to: string })
|
||||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "click" } & StepCommonV2 & { target: TargetDescriptorV2 })
|
||||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "hover" } & StepCommonV2 & { target: TargetDescriptorV2 })
|
||||||
| ({ op: "fill" } & StepCommon & {
|
| ({ op: "fill" } & StepCommonV2 & {
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptorV2;
|
||||||
value: string;
|
value: string;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
})
|
})
|
||||||
| ({ op: "select" } & StepCommon & {
|
| ({ op: "select" } & StepCommonV2 & {
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptorV2;
|
||||||
selection: SelectedOption[];
|
selection: SelectedOptionV2[];
|
||||||
})
|
})
|
||||||
| ({ op: "press" } & StepCommon & {
|
| ({ op: "press" } & StepCommonV2 & {
|
||||||
key: string;
|
key: string;
|
||||||
modifiers?: KeyModifier[];
|
modifiers?: KeyModifier[];
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptorV2;
|
||||||
});
|
});
|
||||||
|
|
||||||
export interface Trace {
|
export interface TraceV2 {
|
||||||
recorded_at: string;
|
recorded_at: string;
|
||||||
started_at?: string;
|
started_at?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
entry: TraceEntry;
|
entry: TraceEntry;
|
||||||
pages: PageRef[];
|
pages: PageRefV2[];
|
||||||
steps: Step[];
|
steps: StepV2[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SelectedOptionV3 {
|
||||||
|
value: string;
|
||||||
|
label?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StepV3 =
|
||||||
|
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
|
||||||
|
| ({ op: "switch_tab" } & StepCommonV3)
|
||||||
|
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
|
||||||
|
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
|
||||||
|
| ({ op: "fill" } & StepCommonV3 & {
|
||||||
|
target: TargetDescriptorV3;
|
||||||
|
value: string;
|
||||||
|
commit: FillCommit;
|
||||||
|
redacted?: boolean;
|
||||||
|
})
|
||||||
|
| ({ op: "select" } & StepCommonV3 & {
|
||||||
|
target: TargetDescriptorV3;
|
||||||
|
selection?: SelectedOptionV3[];
|
||||||
|
})
|
||||||
|
| ({ op: "press" } & StepCommonV3 & {
|
||||||
|
key: string;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
target?: TargetDescriptorV3;
|
||||||
|
})
|
||||||
|
| ({ op: "scroll" } & StepCommonV3);
|
||||||
|
|
||||||
|
export interface TraceV3 {
|
||||||
|
version: typeof TRACE_VERSION_V3;
|
||||||
|
recorded_at: string;
|
||||||
|
started_at?: string;
|
||||||
|
purpose?: string;
|
||||||
|
stopped_by: StopReason;
|
||||||
|
entry: TraceEntry;
|
||||||
|
recorder: RecorderInfo;
|
||||||
|
states: TraceStateV3[];
|
||||||
|
steps: StepV3[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordedTrace = TraceV2 | TraceV3;
|
||||||
|
export type RecordedStep = StepV2 | StepV3;
|
||||||
|
|
||||||
export interface RecordStartParams {
|
export interface RecordStartParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
tab_id?: number;
|
tab_id?: number;
|
||||||
url?: string;
|
url?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
|
max_page_tokens?: number;
|
||||||
|
redact_values?: boolean;
|
||||||
|
/** Omitted means v2; `3` requests a state-linked v3 trace. */
|
||||||
|
trace_version?: number;
|
||||||
|
/** Client can decode the v3 `switch_tab` step variant. */
|
||||||
|
supports_tab_switch_steps?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStartResult {
|
export interface RecordStartResult {
|
||||||
@@ -782,7 +839,7 @@ export interface RecordStopParams {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStopResult {
|
export interface RecordStopResult {
|
||||||
trace: Trace;
|
trace: RecordedTrace;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordAwaitParams {
|
export interface RecordAwaitParams {
|
||||||
@@ -791,5 +848,5 @@ export interface RecordAwaitParams {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordAwaitResult {
|
export interface RecordAwaitResult {
|
||||||
trace: Trace;
|
trace: RecordedTrace;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||||
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
|
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
|
||||||
import type { ConnectionState, ProtocolFrame } from "./types";
|
import type { ConnectionState, ProtocolFrame } from "./types";
|
||||||
|
|
||||||
@@ -18,6 +19,7 @@ export interface WSTransportOptions {
|
|||||||
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
|
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
|
||||||
*/
|
*/
|
||||||
webSocketFactory?: WebSocketFactory;
|
webSocketFactory?: WebSocketFactory;
|
||||||
|
diagnostics?: DiagnosticSink;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface CloseLikeEvent {
|
interface CloseLikeEvent {
|
||||||
@@ -44,6 +46,7 @@ export class WSTransport implements Transport {
|
|||||||
private readonly factory: WebSocketFactory;
|
private readonly factory: WebSocketFactory;
|
||||||
private readonly initialDelayMs: number;
|
private readonly initialDelayMs: number;
|
||||||
private readonly maxDelayMs: number;
|
private readonly maxDelayMs: number;
|
||||||
|
private readonly diagnostics?: DiagnosticSink;
|
||||||
|
|
||||||
private socket: WebSocket | null = null;
|
private socket: WebSocket | null = null;
|
||||||
private currentState: ConnectionState = "disconnected";
|
private currentState: ConnectionState = "disconnected";
|
||||||
@@ -63,6 +66,7 @@ export class WSTransport implements Transport {
|
|||||||
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
|
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
|
||||||
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
|
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
|
||||||
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
|
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
|
||||||
|
this.diagnostics = options.diagnostics;
|
||||||
}
|
}
|
||||||
|
|
||||||
get state(): ConnectionState {
|
get state(): ConnectionState {
|
||||||
@@ -70,6 +74,13 @@ export class WSTransport implements Transport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
connect(): Promise<void> {
|
connect(): Promise<void> {
|
||||||
|
this.diagnostics?.("transport.connect.requested", {
|
||||||
|
state: this.currentState,
|
||||||
|
socket_generation: this.socketGeneration,
|
||||||
|
has_socket: this.socket !== null,
|
||||||
|
has_connecting_promise: this.connectingPromise !== null,
|
||||||
|
has_reconnect_timer: this.reconnectTimer !== null,
|
||||||
|
});
|
||||||
if (this.currentState === "connected") return Promise.resolve();
|
if (this.currentState === "connected") return Promise.resolve();
|
||||||
|
|
||||||
this.explicitlyClosed = false;
|
this.explicitlyClosed = false;
|
||||||
@@ -94,6 +105,11 @@ export class WSTransport implements Transport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async disconnect(): Promise<void> {
|
async disconnect(): Promise<void> {
|
||||||
|
this.diagnostics?.("transport.disconnect.requested", {
|
||||||
|
state: this.currentState,
|
||||||
|
socket_generation: this.socketGeneration,
|
||||||
|
has_socket: this.socket !== null,
|
||||||
|
});
|
||||||
this.explicitlyClosed = true;
|
this.explicitlyClosed = true;
|
||||||
if (this.reconnectTimer) {
|
if (this.reconnectTimer) {
|
||||||
clearTimeout(this.reconnectTimer);
|
clearTimeout(this.reconnectTimer);
|
||||||
@@ -124,6 +140,12 @@ export class WSTransport implements Transport {
|
|||||||
|
|
||||||
send(msg: ProtocolFrame): void {
|
send(msg: ProtocolFrame): void {
|
||||||
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
|
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
|
||||||
|
this.diagnostics?.("transport.send.rejected", {
|
||||||
|
state: this.currentState,
|
||||||
|
socket_generation: this.socketGeneration,
|
||||||
|
socket_ready_state: this.socket?.readyState ?? null,
|
||||||
|
frame_kind: "method" in msg ? msg.method : "event" in msg ? msg.event : "response",
|
||||||
|
});
|
||||||
throw new Error("[WSTransport] cannot send while not connected");
|
throw new Error("[WSTransport] cannot send while not connected");
|
||||||
}
|
}
|
||||||
this.socket.send(JSON.stringify(msg));
|
this.socket.send(JSON.stringify(msg));
|
||||||
@@ -152,10 +174,16 @@ export class WSTransport implements Transport {
|
|||||||
const generation = ++this.socketGeneration;
|
const generation = ++this.socketGeneration;
|
||||||
const socket = this.factory(this.url);
|
const socket = this.factory(this.url);
|
||||||
this.socket = socket;
|
this.socket = socket;
|
||||||
|
this.diagnostics?.("transport.socket.created", {
|
||||||
|
socket_generation: generation,
|
||||||
|
reconnect_attempt: this.reconnectAttempt,
|
||||||
|
url: this.url,
|
||||||
|
});
|
||||||
|
|
||||||
socket.addEventListener("open", () => {
|
socket.addEventListener("open", () => {
|
||||||
if (!this.isCurrentSocket(socket, generation)) return;
|
if (!this.isCurrentSocket(socket, generation)) return;
|
||||||
this.reconnectAttempt = 0;
|
this.reconnectAttempt = 0;
|
||||||
|
this.diagnostics?.("transport.socket.open", { socket_generation: generation });
|
||||||
this.setState("connected");
|
this.setState("connected");
|
||||||
const resolve = this.resolveConnect;
|
const resolve = this.resolveConnect;
|
||||||
this.resolveConnect = null;
|
this.resolveConnect = null;
|
||||||
@@ -174,6 +202,7 @@ export class WSTransport implements Transport {
|
|||||||
});
|
});
|
||||||
|
|
||||||
socket.addEventListener("error", () => {
|
socket.addEventListener("error", () => {
|
||||||
|
this.diagnostics?.("transport.socket.error", { socket_generation: generation });
|
||||||
// The 'close' handler always fires after 'error' in browsers, so we
|
// The 'close' handler always fires after 'error' in browsers, so we
|
||||||
// just record the error and let close drive reconnect logic.
|
// just record the error and let close drive reconnect logic.
|
||||||
});
|
});
|
||||||
@@ -197,7 +226,16 @@ export class WSTransport implements Transport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
||||||
if (!this.isCurrentSocket(socket, generation)) return;
|
const isCurrent = this.isCurrentSocket(socket, generation);
|
||||||
|
this.diagnostics?.("transport.socket.close", {
|
||||||
|
socket_generation: generation,
|
||||||
|
current_socket_generation: this.socketGeneration,
|
||||||
|
is_current: isCurrent,
|
||||||
|
explicitly_closed: this.explicitlyClosed,
|
||||||
|
close_code: _ev.code ?? null,
|
||||||
|
close_reason: _ev.reason ?? "",
|
||||||
|
});
|
||||||
|
if (!isCurrent) return;
|
||||||
this.socket = null;
|
this.socket = null;
|
||||||
if (this.explicitlyClosed) {
|
if (this.explicitlyClosed) {
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
@@ -211,9 +249,18 @@ export class WSTransport implements Transport {
|
|||||||
if (this.reconnectTimer) return;
|
if (this.reconnectTimer) return;
|
||||||
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
|
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
|
||||||
this.reconnectAttempt += 1;
|
this.reconnectAttempt += 1;
|
||||||
|
this.diagnostics?.("transport.reconnect.scheduled", {
|
||||||
|
delay_ms: delay,
|
||||||
|
reconnect_attempt: this.reconnectAttempt,
|
||||||
|
socket_generation: this.socketGeneration,
|
||||||
|
});
|
||||||
this.reconnectTimer = setTimeout(() => {
|
this.reconnectTimer = setTimeout(() => {
|
||||||
this.reconnectTimer = null;
|
this.reconnectTimer = null;
|
||||||
if (this.explicitlyClosed) return;
|
if (this.explicitlyClosed) return;
|
||||||
|
this.diagnostics?.("transport.reconnect.timer_fired", {
|
||||||
|
reconnect_attempt: this.reconnectAttempt,
|
||||||
|
socket_generation: this.socketGeneration,
|
||||||
|
});
|
||||||
void this.connect().catch((err) => {
|
void this.connect().catch((err) => {
|
||||||
console.debug("[WSTransport] reconnect attempt failed", err);
|
console.debug("[WSTransport] reconnect attempt failed", err);
|
||||||
});
|
});
|
||||||
@@ -226,7 +273,13 @@ export class WSTransport implements Transport {
|
|||||||
|
|
||||||
private setState(next: ConnectionState): void {
|
private setState(next: ConnectionState): void {
|
||||||
if (this.currentState === next) return;
|
if (this.currentState === next) return;
|
||||||
|
const previous = this.currentState;
|
||||||
this.currentState = next;
|
this.currentState = next;
|
||||||
|
this.diagnostics?.("transport.state.changed", {
|
||||||
|
previous,
|
||||||
|
next,
|
||||||
|
socket_generation: this.socketGeneration,
|
||||||
|
});
|
||||||
for (const h of this.stateHandlers) {
|
for (const h of this.stateHandlers) {
|
||||||
try {
|
try {
|
||||||
h(next);
|
h(next);
|
||||||
|
|||||||
@@ -22,7 +22,8 @@ export default defineConfig({
|
|||||||
outDir: "dist",
|
outDir: "dist",
|
||||||
modules: ["@wxt-dev/module-react"],
|
modules: ["@wxt-dev/module-react"],
|
||||||
manifest: {
|
manifest: {
|
||||||
name: "BrowserSkill",
|
name: "BrowserSkill (Lumi Diagnostics)",
|
||||||
|
version_name: `${EXTENSION_VERSION}-lumi-session-loss-v2`,
|
||||||
description:
|
description:
|
||||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||||
// Flat debugger sessions are required to address out-of-process iframes.
|
// Flat debugger sessions are required to address out-of-process iframes.
|
||||||
@@ -45,7 +46,7 @@ export default defineConfig({
|
|||||||
128: "icon/logo.png",
|
128: "icon/logo.png",
|
||||||
},
|
},
|
||||||
action: {
|
action: {
|
||||||
default_title: "BrowserSkill",
|
default_title: "BrowserSkill (Lumi Diagnostics)",
|
||||||
default_icon: {
|
default_icon: {
|
||||||
16: "icon/logo.png",
|
16: "icon/logo.png",
|
||||||
32: "icon/logo.png",
|
32: "icon/logo.png",
|
||||||
|
|||||||
@@ -44,6 +44,7 @@
|
|||||||
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
||||||
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
||||||
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
||||||
|
"!evals/browser/results/**",
|
||||||
"!**/target/**",
|
"!**/target/**",
|
||||||
"!**/.wxt/**",
|
"!**/.wxt/**",
|
||||||
"!**/.output/**",
|
"!**/.output/**",
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ Drive the user's **real Chromium browser** (with their logins and cookies) throu
|
|||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||||
|
|
||||||
## Mandatory workflow
|
## Mandatory workflow
|
||||||
@@ -210,7 +210,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
|||||||
|
|
||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
|
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||||
|
|
||||||
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
|
|||||||
|
|
||||||
### Recording — `bsk record`
|
### Recording — `bsk record`
|
||||||
|
|
||||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
|
bsk record start --browser <instance-id-or-label> \
|
||||||
|
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||||
|
[--max-page-tokens 3000] [--redact-values] \
|
||||||
|
[--output trace]
|
||||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||||
|
# trace/trace.json — action chain (+ state index when v3)
|
||||||
|
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||||
|
|
||||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||||
```
|
```
|
||||||
|
|
||||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||||
|
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||||
|
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||||
|
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||||
|
|
||||||
## Error handling
|
## Error handling
|
||||||
@@ -286,12 +294,12 @@ bsk record stop [--output trace.json] # terminal fallback if the browser panel
|
|||||||
|
|
||||||
| Code | Meaning | What to do |
|
| Code | Meaning | What to do |
|
||||||
|------|---------|------------|
|
|------|---------|------------|
|
||||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
|
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
||||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||||
|
|
||||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||||
|
|
||||||
|
|||||||
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `bsk doctor` check: every connected browser should use the same
|
/// `bsk doctor` check: connected browsers should speak a protocol the
|
||||||
/// protocol version as the daemon. Minor protocol drift is accepted by
|
/// daemon accepts. A different protocol string is still a live
|
||||||
/// the daemon but flagged here so the user can see who needs updating
|
/// connection — report Ok so agents keep working. The detail is an
|
||||||
/// (M10.4).
|
/// upgrade reminder, not a blocker.
|
||||||
///
|
///
|
||||||
/// Review M2 (round-1 minor): when no browsers are connected, the
|
/// Review M2 (round-1 minor): when no browsers are connected, the
|
||||||
/// check has nothing to compare against, so it now reports
|
/// check has nothing to compare against, so it now reports
|
||||||
@@ -396,13 +396,12 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(", ");
|
.join(", ");
|
||||||
CheckResult::fail(
|
CheckResult::ok(
|
||||||
name,
|
name,
|
||||||
format!(
|
format!(
|
||||||
"{} browser(s) have protocol minor drift from the daemon: {stale}",
|
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
|
||||||
status.version_skew_browsers.len()
|
status.version_skew_browsers.len()
|
||||||
),
|
),
|
||||||
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -581,7 +580,7 @@ mod m2_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn browsers_check_reports_fail_when_skew_present() {
|
fn browsers_check_reports_ok_when_skew_present() {
|
||||||
let status = fake_status(
|
let status = fake_status(
|
||||||
vec![BrowserStatusEntry {
|
vec![BrowserStatusEntry {
|
||||||
instance_id: "alpha".into(),
|
instance_id: "alpha".into(),
|
||||||
@@ -605,9 +604,17 @@ mod m2_tests {
|
|||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
assert_eq!(check.status, CheckStatus::Fail);
|
assert_eq!(check.status, CheckStatus::Ok);
|
||||||
assert!(check.detail.contains("alpha"));
|
assert!(check.detail.contains("alpha"));
|
||||||
assert!(check.hint.is_some());
|
assert!(
|
||||||
|
check
|
||||||
|
.detail
|
||||||
|
.contains("still usable — continue, and upgrade soon")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
check.hint.is_none(),
|
||||||
|
"a protocol-version note must not hint to stop"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -635,7 +642,7 @@ mod m2_tests {
|
|||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
assert_eq!(check.status, CheckStatus::Fail);
|
assert_eq!(check.status, CheckStatus::Ok);
|
||||||
assert!(
|
assert!(
|
||||||
check
|
check
|
||||||
.detail
|
.detail
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user