figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(recording): using unified geometry

Browse Source
This commit is contained in:
Ljy-0827
2026-08-23 21:26:40 +08:00
committed by GitHub
parent 21476c3884
commit 87cdcfe5ca
35 changed files with 1952 additions and 2087 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/content/record-capture.ts
+12 -9
View File
@@ -1,9 +1,9 @@
import {
type CaptureTargetDescriptor,
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
@@ -27,7 +27,7 @@ import {
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import { shouldRecordPress } from "@/lib/recording/draft-policy";
import {
closestHoverSurfaceCandidate,
collectHoverSurfaceStates,
@@ -65,14 +65,14 @@ export interface RecordCaptureController {
interface FillSession {
element: FillableElement;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
baselineValue: string;
lastValue: string;
}
interface HoverCandidate {
element: Element;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
recordedAt: number;
score: number;
eligible: boolean;
@@ -232,7 +232,10 @@ function collectHoverTriggerLabelText(root: Element): string {
return normalizeLabelText(text);
}
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
function compactHoverTargetName(
el: Element,
desc: CaptureTargetDescriptor,
): CaptureTargetDescriptor {
if (!desc.name) return desc;
const fullText = normalizeLabelText(el.textContent ?? "");
const compactName = collectHoverTriggerLabelText(el);
@@ -250,7 +253,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
return desc;
}
function isWeakHoverTarget(target: TargetDescriptor): boolean {
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
@@ -261,9 +264,9 @@ function looksLikeAvatarElement(el: Element): boolean {
function normalizeHoverTarget(
el: Element,
desc: TargetDescriptor,
desc: CaptureTargetDescriptor,
decision: HoverTriggerDecision,
): TargetDescriptor {
): CaptureTargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
@@ -277,7 +280,7 @@ function normalizeHoverTarget(
return desc;
}
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return {
apps/extension/src/lib/__tests__/document-settle.test.ts
+101
View File
@@ -0,0 +1,101 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForDocumentSettled } from "../recording/document-settle";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForDocumentSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const controller = new AbortController();
const settled = waitForDocumentSettled(
cdp,
{ target: { tabId: 7 } },
{
signal: controller.signal,
},
);
controller.abort();
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
it("probes a same-process iframe through its isolated execution context", async () => {
vi.useFakeTimers();
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
const settled = waitForDocumentSettled(cdp, {
target: { tabId: 7 },
frameId: "child-frame",
});
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledWith(
7,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child-frame" }),
);
expect(send).toHaveBeenCalledWith(
7,
"Runtime.evaluate",
expect.objectContaining({ contextId: 91 }),
);
});
it("routes an OOPIF probe through its CDP target session", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const target = { tabId: 7, sessionId: "oopif-session" };
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 27 }),
);
});
});
apps/extension/src/lib/__tests__/match-target.test.ts
-116
View File
@@ -1,116 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import { matchTarget } from "../match-target";
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
tag: "button",
attrs: {},
rect: { x: 10, y: 20, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
...overrides,
};
}
describe("matchTarget", () => {
const refs: RenderedRef[] = [
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
];
it("matches a unique node by viewport coordinates", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42)],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a non-fixed node after the top frame scrolls", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 40,
scrollY: 300,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 50, y: 320, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
rect: { x: 10, y: 20, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a static node by document coordinates when scroll changed after observation", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 50, w: 100, h: 30 },
scrollX: 0,
scrollY: 250,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 10, y: 300, w: 100, h: 30 },
localRect: { x: 10, y: 200, w: 100, h: 30 },
rect: { x: 10, y: 200, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("returns unmatched when multiple candidates match", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42), node(43)],
refs,
});
expect(target.unmatched).toBe(true);
});
it("uses viewport coordinates for fixed elements", () => {
const target = matchTarget({
geometry: {
rect: { x: 5, y: 5, w: 80, h: 24 },
scrollX: 100,
scrollY: 200,
position: "fixed",
tag: "button",
},
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
refs,
});
expect(target.ref).toBe("e1");
});
});
apps/extension/src/lib/__tests__/page-settled.test.ts
-43
View File
@@ -1,43 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForPageSettled } from "../page-settled";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForPageSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForPageSettled(cdp, 7);
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
let cancelled = false;
const { cdp, send } = quietCdp();
const settled = waitForPageSettled(cdp, 7, { cancelled: () => cancelled });
cancelled = true;
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
});
apps/extension/src/lib/__tests__/record-observation.test.ts
-206
View File
@@ -1,206 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep } from "@/transport/types";
import { resetStateIdCounterForTests } from "../record-constants";
import {
applyTargetMatching,
buildTraceV3,
createObservationState,
flushPendingRedirectLanding,
rememberStepOnPage,
scheduleRedirectLandingFlush,
} from "../record-observation";
import { registerObservation } from "../trace-reducer";
const URL = "https://example.com/login";
function capturedInput(): CapturedNode {
return {
backendNodeId: 42,
parentBackendNodeId: null,
tag: "input",
attrs: {},
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const obs = createObservationState({ redactValues });
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
obs.lastSettled = {
stateId,
captured: [capturedInput()],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
url: URL,
vomText: rawVomText,
};
const draft: DraftTraceStep = {
op: "fill",
target: { unmatched: true },
value,
geometry: {
rect: { x: 20, y: 40, w: 200, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "input",
},
};
applyTargetMatching(obs, draft, 1);
draft.postStateId = stateId;
rememberStepOnPage(obs, draft, 1);
return (
buildTraceV3({
obs,
steps: [draft],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]?.body ?? ""
);
}
describe("record observation annotations", () => {
beforeEach(() => {
resetStateIdCounterForTests();
});
it("omits a fill literal from finalized state bodies when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps fill details in ordinary recording annotations", () => {
const value = "ordinary text";
const body = finalizedFillBody(value, false);
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
});
});
describe("applyTargetMatching without a settled observation", () => {
it("keeps the capture role/name instead of a bare unmatched target", () => {
const obs = createObservationState();
const draft: DraftTraceStep = {
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
};
applyTargetMatching(obs, draft, 1);
expect(draft.preStateId).toBeUndefined();
expect(draft.target).toEqual({
role: "button",
name: "新建",
unmatched: true,
});
});
});
describe("applyTargetMatching while the previous action is still settling", () => {
it("does not bind a new unmatched control to the stale observation", () => {
const obs = createObservationState();
const stateId = registerObservation(obs.stateRegistry, {
url: URL,
rawVomText: '@vom 1\n@e1 textbox "Search"',
});
obs.lastSettled = {
stateId,
captured: [capturedInput()],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Search", line: 1 }],
url: URL,
vomText: '@vom 1\n@e1 textbox "Search"',
};
obs.settles.set(0, { draftIndex: 0, cancelled: false });
const draft: DraftTraceStep = {
op: "click",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "Confirm" },
geometry: {
rect: { x: 400, y: 300, w: 80, h: 30 },
scrollX: 0,
scrollY: 0,
position: "fixed",
tag: "button",
},
};
applyTargetMatching(obs, draft, 2);
expect(draft.preStateId).toBeUndefined();
expect(draft.target).toEqual({ role: "button", name: "Confirm", unmatched: true });
});
});
describe("redirect landing coalescing", () => {
it("does not consume a newer redirect hop while reading tab metadata", async () => {
vi.useFakeTimers();
try {
const finalUrl = "https://example.com/final";
const intermediateUrl = "https://example.com/intermediate";
const obs = createObservationState();
obs.lastSettled = {
stateId: "s-final",
captured: [],
refs: [],
url: finalUrl,
vomText: "@vom 1\nFinal",
};
const steps: DraftTraceStep[] = [];
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const get = vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: finalUrl });
const tabsApi: ChromeTabsApi = {
get,
query: vi.fn(async () => []),
};
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Runtime.evaluate") {
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
}
throw new Error(`unexpected CDP method ${method}`);
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, intermediateUrl);
await vi.advanceTimersByTimeAsync(180);
expect(get).toHaveBeenCalledTimes(1);
scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, finalUrl);
resolveFirstTab({ id: 7, url: intermediateUrl } as chrome.tabs.Tab);
await Promise.resolve();
await vi.advanceTimersByTimeAsync(1_000);
await flushPendingRedirectLanding(obs, steps, cdp, 7, tabsApi);
expect(steps).toEqual([]);
expect(obs.pendingRedirect).toBeNull();
} finally {
vi.useRealTimers();
}
});
});
apps/extension/src/lib/__tests__/recording-observation.test.ts
+150
View File
@@ -0,0 +1,150 @@
import { describe, expect, it } from "vitest";
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import type { RecordingDraftStep } from "../recording/types";
const URL = "https://example.com/login";
function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({
url: URL,
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
});
session.cursor.lastSettled = {
stateId: state.id,
rootFrameId: "root",
index: new ObservationNodeIndex({
rootFrameId: "root",
frameDocuments: [
{
frameId: "root",
domNodes: [
{
backendNodeId: 42,
parentBackendNodeId: null,
frameId: "root",
tag: "input",
attrs: {},
rect: { x: 20, y: 40, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
}),
url: URL,
};
return session;
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = {
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Password" },
value,
targetHint: {
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
},
};
session.bindDraft(draft, 1);
draft.postStateId = draft.preStateId;
return buildTraceV3({
registry: session.registry,
drafts: [draft],
annotations: session.annotations,
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]!.body;
}
describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps ordinary fill details", () => {
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
});
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({
url: "https://example.com/a\nb",
title: "hello\nworld",
rawVomText: "@vom 1",
});
const trace = buildTraceV3({
registry,
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
});
});
describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry();
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
});
it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry();
registry.register({ url: URL, rawVomText: "same" });
const state = registry.register({
url: URL,
title: "Login",
rawVomText: "same",
truncated: true,
});
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
});
});
describe("draft binding", () => {
it("keeps capture semantics when no observation exists", () => {
const session = new RecordingObservationSession();
const draft: RecordingDraftStep = {
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
};
session.bindDraft(draft, 1);
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
expect(draft.preStateId).toBeUndefined();
});
it("does not bind an unmatched new action to a stale observation", () => {
const session = sessionWithInput();
const draft: RecordingDraftStep = {
op: "click",
captureTarget: { tag: "button", role: "button", name: "Confirm" },
targetHint: {
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
},
};
session.bindDraft(draft, 2, true);
expect(draft.preStateId).toBeUndefined();
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
role: "button",
name: "Confirm",
unmatched: true,
});
});
});
apps/extension/src/lib/__tests__/settle-controller.test.ts
+136
View File
@@ -0,0 +1,136 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { SettleController } from "../recording/settle-controller";
import type { RecordingDraftStep } from "../recording/types";
const OBSERVATION: RegisteredObservation = {
stateId: "s-next",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: [], refs: [] }),
url: "https://example.com/next",
};
describe("SettleController", () => {
afterEach(() => vi.useRealTimers());
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
let firstSignal: AbortSignal | undefined;
vi.spyOn(session, "capture")
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
firstSignal = signal;
return new Promise<RegisteredObservation>((_resolve, reject) => {
signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
});
})
.mockResolvedValueOnce(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.schedule(drafts, 0);
await vi.advanceTimersByTimeAsync(180);
drafts.push({
op: "click",
captureTarget: { tag: "button", name: "Second" },
preStateId: "s-next",
});
controller.schedule(drafts, 1);
expect(firstSignal?.aborted).toBe(true);
expect(drafts[0]?.postStateId).toBe("s-next");
await vi.advanceTimersByTimeAsync(500);
await controller.flush();
expect(drafts[1]?.postStateId).toBe("s-next");
});
it("does not consume a newer redirect while reading the prior landing URL", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const tabsApi: ChromeTabsApi = {
get: vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
session.cursor.lastSettled = OBSERVATION;
const drafts: RecordingDraftStep[] = [];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
await vi.advanceTimersByTimeAsync(180);
expect(tabsApi.get).toHaveBeenCalledTimes(1);
controller.scheduleRedirect(drafts, OBSERVATION.url);
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(500);
await controller.flushRedirects();
expect(drafts).toEqual([]);
expect(controller.hasPending).toBe(false);
});
it("settles an action in its own OOPIF document scope", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
const target = { tabId: 7, sessionId: "oopif-session" };
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await controller.flush();
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 39 }),
);
expect(drafts[0]?.postStateId).toBe("s-next");
});
});
apps/extension/src/lib/__tests__/recording-step-buffer.test.ts → apps/extension/src/lib/__tests__/step-buffer.test.ts
+8 -8
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
@@ -12,7 +12,7 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
@@ -23,7 +23,7 @@ describe("recording-step-buffer", () => {
steps: [
{
op: "click" as const,
target: { tag: "button", role: "button", name: "发布" },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
@@ -34,8 +34,8 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/b",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
@@ -46,7 +46,7 @@ describe("recording-step-buffer", () => {
steps: [
{
op: "select" as const,
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
@@ -58,9 +58,9 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigated_to: "https://example.com/list?cat=tech",
navigatedTo: "https://example.com/list?cat=tech",
},
]);
});
apps/extension/src/lib/__tests__/target-matcher.test.ts
+126
View File
@@ -0,0 +1,126 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { matchObservationTarget } from "../recording/target-matcher";
function node(backendNodeId: number, frameId: string, x = 10): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "button",
attrs: {},
rect: { x, y: 20, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function observation(
documents: Array<{ frameId: string; domNodes: CapturedNode[] }>,
refs: RenderedRef[],
): RegisteredObservation {
return {
stateId: "s1",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: documents, refs }),
url: "https://example.com",
};
}
describe("matchObservationTarget", () => {
it("matches a unique node using canonical top-level viewport geometry", () => {
const target = matchObservationTarget({
observation: observation(
[{ frameId: "root", domNodes: [node(42, "root")] }],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("uses frame id with backend node id so sibling frames cannot collide", () => {
const target = matchObservationTarget({
observation: observation(
[
{ frameId: "left", domNodes: [node(42, "left")] },
{ frameId: "right", domNodes: [node(42, "right")] },
],
[
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
],
),
hint: {
frameId: "right",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e2");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
[
{ frameId: "root", domNodes: [] },
{ frameId: "child", domNodes: [node(42, "child")] },
],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
[{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }],
[
{ ref: "e1", backendNodeId: 42, line: 1 },
{ ref: "e2", backendNodeId: 43, line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target.unmatched).toBe(true);
});
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
const target = matchObservationTarget({
observation: observation(
[
{ frameId: "root", domNodes: [node(41, "root")] },
{ frameId: "child", domNodes: [node(42, "child")] },
],
[
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
],
),
hint: { frameId: "child" },
fallback: { tag: "button", role: "button", name: "保存" },
});
expect(target.ref).toBe("e2");
});
it("uses semantics to disambiguate equal geometry in one frame", () => {
const target = matchObservationTarget({
observation: observation(
[{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }],
[
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", role: "button", name: "取消" },
});
expect(target.ref).toBe("e2");
});
});
apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts
+170 -49
View File
@@ -1,75 +1,196 @@
import { describe, expect, it } from "vitest";
import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2";
import type { DraftTraceStep } from "@/transport/types";
import type { RecordingDraftStep } from "@/lib/recording/types";
import { shouldRecordPress } from "../recording/draft-policy";
import { buildTraceV2 } from "../recording/trace-reducer-v2";
describe("trace-reducer-v2", () => {
it("drops scroll and bare character press steps", () => {
const steps: DraftTraceStep[] = [
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
return buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
...(startUrl ? { startUrl } : {}),
});
}
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
{
op: "scroll",
page_url: "https://example.com/",
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigatedTo: "https://example.com/results/42",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/p/99",
pageUrl: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/",
pageUrl: "https://example.com/p/99",
},
];
const trace = buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
const { pages, steps } = reduceTraceSteps(
drafts,
"https://example.com/search?q=hello&utm_source=x",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
});
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
});
expect(trace.steps).toHaveLength(0);
expect(trace.pages).toHaveLength(1);
});
it("preserves hover steps supported by protocol v2", () => {
const trace = buildTraceV2({
steps: [
it("collapses consecutive navigations", () => {
const { steps } = reduceTraceSteps([
{ op: "navigate", url: "https://a.example/redirect1" },
{ op: "navigate", url: "https://a.example/final" },
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("keeps committed empty fill steps", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "结束" },
page_url: "https://example.com/",
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
value: "",
pageUrl: "https://example.com/search",
},
],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
});
"https://example.com/search",
);
expect(trace.steps).toEqual([
expect(steps).toEqual([
expect.objectContaining({
op: "hover",
target: expect.objectContaining({ tag: "button", name: "结束" }),
op: "fill",
value: "",
}),
]);
});
it("maps captureTarget to v2 target with required tag", () => {
const steps: DraftTraceStep[] = [
{
op: "click",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "Submit" },
page_url: "https://example.com/",
},
];
const trace = buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigatedTo: "https://example.com/list?cat=tech",
pageUrl: "https://example.com/list",
},
],
"https://example.com/list",
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
});
expect(trace.steps).toHaveLength(1);
expect(trace.steps[0]?.op).toBe("click");
if (trace.steps[0]?.op === "click") {
expect(trace.steps[0].target.tag).toBe("button");
expect(trace.steps[0].target.name).toBe("Submit");
}
expect("version" in trace).toBe(false);
});
it("records Enter presses", () => {
expect(shouldRecordPress("Enter")).toBe(true);
it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
pageUrl: "https://example.com/app",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
pageUrl: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
buildTraceV2({
steps: [{ op: "navigate", url: "https://example.com/other" }],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/start",
}).entry.start_url,
).toBe("https://example.com/start");
});
});
apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts
+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => {
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
{
op: "navigate",
url: "https://example.com/final",
transitionQualifiers: ["server_redirect"],
preStateId: "s2",
postStateId: "s3",
},
];
const before = structuredClone(drafts);
const reduced = reduceTraceStepsV3(drafts);
expect(drafts).toEqual(before);
expect(reduced.steps).toEqual([
expect.objectContaining({
id: 1,
op: "navigate",
state: "s1",
to: "https://example.com/final",
result: { state: "s3" },
}),
]);
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
});
it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
const trace = buildTraceV3({
registry,
drafts: [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "Save" },
preStateId: state.id,
postStateId: state.id,
},
],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.version).toBe(TRACE_VERSION_V3);
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
expect(trace.steps[0]).toMatchObject({
op: "click",
target: { role: "button", name: "Save", unmatched: true },
});
});
});
apps/extension/src/lib/__tests__/trace-reducer.test.ts
-186
View File
@@ -1,186 +0,0 @@
import { describe, expect, it } from "vitest";
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: DraftTraceStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/p/99",
},
];
const { pages, steps } = reduceTraceSteps(
drafts,
"https://example.com/search?q=hello&utm_source=x",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
});
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
});
});
it("collapses consecutive navigations", () => {
const { steps } = reduceTraceSteps([
{ op: "navigate", url: "https://a.example/redirect1" },
{ op: "navigate", url: "https://a.example/final" },
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("keeps committed empty fill steps", () => {
const { steps } = reduceTraceSteps(
[
{
op: "fill",
target: { tag: "input", role: "textbox", name: "Search query" },
value: "",
page_url: "https://example.com/search",
},
],
"https://example.com/search",
);
expect(steps).toEqual([
expect.objectContaining({
op: "fill",
value: "",
}),
]);
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
},
],
"https://example.com/list",
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
});
});
it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { tag: "span", role: "button", name: "Account" },
page_url: "https://example.com/app",
},
{
op: "click",
target: { tag: "a", role: "link", name: "Profile" },
page_url: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
"https://example.com/start",
),
).toBe("https://example.com/start");
});
});
apps/extension/src/lib/describe-target.ts
+1 -4
View File
@@ -1,5 +1,5 @@
/**
* Build a semantic TargetDescriptor for an interacted element.
* Build a semantic capture descriptor for an interacted element.
*
* Trace steps are an LLM *textbook*: each click must say what to look for
* on screen (usually a short visible name). Tag-only noise like
@@ -16,9 +16,6 @@ export interface CaptureTargetDescriptor {
nearby_label?: string;
}
/** @deprecated Capture-time alias retained until the recorder integration migrates. */
export type TargetDescriptor = CaptureTargetDescriptor;
/** Max length for a label that is still a useful “find this on screen” hint. */
const ACTIONABLE_LABEL_MAX = 48;
apps/extension/src/lib/format-observation-file.ts
-62
View File
@@ -1,62 +0,0 @@
import type { StepV3 } from "@/transport/types";
import { OBSERVATION_FILE_VERSION } from "./record-constants";
export interface ObservationAnnotation {
stepId: number;
op: StepV3["op"];
line: number;
stateId: string;
detail?: string;
}
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
const suffix = detail ? `: ${detail}` : "";
return ` ⟵ step ${stepId}: ${op}${suffix}`;
}
/** Serialize a page observation file (front matter + VOM body + step annotations). */
export function formatObservationFile(input: {
stateId: string;
url: string;
title?: string;
stepsHere: number[];
body: string;
annotations?: ObservationAnnotation[];
}): string {
const lines: string[] = [
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
`state: ${input.stateId}`,
`url: ${input.url}`,
];
if (input.title) lines.push(`title: ${input.title}`);
if (input.stepsHere.length > 0) {
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
}
lines.push("---");
const bodyLines = input.body.split("\n");
const annotationMap = new Map<number, ObservationAnnotation[]>();
for (const ann of input.annotations ?? []) {
const bucket = annotationMap.get(ann.line) ?? [];
bucket.push(ann);
annotationMap.set(ann.line, bucket);
}
for (let i = 0; i < bodyLines.length; i += 1) {
let line = bodyLines[i] ?? "";
const anns = annotationMap.get(i);
if (anns) {
for (const ann of anns) {
line += formatAnnotation(ann);
}
}
lines.push(line);
}
return `${lines.join("\n")}\n`;
}
/** Hash input: VOM body **before** annotations are inserted. */
export function observationBodyForHash(vomText: string): string {
return vomText;
}
apps/extension/src/lib/match-target.ts
-103
View File
@@ -1,103 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CapturedNode } from "@/tools/vom/capture";
import type { CaptureGeometry, TargetDescriptorV3 } from "@/transport/types";
import type { CaptureTargetDescriptor } from "./describe-target";
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
export interface MatchTargetInput {
geometry: CaptureGeometry;
captured: CapturedNode[];
refs: RenderedRef[];
fallback?: CaptureTargetDescriptor;
}
function withinTolerance(a: number, b: number): boolean {
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
}
function rectsMatch(
a: { x: number; y: number; w: number; h: number },
b: { x: number; y: number; w: number; h: number },
): boolean {
return (
withinTolerance(a.x, b.x) &&
withinTolerance(a.y, b.y) &&
withinTolerance(a.w, b.w) &&
withinTolerance(a.h, b.h)
);
}
function nodeViewportRect(
node: CapturedNode,
): { x: number; y: number; w: number; h: number } | null {
return node.localRect ?? node.rect;
}
function matchingRects(
geometry: CaptureGeometry,
node: CapturedNode,
): {
target: { x: number; y: number; w: number; h: number };
node: { x: number; y: number; w: number; h: number } | null;
} {
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
if (!topFrame || viewportPosition || !node.documentRect) {
return { target: geometry.rect, node: nodeViewportRect(node) };
}
return {
target: {
x: geometry.rect.x + geometry.scrollX,
y: geometry.rect.y + geometry.scrollY,
w: geometry.rect.w,
h: geometry.rect.h,
},
node: node.documentRect,
};
}
function tagMatches(geometryTag: string, nodeTag: string): boolean {
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
}
/** Best description available when the element cannot be located in the VOM. */
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
if (!fallback) {
return { unmatched: true };
}
return {
...(fallback.role ? { role: fallback.role } : {}),
...(fallback.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
/** Locate the interacted element in the last settled observation by geometry. */
export function matchTarget(input: MatchTargetInput): TargetDescriptorV3 {
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
const candidates = input.captured.filter((node) => {
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
if (!tagMatches(input.geometry.tag, node.tag)) return false;
const rects = matchingRects(input.geometry, node);
if (!rects.node) return false;
return rectsMatch(rects.target, rects.node);
});
if (candidates.length !== 1) {
return fallbackDescriptor(input.fallback);
}
const backendNodeId = candidates[0]!.backendNodeId;
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
if (!refEntry) {
return fallbackDescriptor(input.fallback);
}
return {
ref: refEntry.ref,
...(refEntry.role ? { role: refEntry.role } : {}),
...(refEntry.name ? { name: refEntry.name } : {}),
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
};
}
apps/extension/src/lib/page-settled.ts
-92
View File
@@ -1,92 +0,0 @@
// Deciding *when* a page is worth observing.
//
// A fixed delay cannot serve both a modal that toggles in one frame and a
// route change that renders for a second. Instead of guessing, ask the page:
// a document that has stopped mutating and is no longer loading is done
// reacting to whatever the user just did.
import type { CdpRunner } from "@/tools/shared";
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
/**
* Installed once per document. Records only the timestamp of the last DOM
* change so each poll stays O(1) no matter how large the page is.
*/
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export type SettleOutcome =
/** The page stopped changing on its own. */
| "quiet"
/** Still changing when the budget ran out; observe it as it is. */
| "timeout"
/** A newer action took over; this observation is no longer wanted. */
| "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
try {
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
expression: QUIET_PROBE,
returnByValue: true,
});
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
// not a reason to give up.
return null;
}
}
/** Wait until the page has finished reacting, or until the budget runs out. */
export async function waitForPageSettled(
cdp: CdpRunner,
tabId: number,
options: { cancelled?: () => boolean } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
for (;;) {
if (options.cancelled?.()) return "cancelled";
await sleep(SETTLE_POLL_MS);
if (options.cancelled?.()) return "cancelled";
const probe = await readQuietProbe(cdp, tabId);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe) continue;
if (probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
apps/extension/src/lib/record-bridge.ts
+2 -2
View File
@@ -3,7 +3,7 @@
* service worker and a tab's content script.
*/
import type { TargetDescriptor } from "./describe-target";
import type { CaptureTargetDescriptor } from "./describe-target";
export const RECORD_START = "bsk-record-start";
export const RECORD_STEP = "bsk-record-step";
@@ -42,7 +42,7 @@ export interface RecordStartMessage {
export interface RecordStepPayload {
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
target?: TargetDescriptor;
target?: CaptureTargetDescriptor;
value?: string;
key?: string;
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
apps/extension/src/lib/record-constants.ts
-97
View File
@@ -1,97 +0,0 @@
import type { FillCommit, NavigationCause } from "@/transport/types";
/**
* Floor before a post-action observation may be taken. A page usually has not
* started reacting in the first frames after an action, and capturing then
* would record the page as it was, not as the action left it.
*/
export const SETTLE_MIN_MS = 150;
/** How long the DOM must stop changing before a page counts as settled. */
export const SETTLE_QUIET_MS = 250;
/** Upper bound on waiting for a page to settle; animations never stop. */
export const SETTLE_MAX_MS = 2_000;
/** How often to ask the page whether it has gone quiet. */
export const SETTLE_POLL_MS = 60;
/** Minimum interval between consecutive observations on the same tab. */
export const OBSERVATION_MIN_INTERVAL_MS = 200;
/** Delay before retrying a capture that lost its execution context. */
export const CAPTURE_RETRY_DELAY_MS = 250;
/** Document-coordinate matching tolerance in CSS pixels. */
export const GEOM_MATCH_TOLERANCE_PX = 2;
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
/** VOM observation file format version (front matter header). */
export const OBSERVATION_FILE_VERSION = 1;
/** FNV-1a 64-bit offset basis. */
const FNV_OFFSET = 0xcbf29ce484222325n;
/** FNV-1a 64-bit prime. */
const FNV_PRIME = 0x100000001b3n;
/** Content-hash for state deduplication (sync, non-cryptographic). */
export function fnv1a64(text: string): string {
let hash = FNV_OFFSET;
for (let i = 0; i < text.length; i += 1) {
hash ^= BigInt(text.charCodeAt(i));
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
}
return hash.toString(16).padStart(16, "0");
}
let nextStateSerial = 0;
/** Monotonic state id generator (`s1`, `s2`, …). */
export function nextStateId(): string {
nextStateSerial += 1;
return `s${nextStateSerial}`;
}
/** Test seam: reset the state id counter. */
export function resetStateIdCounterForTests(): void {
nextStateSerial = 0;
}
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
export interface NavigationTransitionMeta {
transitionType?: string;
transitionQualifiers?: string[];
navigationActionPending?: boolean;
}
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
const qualifiers = meta.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
if (qualifiers.includes("from_address_bar")) return "user_typed";
const type = meta.transitionType ?? "";
const mapped = TRANSITION_TO_CAUSE[type];
if (mapped) return mapped;
if (!type && meta.navigationActionPending === false) return "script";
return "browser";
}
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
apps/extension/src/lib/record-observation.ts
-681
View File
@@ -1,681 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
import { waitForPageSettled } from "@/lib/page-settled";
import {
CAPTURE_RETRY_DELAY_MS,
DEFAULT_MAX_PAGE_TOKENS,
OBSERVATION_MIN_INTERVAL_MS,
} from "@/lib/record-constants";
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
import { captureVomObservation } from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep, StepV3, StopReason, TraceState, TraceV3 } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
export interface LastSettledObservation {
stateId: string;
captured: CapturedNode[];
refs: RenderedRef[];
url: string;
title?: string;
vomText: string;
}
/** A post-action observation that has been queued but not yet written down. */
interface PendingSettle {
draftIndex: number;
/** Set when a newer action has already decided where this step landed. */
cancelled: boolean;
}
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
export interface PendingRedirectLanding {
url: string;
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
generation: number;
}
export interface RecordingObservationState {
stateRegistry: Map<string, StateRegistryEntry>;
lastSettled: LastSettledObservation | null;
maxPageTokens: number;
redactValues: boolean;
lastCaptureAtMs: number;
/**
* Tail of the settle chain. Observations run one at a time so that a slow
* capture can never land after a faster one and report the pages out of the
* order the user visited them.
*/
settleQueue: Promise<void>;
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
settles: Map<number, PendingSettle>;
stepAnnotations: Map<number, ObservationAnnotation[]>;
/**
* OAuth / server redirects are coalesced here: intermediate hops only update
* `url`+`generation`, and one `navigate` is emitted after the page settles.
*/
pendingRedirect: PendingRedirectLanding | null;
redirectFlushQueue: Promise<void>;
}
export function createObservationState(options?: {
maxPageTokens?: number;
redactValues?: boolean;
}): RecordingObservationState {
return {
stateRegistry: new Map(),
lastSettled: null,
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
redactValues: options?.redactValues ?? false,
lastCaptureAtMs: 0,
settleQueue: Promise.resolve(),
settles: new Map(),
stepAnnotations: new Map(),
pendingRedirect: null,
redirectFlushQueue: Promise.resolve(),
};
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureAndRegisterObservation(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
urlOverride?: string,
): Promise<LastSettledObservation> {
const now = Date.now();
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
if (waitMs > 0) await sleep(waitMs);
const { url, title } = urlOverride
? { url: urlOverride, title: undefined }
: await readTabMeta(tabsApi, tabId);
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
const resolvedTitle = title ?? meta.title;
const rendered = await captureVomObservation(cdp, tabId, url, {
maxTokens: obs.maxPageTokens,
redactValues: obs.redactValues,
conditionalSurfaceProbe: false,
});
const stateId = registerObservation(obs.stateRegistry, {
url,
title: resolvedTitle,
rawVomText: rendered.text,
truncated: rendered.truncated,
});
obs.lastCaptureAtMs = Date.now();
const settled: LastSettledObservation = {
stateId,
captured: rendered.captured,
refs: rendered.refs,
url,
title: resolvedTitle,
vomText: rendered.text,
};
obs.lastSettled = settled;
return settled;
}
/**
* Bind a draft to the page it was performed on: origin state, geometric target
* match, and the inline annotation. All three read the *current* observation,
* which is only the page the user acted on while the draft is still fresh —
* once the action settles, `lastSettled` is the destination page instead.
*/
export function applyTargetMatching(
obs: RecordingObservationState,
draft: DraftTraceStep,
stepId: number,
): void {
if (!obs.lastSettled) {
// Initial observation can fail or race the first action. Keep whatever
// the content script captured so the exported step is still teachable.
if ("target" in draft && "captureTarget" in draft) {
draft.target = fallbackDescriptor(draft.captureTarget);
}
rememberStepAnnotation(obs, stepId, draft);
return;
}
if ("target" in draft) {
draft.target =
"geometry" in draft && draft.geometry
? matchTarget({
geometry: draft.geometry,
captured: obs.lastSettled.captured,
refs: obs.lastSettled.refs,
fallback: draft.captureTarget,
})
: // No geometry to match on, but the capture still knew what the user
// touched — keep that instead of an anonymous unmatched target.
fallbackDescriptor(draft.captureTarget);
}
// A pending settle means `lastSettled` may still describe the page before
// the previous action. If the new control cannot be found there, attaching
// that stale state would be worse than leaving the origin unknown for the
// reducer to repair from the observations that do exist.
const originLooksStale =
obs.settles.size > 0 && (!("target" in draft) || draft.target?.unmatched === true);
if (originLooksStale) {
rememberStepAnnotation(obs, stepId, draft);
return;
}
draft.preStateId = obs.lastSettled.stateId;
rememberStepAnnotation(obs, stepId, draft);
}
function fillDetailForDraft(
obs: RecordingObservationState,
draft: DraftTraceStep,
): string | undefined {
if (obs.redactValues) return undefined;
if (draft.op === "fill") return JSON.stringify(draft.value);
return undefined;
}
function refLineForDraft(
obs: RecordingObservationState,
draft: DraftTraceStep,
): number | undefined {
if (!("target" in draft) || !obs.lastSettled) return undefined;
const targetRef = draft.target?.ref;
if (!targetRef) return undefined;
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
}
function rememberStepAnnotation(
obs: RecordingObservationState,
stepId: number,
draft: DraftTraceStep,
): void {
const line = refLineForDraft(obs, draft);
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
if (!draft.preStateId) return;
const ann: ObservationAnnotation = {
stepId,
op: draft.op,
line,
stateId: draft.preStateId,
detail: fillDetailForDraft(obs, draft),
};
const bucket = obs.stepAnnotations.get(line) ?? [];
bucket.push(ann);
obs.stepAnnotations.set(line, bucket);
}
/** Record the step under the page it was performed on, not the one it led to. */
export function rememberStepOnPage(
obs: RecordingObservationState,
draft: DraftTraceStep,
stepId: number,
): void {
const stateId = draft.preStateId;
if (!stateId) return;
const entry = obs.stateRegistry.get(stateId);
if (entry && !entry.stepsHere.includes(stepId)) {
entry.stepsHere.push(stepId);
}
}
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
obs.pendingRedirect = null;
}
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
}
/**
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
* to the final URL. Later hops only bump `generation` / replace `url`.
*/
export function scheduleRedirectLandingFlush(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
url: string,
options: { cancelled?: () => boolean } = {},
): void {
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
obs.pendingRedirect = { url, generation };
enqueueRedirectFlush(obs, () =>
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
);
}
/**
* Drain any coalesced redirect so the next action matches against the real
* landing page. Safe to call when nothing is pending.
*/
export async function flushPendingRedirectLanding(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner | undefined,
tabId: number,
tabsApi: ChromeTabsApi,
options: { cancelled?: () => boolean } = {},
): Promise<void> {
if (obs.pendingRedirect && cdp) {
enqueueRedirectFlush(obs, () =>
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
);
}
await obs.redirectFlushQueue;
}
/**
* Wait until the redirect chain stops changing the document, then emit one
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
*/
async function coalesceRedirectLanding(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
options: { cancelled?: () => boolean },
): Promise<void> {
while (obs.pendingRedirect) {
if (options.cancelled?.()) {
obs.pendingRedirect = null;
return;
}
const snap = obs.pendingRedirect;
const outcome = await waitForPageSettled(cdp, tabId, {
cancelled: () =>
!!options.cancelled?.() ||
obs.pendingRedirect === null ||
obs.pendingRedirect.generation !== snap.generation,
});
if (options.cancelled?.()) {
obs.pendingRedirect = null;
return;
}
// A newer hop cancelled this wait — loop and settle against the latest URL.
if (outcome === "cancelled") continue;
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
continue;
}
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
// `tabs.get` is asynchronous. A newer redirect hop may have arrived while
// it was in flight; only the generation that initiated this read may
// consume the pending landing.
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
continue;
}
obs.pendingRedirect = null;
if (!finalUrl || finalUrl === "about:blank") return;
if (obs.lastSettled?.url === finalUrl) return;
const last = steps[steps.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) {
const draftIndex = steps.length - 1;
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
await obs.settleQueue;
}
return;
}
if (outcome === "timeout") {
console.debug(
`[bsk record] redirect landing still changing after settle budget; ` +
`recording navigate to ${finalUrl}`,
);
}
const draft: DraftTraceStep = {
op: "navigate",
url: finalUrl,
page_url: finalUrl,
cause: "browser",
preStateId: obs.lastSettled?.stateId,
};
steps.push(draft);
const draftIndex = steps.length - 1;
const stepId = draftIndex + 1;
rememberStepOnPage(obs, draft, stepId);
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
// Callers that flush before the next action need `lastSettled` to already
// be the landing page so target matching does not use the pre-redirect view.
await obs.settleQueue;
return;
}
}
/**
* A capture that lands mid-navigation fails on a destroyed execution context.
* That is transient, so give it one more chance before the step has to fall
* back to a stale state.
*/
async function captureWithRetry(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
urlOverride?: string,
): Promise<LastSettledObservation> {
try {
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
} catch (err) {
console.debug("[bsk record] observation failed, retrying once", err);
await sleep(CAPTURE_RETRY_DELAY_MS);
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
}
}
export async function settleDraftObservation(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
draftIndex: number,
pending?: PendingSettle,
): Promise<void> {
const cancelled = () => pending?.cancelled === true;
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
if (outcome === "cancelled") return;
if (outcome === "timeout") {
console.debug(
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
"budget; observing it as it stands",
);
}
const started = steps[draftIndex];
if (!started) return;
// The URL attached to the draft may only be an intermediate redirect hop.
// Once the page is settled, register the capture against live tab metadata.
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
// The observation still counts as the latest view of the page even when this
// step no longer wants it — the next action will start from it.
if (cancelled()) return;
// Re-read the slot: a navigation observed while the capture ran may have
// rewritten this draft, and the observation belongs to whatever occupies the
// slot now — writing to the object we started with would strand it.
const draft = steps[draftIndex] ?? started;
draft.postStateId = settled.stateId;
}
/**
* Recover landing states from the shape of the recording itself: wherever the
* next action was performed is, by definition, where the previous one landed.
* Only the immediately following draft counts — a later one would claim a page
* that several unobserved actions away, which is worse than admitting we saw
* no change.
*/
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
for (let i = 0; i < steps.length - 1; i += 1) {
const draft = steps[i];
const next = steps[i + 1];
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
draft.postStateId = next.preStateId;
console.debug(
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
`using where step ${i + 2} started (${next.preStateId})`,
);
}
}
/**
* Last chance for the closing actions of a recording. Stopping right after the
* final action is normal, and so is that action navigating away, so take one
* final look at the page. Only trailing drafts may claim it: an earlier step
* did not land on whatever the user happens to be looking at when they stop.
*/
export async function settleUnsettledDrafts(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
): Promise<void> {
const trailing: DraftTraceStep[] = [];
for (let i = steps.length - 1; i >= 0; i -= 1) {
const draft = steps[i];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
let settled: LastSettledObservation;
try {
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
} catch (err) {
console.warn("[bsk record] final observation at stop failed", err);
return;
}
for (const draft of trailing) draft.postStateId = settled.stateId;
console.debug(
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
`(${settled.stateId} ${settled.url})`,
);
}
/**
* Performing a new action ends the previous one: the page the user reached for
* is, by definition, where the earlier step left them. Taking the landing from
* the newer step also keeps the trace monotonic — a capture that finished late
* would otherwise credit an earlier step with a page that only exists because
* of the newer action.
*/
function supersedeEarlierSettles(
obs: RecordingObservationState,
draftIndex: number,
steps: DraftTraceStep[],
): void {
const landing = steps[draftIndex]?.preStateId;
for (const [index, pending] of obs.settles) {
if (index >= draftIndex) continue;
pending.cancelled = true;
obs.settles.delete(index);
const draft = steps[index];
if (!draft || draft.postStateId) continue;
if (!landing) continue;
draft.postStateId = landing;
console.debug(
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
`${draftIndex + 1} started; landing it on ${landing}`,
);
}
}
export function scheduleDraftSettle(
obs: RecordingObservationState,
draftIndex: number,
stepId: number,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
): void {
supersedeEarlierSettles(obs, draftIndex, steps);
// Rescheduling the same step (a navigation showed up after the action)
// replaces the pending observation rather than racing it.
const superseded = obs.settles.get(draftIndex);
if (superseded) superseded.cancelled = true;
const pending: PendingSettle = { draftIndex, cancelled: false };
obs.settles.set(draftIndex, pending);
enqueueSettle(obs, async () => {
if (pending.cancelled) return;
try {
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
} catch (err) {
// Recoverable: stop-time repair still gives the step a landing page.
const op = steps[draftIndex]?.op ?? "?";
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
} finally {
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
}
});
}
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
}
export function cancelPendingSettles(obs: RecordingObservationState): void {
for (const pending of obs.settles.values()) pending.cancelled = true;
obs.settles.clear();
}
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
const MAX_SETTLE_FLUSH_ROUNDS = 10;
/**
* Drain settle work before exporting, including work queued while draining —
* a navigation observed during the last capture schedules another one, and
* the trace would drop it if stop did not wait.
*/
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
const drained = obs.settleQueue;
await drained;
if (obs.settleQueue === drained) return;
}
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
}
function finalizeStateBodies(
entries: StateRegistryEntry[],
annotationsByState: Map<string, ObservationAnnotation[]>,
stepIdByDraftId: Map<number, number>,
idByOldId: Map<string, string>,
): TraceState[] {
return entries.map((entry) => {
const id = idByOldId.get(entry.id) ?? entry.id;
const body = formatObservationFile({
stateId: id,
url: entry.url,
title: entry.title,
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
body: entry.rawVomText,
annotations: annotationsByState.get(entry.id) ?? [],
});
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body,
...(entry.truncated ? { truncated: true } : {}),
};
});
}
/** Draft ids only become step ids after collapsing and filtering. */
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
const mapped = new Set<number>();
for (const draftId of draftIds) {
const stepId = stepIdByDraftId.get(draftId);
if (stepId !== undefined) mapped.add(stepId);
}
return [...mapped].sort((a, b) => a - b);
}
function collectAnnotationsByState(
obs: RecordingObservationState,
stepIdByDraftId: Map<number, number>,
): Map<string, ObservationAnnotation[]> {
const byState = new Map<string, ObservationAnnotation[]>();
for (const anns of obs.stepAnnotations.values()) {
for (const ann of anns) {
const stepId = stepIdByDraftId.get(ann.stepId);
if (stepId === undefined) continue;
const bucket = byState.get(ann.stateId) ?? [];
bucket.push({ ...ann, stepId });
byState.set(ann.stateId, bucket);
}
}
return byState;
}
/**
* Keep only the pages the published steps point at. Redirect hops and
* mid-load captures land in the registry too, and shipping them would invite
* a reader to treat a page the flow merely passed through as a real stop.
* With no steps at all the first observation is the whole artifact, so it
* stays.
*/
function selectPublishedStates(
registry: Map<string, StateRegistryEntry>,
steps: StepV3[],
): StateRegistryEntry[] {
const entries = [...registry.values()];
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set<string>();
for (const step of steps) {
referenced.add(step.state);
referenced.add(step.result.state);
}
return entries.filter((entry) => referenced.has(entry.id));
}
export function buildTraceV3(input: {
obs: RecordingObservationState;
steps: DraftTraceStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
}): TraceV3 {
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
const published = selectPublishedStates(input.obs.stateRegistry, steps);
// Renumber so the shipped dictionary reads s1..sN without holes where the
// dropped captures used to be.
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
for (const step of steps) {
step.state = idByOldId.get(step.state) ?? step.state;
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
}
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
return {
version: 3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: startUrl },
recorder: { bsk: input.bskVersion, vom: 1 },
states,
steps,
};
}
apps/extension/src/lib/recording/document-settle.ts
+134
View File
@@ -0,0 +1,134 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
const SETTLE_MIN_MS = 150;
const SETTLE_QUIET_MS = 250;
const SETTLE_MAX_MS = 2_000;
const SETTLE_POLL_MS = 60;
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export interface DocumentSettleScope {
target: CdpTarget;
/** Omit only for the target's root document. */
frameId?: string;
}
type SettleOutcome = "quiet" | "timeout" | "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
interface ProbeContext {
executionContextId?: number;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
async function executionContextForScope(
cdp: CdpRunner,
scope: DocumentSettleScope,
): Promise<number | undefined> {
if (!scope.frameId) return undefined;
const result = await sendToCdpTarget<{ executionContextId?: number }>(
cdp,
scope.target,
"Page.createIsolatedWorld",
{
frameId: scope.frameId,
worldName: SETTLE_WORLD_NAME,
grantUniveralAccess: false,
},
);
return result.executionContextId;
}
async function readQuietProbe(
cdp: CdpRunner,
scope: DocumentSettleScope,
context: ProbeContext,
): Promise<QuietProbe | null> {
try {
if (scope.frameId && context.executionContextId === undefined) {
context.executionContextId = await executionContextForScope(cdp, scope);
if (context.executionContextId === undefined) return null;
}
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
cdp,
scope.target,
"Runtime.evaluate",
{
expression: QUIET_PROBE,
returnByValue: true,
...(context.executionContextId !== undefined
? { contextId: context.executionContextId }
: {}),
},
);
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
context.executionContextId = undefined;
return null;
}
}
export async function waitForDocumentSettled(
cdp: CdpRunner,
scope: DocumentSettleScope,
options: { signal?: AbortSignal } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
const context: ProbeContext = {};
for (;;) {
if (options.signal?.aborted) return "cancelled";
await sleep(SETTLE_POLL_MS, options.signal);
if (options.signal?.aborted) return "cancelled";
const probe = await readQuietProbe(cdp, scope, context);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe || probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
apps/extension/src/lib/recording/draft-policy.ts
+34
View File
@@ -0,0 +1,34 @@
import type { RecordingDraftStep } from "./types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
}
export function resolveDraftStartUrl(
drafts: RecordingDraftStep[],
explicitStartUrl?: string,
fallbackUrl?: string,
): string {
if (explicitStartUrl) return explicitStartUrl;
const navigation = drafts.find(
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
);
if (navigation) return navigation.url;
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
}
apps/extension/src/lib/recording/observation-capture.ts
+115
View File
@@ -0,0 +1,115 @@
import type { RenderedRef } from "@browser-skill/vom";
import { captureVomObservation } from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { CapturedNode } from "@/tools/vom/capture";
export interface IndexedObservationNode {
frameId: string;
node: CapturedNode;
ref?: RenderedRef;
}
export interface CapturedRecordingObservation {
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
title?: string;
vomText: string;
truncated: boolean;
}
export interface RegisteredObservation {
stateId: string;
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
function frameTagKey(frameId: string, tag: string): string {
return `${frameId}:${tag.toLowerCase()}`;
}
export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
constructor(input: {
rootFrameId: string;
frameDocuments: Array<{ frameId: string; domNodes: CapturedNode[] }>;
refs: RenderedRef[];
}) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
this.#refById.set(ref.ref, ref);
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const document of input.frameDocuments) {
for (const node of document.domNodes) {
const frameId = node.frameId ?? document.frameId;
const entry = { frameId, node, ref: refByNode.get(nodeKey(frameId, node.backendNodeId)) };
const key = frameTagKey(frameId, node.tag);
const bucket = this.#nodesByFrameTag.get(key) ?? [];
bucket.push(entry);
this.#nodesByFrameTag.set(key, bucket);
}
}
}
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
}
ref(refId: string): RenderedRef | undefined {
return this.#refById.get(refId);
}
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureRecordingObservation(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
maxTokens: number;
redactValues: boolean;
signal?: AbortSignal;
}): Promise<CapturedRecordingObservation> {
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
maxTokens: input.maxTokens,
redactValues: input.redactValues,
conditionalSurfaceProbe: false,
signal: input.signal,
});
return {
rootFrameId: captured.rootFrameId,
index: new ObservationNodeIndex(captured),
url,
title,
vomText: captured.text,
truncated: captured.truncated,
};
}
apps/extension/src/lib/recording/observation-session.ts
+122
View File
@@ -0,0 +1,122 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
import { RecordingStateRegistry } from "./state-registry";
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
const MIN_CAPTURE_INTERVAL_MS = 200;
export interface TabObservationCursor {
lastSettled: RegisteredObservation | null;
lastCaptureAt: number;
}
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return draft.op !== "navigate" && draft.op !== "scroll";
}
export class RecordingObservationSession {
readonly registry: RecordingStateRegistry;
readonly cursor: TabObservationCursor;
readonly annotations: StepAnnotation[];
readonly #maxTokens: number;
readonly #redactValues: boolean;
constructor(
options: {
registry?: RecordingStateRegistry;
cursor?: TabObservationCursor;
annotations?: StepAnnotation[];
maxTokens?: number;
redactValues?: boolean;
} = {},
) {
this.registry = options.registry ?? new RecordingStateRegistry();
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
this.annotations = options.annotations ?? [];
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
this.#redactValues = options.redactValues ?? false;
}
async capture(
cdp: CdpRunner,
tabsApi: ChromeTabsApi,
tabId: number,
signal?: AbortSignal,
): Promise<RegisteredObservation> {
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
if (waitMs > 0) await abortableDelay(waitMs, signal);
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
const captured = await captureRecordingObservation({
cdp,
tabsApi,
tabId,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
signal,
});
const state = this.registry.register({
url: captured.url,
title: captured.title,
rawVomText: captured.vomText,
truncated: captured.truncated,
});
const observation: RegisteredObservation = {
stateId: state.id,
rootFrameId: captured.rootFrameId,
index: captured.index,
url: captured.url,
};
this.cursor.lastSettled = observation;
this.cursor.lastCaptureAt = Date.now();
return observation;
}
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
const observation = this.cursor.lastSettled;
if (isTargeted(draft)) {
draft.matchedTarget = observation
? matchObservationTarget({
observation,
hint: draft.targetHint,
fallback: draft.captureTarget,
})
: unmatchedTarget(draft.captureTarget);
}
if (!observation) return;
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
if (previousActionPending && unmatched) return;
draft.preStateId = observation.stateId;
this.registry.markStep(observation.stateId, draftId);
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
const ref = observation.index.ref(draft.matchedTarget.ref);
if (!ref) return;
this.annotations.push({
draftId,
op: draft.op,
line: ref.line,
stateId: observation.stateId,
...(draft.op === "fill" && !this.#redactValues
? { detail: JSON.stringify(draft.value) }
: {}),
});
}
}
apps/extension/src/lib/recording/settle-controller.ts
+232
View File
@@ -0,0 +1,232 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import { RecordingObservationSession } from "./observation-session";
import type { RecordingDraftStep } from "./types";
interface PendingSettle {
abort: AbortController;
scope: DocumentSettleScope;
}
interface PendingRedirect {
url: string;
abort: AbortController;
}
const CAPTURE_RETRY_DELAY_MS = 250;
function delay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
for (let index = 0; index < drafts.length - 1; index += 1) {
const draft = drafts[index];
const next = drafts[index + 1];
if (draft && next && !draft.postStateId && next.preStateId) {
draft.postStateId = next.preStateId;
}
}
}
export class SettleController {
readonly #session: RecordingObservationSession;
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #tabId: number;
readonly #rootScope: DocumentSettleScope;
readonly #pending = new Map<number, PendingSettle>();
#queue = Promise.resolve();
#pendingRedirect: PendingRedirect | null = null;
#redirectQueue = Promise.resolve();
constructor(input: {
session: RecordingObservationSession;
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
}) {
this.#session = input.session;
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#tabId = input.tabId;
this.#rootScope = { target: { tabId: input.tabId } };
}
get hasPending(): boolean {
return this.#pending.size > 0 || this.#pendingRedirect !== null;
}
async #captureWithRetry(signal?: AbortSignal) {
try {
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
} catch (error) {
if (signal?.aborted) throw error;
await delay(CAPTURE_RETRY_DELAY_MS, signal);
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
}
}
schedule(
drafts: RecordingDraftStep[],
draftIndex: number,
scope: DocumentSettleScope = this.#rootScope,
): void {
if (scope.target.tabId !== this.#tabId) {
throw new Error(
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
);
}
const landing = drafts[draftIndex]?.preStateId;
for (const [index, pending] of this.#pending) {
if (index >= draftIndex) continue;
pending.abort.abort();
this.#pending.delete(index);
if (landing && drafts[index] && !drafts[index].postStateId) {
drafts[index].postStateId = landing;
}
}
this.#pending.get(draftIndex)?.abort.abort();
const pending = { abort: new AbortController(), scope };
this.#pending.set(draftIndex, pending);
const task = async () => {
if (pending.abort.signal.aborted) return;
try {
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled") return;
const observation = await this.#captureWithRetry(pending.abort.signal);
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
drafts[draftIndex].postStateId = observation.stateId;
}
} catch (error) {
if (!isAbortError(error)) {
console.warn(
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
error,
);
}
} finally {
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
}
};
this.#queue = this.#queue.then(task, task).catch(() => {});
}
cancel(): void {
for (const pending of this.#pending.values()) pending.abort.abort();
this.#pending.clear();
this.clearRedirect();
}
async flush(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#queue;
await current;
if (current === this.#queue) return;
}
console.warn("[bsk record] settle queue kept growing; using completed observations");
}
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
const trailing: RecordingDraftStep[] = [];
for (let index = drafts.length - 1; index >= 0; index -= 1) {
const draft = drafts[index];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
try {
const observation = await this.#captureWithRetry();
for (const draft of trailing) draft.postStateId = observation.stateId;
} catch (error) {
console.warn("[bsk record] final observation at stop failed", error);
}
}
clearRedirect(): void {
this.#pendingRedirect?.abort.abort();
this.#pendingRedirect = null;
}
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
this.#pendingRedirect?.abort.abort();
const pending: PendingRedirect = { url, abort: new AbortController() };
this.#pendingRedirect = pending;
const task = () => this.#settleRedirect(drafts, pending);
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
}
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
let finalUrl = pending.url;
try {
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
} catch {
// The navigation event URL remains the best available destination.
}
if (this.#pendingRedirect !== pending) return;
this.#pendingRedirect = null;
if (
!finalUrl ||
finalUrl === "about:blank" ||
this.#session.cursor.lastSettled?.url === finalUrl
) {
return;
}
const last = drafts[drafts.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
await this.flush();
return;
}
const draft: RecordingDraftStep = {
op: "navigate",
url: finalUrl,
pageUrl: finalUrl,
cause: "browser",
preStateId: this.#session.cursor.lastSettled?.stateId,
};
drafts.push(draft);
const draftIndex = drafts.length - 1;
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
this.schedule(drafts, draftIndex);
await this.flush();
}
async flushRedirects(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#redirectQueue;
await current;
if (current === this.#redirectQueue) return;
}
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
}
}
apps/extension/src/lib/recording/state-registry.ts
+56
View File
@@ -0,0 +1,56 @@
export interface RecordedStateEntry {
id: string;
url: string;
title?: string;
rawVomText: string;
truncated: boolean;
stepsHere: number[];
}
function stateIdentity(url: string, body: string): string {
return `${url}\0${body}`;
}
export class RecordingStateRegistry {
readonly #entriesById = new Map<string, RecordedStateEntry>();
readonly #idByIdentity = new Map<string, string>();
#nextId = 1;
register(input: {
url: string;
title?: string;
rawVomText: string;
truncated?: boolean;
}): RecordedStateEntry {
const identity = stateIdentity(input.url, input.rawVomText);
const existingId = this.#idByIdentity.get(identity);
if (existingId) {
const existing = this.#entriesById.get(existingId)!;
if (!existing.title && input.title) existing.title = input.title;
if (input.truncated) existing.truncated = true;
return existing;
}
const entry: RecordedStateEntry = {
id: `s${this.#nextId}`,
url: input.url,
...(input.title ? { title: input.title } : {}),
rawVomText: input.rawVomText,
truncated: input.truncated ?? false,
stepsHere: [],
};
this.#nextId += 1;
this.#entriesById.set(entry.id, entry);
this.#idByIdentity.set(identity, entry.id);
return entry;
}
values(): RecordedStateEntry[] {
return [...this.#entriesById.values()];
}
markStep(stateId: string, draftId: number): void {
const entry = this.#entriesById.get(stateId);
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
}
}
apps/extension/src/lib/recording-step-buffer.ts → apps/extension/src/lib/recording/step-buffer.ts
+17 -17
View File
@@ -1,8 +1,8 @@
import type { DraftTraceStep } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
import type { RecordStepPayload } from "../record-bridge";
import type { RecordingDraftStep } from "./types";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
steps: RecordingDraftStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
@@ -10,33 +10,33 @@ export interface RecordingStepBuffer {
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
const pageUrl = payload.page_url;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
captureTarget: payload.target,
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "hover":
return payload.target
? {
op: "hover",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
captureTarget: payload.target,
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
op: "fill",
target: payload.target,
captureTarget: payload.target,
value: payload.value ?? "",
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "press":
@@ -44,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
? {
op: "press",
key: payload.key,
...(payload.target ? { target: payload.target } : {}),
...(payload.target ? { captureTarget: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
target: payload.target,
captureTarget: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "navigate":
@@ -69,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select") {
buffer.steps[i] = { ...step, navigated_to: url };
buffer.steps[i] = { ...step, navigatedTo: url };
return true;
}
break;
@@ -96,7 +96,7 @@ export function observeRecordedNavigation(
buffer.steps.push({
op: "navigate",
url,
page_url: url,
pageUrl: url,
});
}
return;
@@ -107,7 +107,7 @@ export function observeRecordedNavigation(
buffer.steps.push({
op: "navigate",
url,
page_url: url,
pageUrl: url,
});
}
apps/extension/src/lib/recording/target-matcher.ts
+76
View File
@@ -0,0 +1,76 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { TargetDescriptorV3 } from "@/transport/types";
import type { CaptureTargetDescriptor } from "../describe-target";
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
import type { TargetGeometry, TargetMatchHint } from "./types";
const MATCH_TOLERANCE_PX = 2;
function close(a: number, b: number): boolean {
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
}
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
return candidate.node.rect !== null && rectMatches(geometry.rect, candidate.node.rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
return {
...(fallback?.role ? { role: fallback.role } : {}),
...(fallback?.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
function normalized(value: string | undefined): string {
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
}
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
const role = normalized(fallback?.role);
const name = normalized(fallback?.name);
if (!role && !name) return false;
if (role && normalized(ref.role) !== role) return false;
if (name && normalized(ref.name) !== name) return false;
return true;
}
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
return {
ref: ref.ref,
...(ref.role ? { role: ref.role } : {}),
...(ref.name ? { name: ref.name } : {}),
...(ref.ctx ? { ctx: ref.ctx } : {}),
};
}
export function matchObservationTarget(input: {
observation: RegisteredObservation;
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
);
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
return unmatchedTarget(input.fallback);
}
const semanticMatches = input.observation.index
.refs(frameId)
.filter((ref) => matchesSemantics(ref, input.fallback));
return semanticMatches.length === 1
? descriptor(semanticMatches[0]!)
: unmatchedTarget(input.fallback);
}
apps/extension/src/lib/recording/trace-builder-v3.ts
+87
View File
@@ -0,0 +1,87 @@
import {
type StepV3,
type StopReason,
TRACE_VERSION_V3,
type TraceStateV3,
type TraceV3,
VOM_FORMAT_VERSION,
} from "@/transport/types";
import { resolveDraftStartUrl } from "./draft-policy";
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
import { formatTraceStateBody } from "./trace-state-body";
import type { RecordingDraftStep, StepAnnotation } from "./types";
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
const entries = registry.values();
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
return entries.filter((entry) => referenced.has(entry.id));
}
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
return [
...new Set(
draftIds.flatMap((id) => {
const stepId = stepIdByDraftId.get(id);
return stepId === undefined ? [] : [stepId];
}),
),
].sort((a, b) => a - b);
}
export function buildTraceV3(input: {
registry: RecordingStateRegistry;
drafts: RecordingDraftStep[];
annotations?: StepAnnotation[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts);
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
for (const annotation of input.annotations ?? []) {
const bucket = annotationsByState.get(annotation.stateId) ?? [];
bucket.push(annotation);
annotationsByState.set(annotation.stateId, bucket);
}
const steps = reduced.steps.map((step) => ({
...step,
state: publishedId.get(step.state) ?? step.state,
result: { state: publishedId.get(step.result.state) ?? step.result.state },
}));
const states: TraceStateV3[] = entries.map((entry) => {
const id = publishedId.get(entry.id) ?? entry.id;
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body: formatTraceStateBody({
stateId: id,
url: entry.url,
title: entry.title,
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
vomText: entry.rawVomText,
annotations: annotationsByState.get(entry.id) ?? [],
stepIdByDraftId: reduced.stepIdByDraftId,
}),
...(entry.truncated ? { truncated: true } : {}),
};
});
return {
version: TRACE_VERSION_V3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
states,
steps,
};
}
apps/extension/src/lib/trace-reducer.ts → apps/extension/src/lib/recording/trace-reducer-v2.ts
+51 -56
View File
@@ -1,30 +1,10 @@
import type { DraftTraceStep, PageRefV2, SelectedOptionV2, StepV2 } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
// Drop bare character typing — FillSession already records the value.
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
import type { RecordingDraftStep } from "./types";
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
const out: RecordingDraftStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
if (step.pageUrl) urls.push(step.pageUrl);
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
}
const seen = new Set<string>();
const unique: string[] = [];
@@ -58,7 +38,7 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
}
function buildPageRegistry(
steps: DraftTraceStep[],
steps: RecordingDraftStep[],
startUrl?: string,
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
@@ -81,9 +61,9 @@ function pageIdFor(
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.pageUrl ?? step.url;
if (step.pageUrl) return step.pageUrl;
return fallbackUrl;
}
@@ -110,7 +90,7 @@ function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
}
function toV2Step(
step: DraftTraceStep,
step: RecordingDraftStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
@@ -129,28 +109,31 @@ function toV2Step(
to: step.url,
};
case "click":
if (!step.captureTarget) return null;
return withEffect(
{
op: "click",
id,
page,
target: step.target,
target: step.captureTarget,
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "hover":
if (!step.captureTarget) return null;
return {
op: "hover",
id,
page,
target: step.target,
target: step.captureTarget,
};
case "fill":
if (!step.captureTarget) return null;
return {
op: "fill",
id,
page,
target: step.target,
target: step.captureTarget,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
@@ -161,26 +144,29 @@ function toV2Step(
id,
page,
key: step.key,
...(step.target ? { target: step.target } : {}),
...(step.captureTarget ? { target: step.captureTarget } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "select":
if (!step.captureTarget) return null;
return withEffect(
{
op: "select",
id,
page,
target: step.target,
target: step.captureTarget,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "scroll":
return null;
}
}
export interface ReducedTrace {
interface ReducedTrace {
pages: PageRefV2[];
steps: StepV2[];
}
@@ -189,7 +175,7 @@ export interface ReducedTrace {
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
*/
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: StepV2[] = [];
@@ -197,8 +183,8 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
else if (draft.pageUrl) lastUrl = draft.pageUrl;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
@@ -207,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
return { pages, steps: out };
}
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
function resolveTraceStartUrl(
drafts: RecordingDraftStep[],
startUrl?: string,
pages?: PageRefV2[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
return step.op === "navigate";
});
if (navigate) return navigate.url;
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return pages?.[0]?.url ?? "about:blank";
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
}
export function buildTraceV2(input: {
steps: RecordingDraftStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}): TraceV2 {
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
pages,
steps,
};
}
apps/extension/src/lib/recording/trace-reducer-v3.ts
+129
View File
@@ -0,0 +1,129 @@
import type { NavigationCause, StepV3 } from "@/transport/types";
import { shouldIncludeDraft } from "./draft-policy";
import { unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep } from "./types";
interface CollapsedDraft {
draft: RecordingDraftStep;
draftIds: number[];
}
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
}
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
const output: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const previous = output[output.length - 1];
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
previous.draft = {
...previous.draft,
url: step.url,
postStateId: step.postStateId ?? previous.draft.postStateId,
};
previous.draftIds.push(index + 1);
return;
}
output.push({ draft: { ...step }, draftIds: [index + 1] });
});
return output;
}
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
if (step.cause) return step.cause;
const qualifiers = step.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.includes("from_address_bar")) return "user_typed";
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
}
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
const common = { id, state, result: { state: resultState } };
switch (draft.op) {
case "navigate":
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
case "click":
return {
op: "click",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "hover":
return {
op: "hover",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
return {
op: "fill",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
value: draft.value,
commit: draft.commit ?? "blur",
...(draft.redacted ? { redacted: true } : {}),
};
case "press":
return {
op: "press",
...common,
key: draft.key,
...(draft.captureTarget || draft.matchedTarget
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
: {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return {
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
selection: selection(draft.values, draft.labels),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTraceV3 {
steps: StepV3[];
stepIdByDraftId: Map<number, number>;
}
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1);
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
}
return { steps: output, stepIdByDraftId };
}
apps/extension/src/lib/recording/trace-state-body.ts
+39
View File
@@ -0,0 +1,39 @@
import type { StepAnnotation } from "./types";
function annotationText(annotation: StepAnnotation, stepId: number): string {
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
}
export function formatTraceStateBody(input: {
stateId: string;
url: string;
title?: string;
stepIds: number[];
vomText: string;
annotations: StepAnnotation[];
stepIdByDraftId: Map<number, number>;
}): string {
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
lines.push(`url: ${JSON.stringify(input.url)}`);
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
lines.push("---");
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
for (const annotation of input.annotations) {
const stepId = input.stepIdByDraftId.get(annotation.draftId);
if (stepId === undefined) continue;
const bucket = byLine.get(annotation.line) ?? [];
bucket.push({ annotation, stepId });
byLine.set(annotation.line, bucket);
}
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
let line = bodyLine;
for (const item of byLine.get(lineIndex) ?? []) {
line += annotationText(item.annotation, item.stepId);
}
lines.push(line);
});
return `${lines.join("\n")}\n`;
}
apps/extension/src/lib/recording/types.ts
+83
View File
@@ -0,0 +1,83 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type {
FillCommit,
KeyModifier,
NavigationCause,
StepV3,
TargetDescriptorV3,
} from "@/transport/types";
export interface TargetGeometry {
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
rect: { x: number; y: number; w: number; h: number };
tag: string;
}
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means the current top frame, never an unrestricted frame search. */
frameId?: string;
}
export interface StepAnnotation {
draftId: number;
op: StepV3["op"];
line: number;
stateId: string;
detail?: string;
}
interface DraftStateLink {
pageUrl?: string;
preStateId?: string;
postStateId?: string;
}
interface DraftTarget {
captureTarget?: CaptureTargetDescriptor;
targetHint?: TargetMatchHint;
matchedTarget?: TargetDescriptorV3;
}
interface DraftNavigationEffect {
navigatedTo?: string;
}
export type RecordingDraftStep =
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
| ({ op: "hover" } & DraftStateLink & DraftTarget)
| ({
op: "fill";
value: string;
commit?: FillCommit;
redacted?: boolean;
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "press";
key: string;
modifiers?: KeyModifier[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "select";
values: string[];
labels?: string[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({
op: "navigate";
url: string;
cause?: NavigationCause;
transitionType?: string;
transitionQualifiers?: string[];
} & DraftStateLink);
export type TargetedRecordingDraft = Extract<
RecordingDraftStep,
{ op: "click" | "hover" | "fill" | "press" | "select" }
>;
apps/extension/src/lib/trace-reducer-v2.ts
-301
View File
@@ -1,301 +0,0 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type { DraftTraceStep, KeyModifier } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export interface TargetDescriptorV2 {
role?: string;
name?: string;
tag: string;
name_attr?: string;
placeholder?: string;
nearby_label?: string;
}
export interface PageRef {
id: string;
url: string;
title?: string;
}
export interface SelectedOptionV2 {
value: string;
label?: string;
}
export interface StepEffectV2 {
navigated_to: string;
}
export type StepV2 =
| { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 }
| { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 }
| { op: "hover"; id: number; page: string; target: TargetDescriptorV2 }
| {
op: "fill";
id: number;
page: string;
target: TargetDescriptorV2;
value: string;
redacted?: boolean;
effect?: StepEffectV2;
}
| {
op: "select";
id: number;
page: string;
target: TargetDescriptorV2;
selection: SelectedOptionV2[];
effect?: StepEffectV2;
}
| {
op: "press";
id: number;
page: string;
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptorV2;
effect?: StepEffectV2;
};
export interface TraceV2 {
recorded_at: string;
started_at?: string;
purpose?: string;
entry: { start_url: string };
pages: PageRef[];
steps: StepV2[];
}
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "scroll") return false;
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
out[out.length - 1] = step;
continue;
}
out.push(step);
}
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
if (step.op === "navigate") {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
}
const seen = new Set<string>();
const unique: string[] = [];
for (const url of urls) {
if (!url || seen.has(url)) continue;
seen.add(url);
unique.push(url);
}
return unique;
}
function buildPageRegistry(
steps: DraftTraceStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
const id = `p${index + 1}`;
urlToId.set(url, id);
return { id, url };
});
return { pages, urlToId };
}
function pageIdFor(
url: string | undefined,
urlToId: Map<string, string>,
fallbackUrl?: string,
): string {
if (url && urlToId.has(url)) return urlToId.get(url)!;
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): StepEffectV2 | undefined {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 {
return {
tag: capture?.tag ?? "unknown",
...(capture?.role ? { role: capture.role } : {}),
...(capture?.name ? { name: capture.name } : {}),
...(capture?.name_attr ? { name_attr: capture.name_attr } : {}),
...(capture?.placeholder ? { placeholder: capture.placeholder } : {}),
...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}),
};
}
function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined {
if (!("target" in step) && !("captureTarget" in step)) return undefined;
const capture = "captureTarget" in step ? step.captureTarget : undefined;
if (capture) return captureTargetToV2Target(capture);
if ("target" in step && step.target && "tag" in (step.target as object)) {
const legacy = step.target as TargetDescriptorV2 & { tag?: string };
if (legacy.tag) return legacy;
}
return undefined;
}
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function toV2Step(
step: DraftTraceStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): StepV2 | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
const effect =
"navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined;
switch (step.op) {
case "navigate":
return {
op: "navigate",
id,
page: pageIdFor(step.url, urlToId, fallbackUrl),
to: step.url,
};
case "click": {
const target = targetForDraft(step);
if (!target) return null;
return { op: "click", id, page, target, ...(effect ? { effect } : {}) };
}
case "hover": {
const target = targetForDraft(step);
if (!target) return null;
return { op: "hover", id, page, target };
}
case "fill": {
const target = targetForDraft(step);
if (!target) return null;
return {
op: "fill",
id,
page,
target,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
}
case "press":
return {
op: "press",
id,
page,
key: step.key,
...(step.target ? { target: targetForDraft(step) } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
...(effect ? { effect } : {}),
};
case "select": {
const target = targetForDraft(step);
if (!target) return null;
return {
op: "select",
id,
page,
target,
selection: toSelection(step.values, step.labels),
...(effect ? { effect } : {}),
};
}
case "scroll":
return null;
}
}
export interface BuildTraceV2Input {
steps: DraftTraceStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}
export function buildTraceV2(input: BuildTraceV2Input): TraceV2 {
const collapsed = collapseNavigations(input.steps);
const startUrl =
input.startUrl ??
collapsed.find(
(step): step is Extract<DraftTraceStep, { op: "navigate" }> => step.op === "navigate",
)?.url ??
collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ??
"about:blank";
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: StepV2[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
id += 1;
}
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: startUrl },
pages,
steps: out,
};
}
apps/extension/src/tools/record.ts
+9 -13
View File
@@ -17,11 +17,11 @@ import {
type RecordStartMessage,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types";
import type { SessionManager } from "@/session-manager/manager";
import type {
DraftTraceStep,
RecordAwaitParams,
RecordAwaitResult,
RecordStartParams,
@@ -47,7 +47,7 @@ interface ActiveRecording {
agentWindowId: number;
startUrl?: string;
purpose?: string;
steps: DraftTraceStep[];
steps: RecordingDraftStep[];
startedAt: string;
startedAtMs: number;
finishPromise: Promise<TraceV2>;
@@ -155,16 +155,12 @@ async function sendRecordStartWithAck(
}
function buildTrace(recording: ActiveRecording): TraceV2 {
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
return {
return buildTraceV2({
steps: recording.steps,
startedAt: recording.startedAt,
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
...(recording.purpose ? { purpose: recording.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: recording.startedAt,
entry: { start_url: startUrl },
pages,
steps,
};
});
}
export interface RecordDeps {
apps/extension/src/transport/types.ts
-42
View File
@@ -744,48 +744,6 @@ export interface StepCommonV2 {
effect?: StepEffectV2;
}
/** Capture/buffer draft before v2 reduction. */
export type DraftTraceStep =
| {
op: "click";
target: TargetDescriptorV2;
navigated_to?: string;
page_url?: string;
}
| {
op: "hover";
target: TargetDescriptorV2;
page_url?: string;
}
| {
op: "fill";
target: TargetDescriptorV2;
value: string;
redacted?: boolean;
page_url?: string;
}
| {
op: "press";
key: string;
target?: TargetDescriptorV2;
modifiers?: KeyModifier[];
navigated_to?: string;
page_url?: string;
}
| {
op: "select";
target: TargetDescriptorV2;
values: string[];
labels?: string[];
navigated_to?: string;
page_url?: string;
}
| {
op: "navigate";
url: string;
page_url?: string;
};
/** Exported record-only step (trace v2). */
export type StepV2 =
| ({ op: "navigate" } & StepCommonV2 & { to: string })