mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6300eda0aa | ||
|
|
87cdcfe5ca | ||
|
|
21476c3884 |
@@ -1,9 +1,9 @@
|
|||||||
import {
|
import {
|
||||||
|
type CaptureTargetDescriptor,
|
||||||
describeEventTarget,
|
describeEventTarget,
|
||||||
describeTarget,
|
describeTarget,
|
||||||
resolveClickableElement,
|
resolveClickableElement,
|
||||||
resolveHoverElement,
|
resolveHoverElement,
|
||||||
type TargetDescriptor,
|
|
||||||
} from "@/lib/describe-target";
|
} from "@/lib/describe-target";
|
||||||
import {
|
import {
|
||||||
evaluateHoverTrigger,
|
evaluateHoverTrigger,
|
||||||
@@ -27,7 +27,7 @@ import {
|
|||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||||
import {
|
import {
|
||||||
closestHoverSurfaceCandidate,
|
closestHoverSurfaceCandidate,
|
||||||
collectHoverSurfaceStates,
|
collectHoverSurfaceStates,
|
||||||
@@ -65,14 +65,14 @@ export interface RecordCaptureController {
|
|||||||
|
|
||||||
interface FillSession {
|
interface FillSession {
|
||||||
element: FillableElement;
|
element: FillableElement;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
baselineValue: string;
|
baselineValue: string;
|
||||||
lastValue: string;
|
lastValue: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
element: Element;
|
element: Element;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
recordedAt: number;
|
recordedAt: number;
|
||||||
score: number;
|
score: number;
|
||||||
eligible: boolean;
|
eligible: boolean;
|
||||||
@@ -232,7 +232,10 @@ function collectHoverTriggerLabelText(root: Element): string {
|
|||||||
return normalizeLabelText(text);
|
return normalizeLabelText(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
function compactHoverTargetName(
|
||||||
|
el: Element,
|
||||||
|
desc: CaptureTargetDescriptor,
|
||||||
|
): CaptureTargetDescriptor {
|
||||||
if (!desc.name) return desc;
|
if (!desc.name) return desc;
|
||||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
const compactName = collectHoverTriggerLabelText(el);
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
@@ -250,7 +253,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
return !target.role && !target.name && target.tag === "div";
|
return !target.role && !target.name && target.tag === "div";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,9 +264,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
|||||||
|
|
||||||
function normalizeHoverTarget(
|
function normalizeHoverTarget(
|
||||||
el: Element,
|
el: Element,
|
||||||
desc: TargetDescriptor,
|
desc: CaptureTargetDescriptor,
|
||||||
decision: HoverTriggerDecision,
|
decision: HoverTriggerDecision,
|
||||||
): TargetDescriptor {
|
): CaptureTargetDescriptor {
|
||||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
return { ...desc, name: "image" };
|
return { ...desc, name: "image" };
|
||||||
}
|
}
|
||||||
@@ -277,7 +280,7 @@ function normalizeHoverTarget(
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||||
if (!(el instanceof HTMLElement)) return null;
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
const style = hoverTriggerStyle(el);
|
const style = hoverTriggerStyle(el);
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { waitForDocumentSettled } from "../recording/document-settle";
|
||||||
|
|
||||||
|
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||||
|
const send = vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||||
|
send,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("waitForDocumentSettled", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels before probing a superseded observation", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const settled = waitForDocumentSettled(
|
||||||
|
cdp,
|
||||||
|
{ target: { tabId: 7 } },
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
controller.abort();
|
||||||
|
await vi.advanceTimersByTimeAsync(60);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("cancelled");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes a same-process iframe through its isolated execution context", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, {
|
||||||
|
target: { tabId: 7 },
|
||||||
|
frameId: "child-frame",
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
expect.objectContaining({ frameId: "child-frame" }),
|
||||||
|
);
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 91 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes an OOPIF probe through its CDP target session", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 27 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||||
|
const session = new RecordingObservationSession({ redactValues });
|
||||||
|
const state = session.registry.register({
|
||||||
|
url: URL,
|
||||||
|
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||||
|
});
|
||||||
|
session.cursor.lastSettled = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "input",
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
}),
|
||||||
|
url: URL,
|
||||||
|
};
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
const session = sessionWithInput(redactValues);
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
||||||
|
value,
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
draft.postStateId = draft.preStateId;
|
||||||
|
return buildTraceV3({
|
||||||
|
registry: session.registry,
|
||||||
|
drafts: [draft],
|
||||||
|
annotations: session.annotations,
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
}).states[0]!.body;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
it("omits a fill literal when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const body = finalizedFillBody(secret, true);
|
||||||
|
expect(body).toContain("step 1: fill");
|
||||||
|
expect(body).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps ordinary fill details", () => {
|
||||||
|
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({
|
||||||
|
url: "https://example.com/a\nb",
|
||||||
|
title: "hello\nworld",
|
||||||
|
rawVomText: "@vom 1",
|
||||||
|
});
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
||||||
|
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("recording state ownership", () => {
|
||||||
|
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||||
|
const first = new RecordingStateRegistry();
|
||||||
|
const second = new RecordingStateRegistry();
|
||||||
|
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||||
|
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||||
|
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
registry.register({ url: URL, rawVomText: "same" });
|
||||||
|
const state = registry.register({
|
||||||
|
url: URL,
|
||||||
|
title: "Login",
|
||||||
|
rawVomText: "same",
|
||||||
|
truncated: true,
|
||||||
|
});
|
||||||
|
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("draft binding", () => {
|
||||||
|
it("keeps capture semantics when no observation exists", () => {
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not bind an unmatched new action to a stale observation", () => {
|
||||||
|
const session = sessionWithInput();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 2, true);
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "Confirm",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { SettleController } from "../recording/settle-controller";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const OBSERVATION: RegisteredObservation = {
|
||||||
|
stateId: "s-next",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
|
url: "https://example.com/next",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("SettleController", () => {
|
||||||
|
afterEach(() => vi.useRealTimers());
|
||||||
|
|
||||||
|
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
let firstSignal: AbortSignal | undefined;
|
||||||
|
vi.spyOn(session, "capture")
|
||||||
|
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
||||||
|
firstSignal = signal;
|
||||||
|
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce(OBSERVATION);
|
||||||
|
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
controller.schedule(drafts, 0);
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
drafts.push({
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", name: "Second" },
|
||||||
|
preStateId: "s-next",
|
||||||
|
});
|
||||||
|
controller.schedule(drafts, 1);
|
||||||
|
expect(firstSignal?.aborted).toBe(true);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flush();
|
||||||
|
expect(drafts[1]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||||
|
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||||
|
resolveFirstTab = resolve;
|
||||||
|
});
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi
|
||||||
|
.fn()
|
||||||
|
.mockImplementationOnce(() => firstTab)
|
||||||
|
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
session.cursor.lastSettled = OBSERVATION;
|
||||||
|
const drafts: RecordingDraftStep[] = [];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
||||||
|
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flushRedirects();
|
||||||
|
|
||||||
|
expect(drafts).toEqual([]);
|
||||||
|
expect(controller.hasPending).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("settles an action in its own OOPIF document scope", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
|
||||||
|
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
await controller.flush();
|
||||||
|
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 39 }),
|
||||||
|
);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
});
|
||||||
+8
-8
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
||||||
|
|
||||||
describe("recording-step-buffer", () => {
|
describe("recording-step-buffer", () => {
|
||||||
it("stores semantic click without summary", () => {
|
it("stores semantic click without summary", () => {
|
||||||
@@ -12,7 +12,7 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(buffer.pendingNavigation).toBe(true);
|
expect(buffer.pendingNavigation).toBe(true);
|
||||||
@@ -23,7 +23,7 @@ describe("recording-step-buffer", () => {
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
op: "click" as const,
|
op: "click" as const,
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
currentUrl: "https://example.com/a",
|
currentUrl: "https://example.com/a",
|
||||||
@@ -34,8 +34,8 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
navigated_to: "https://example.com/b",
|
navigatedTo: "https://example.com/b",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||||
@@ -46,7 +46,7 @@ describe("recording-step-buffer", () => {
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
op: "select" as const,
|
op: "select" as const,
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -58,9 +58,9 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { matchObservationTarget } from "../recording/target-matcher";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
frameId,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function observation(
|
||||||
|
matchNodes: CaptureVomMatchNode[],
|
||||||
|
refs: RenderedRef[],
|
||||||
|
): RegisteredObservation {
|
||||||
|
return {
|
||||||
|
stateId: "s1",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
||||||
|
url: "https://example.com",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchObservationTarget", () => {
|
||||||
|
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
||||||
|
const geometry = node(42, "child");
|
||||||
|
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
||||||
|
const ref: RenderedRef = {
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
line: 1,
|
||||||
|
};
|
||||||
|
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
||||||
|
|
||||||
|
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a unique node using canonical top-level viewport geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "left"), node(42, "right")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: "right",
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restricts a missing frame hint to the root frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "child")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched for ambiguous geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(41, "root"), node(42, "child")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { frameId: "child" },
|
||||||
|
fallback: { tag: "button", role: "button", name: "保存" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", role: "button", name: "取消" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
});
|
||||||
+37
-27
@@ -1,6 +1,15 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
import { shouldRecordPress } from "../recording/draft-policy";
|
||||||
|
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
||||||
|
|
||||||
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
||||||
|
return buildTraceV2({
|
||||||
|
steps,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
...(startUrl ? { startUrl } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
describe("shouldRecordPress", () => {
|
||||||
it("keeps Enter and Escape", () => {
|
it("keeps Enter and Escape", () => {
|
||||||
@@ -19,35 +28,35 @@ describe("shouldRecordPress", () => {
|
|||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
describe("reduceTraceSteps", () => {
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
it("builds steps with pages dictionary and page id references", () => {
|
||||||
const drafts: DraftTraceStep[] = [
|
const drafts: RecordingDraftStep[] = [
|
||||||
{
|
{
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
url: "https://example.com/search?q=hello&utm_source=x",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
value: "browser skill",
|
value: "browser skill",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "Enter",
|
key: "Enter",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
navigated_to: "https://example.com/results/42",
|
navigatedTo: "https://example.com/results/42",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
navigated_to: "https://example.com/p/99",
|
navigatedTo: "https://example.com/p/99",
|
||||||
page_url: "https://example.com/results/42",
|
pageUrl: "https://example.com/results/42",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "a",
|
key: "a",
|
||||||
page_url: "https://example.com/p/99",
|
pageUrl: "https://example.com/p/99",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -109,9 +118,9 @@ describe("reduceTraceSteps", () => {
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "Search query" },
|
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
||||||
value: "",
|
value: "",
|
||||||
page_url: "https://example.com/search",
|
pageUrl: "https://example.com/search",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/search",
|
"https://example.com/search",
|
||||||
@@ -125,16 +134,16 @@ describe("reduceTraceSteps", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
const { pages, steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
labels: ["技术"],
|
labels: ["技术"],
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
page_url: "https://example.com/list",
|
pageUrl: "https://example.com/list",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/list",
|
"https://example.com/list",
|
||||||
@@ -156,13 +165,13 @@ describe("reduceTraceSteps", () => {
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { tag: "span", role: "button", name: "Account" },
|
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||||
page_url: "https://example.com/app",
|
pageUrl: "https://example.com/app",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "a", role: "link", name: "Profile" },
|
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||||
page_url: "https://example.com/app",
|
pageUrl: "https://example.com/app",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/app",
|
"https://example.com/app",
|
||||||
@@ -177,10 +186,11 @@ describe("reduceTraceSteps", () => {
|
|||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
buildTraceV2({
|
||||||
[{ op: "navigate", url: "https://example.com/other" }],
|
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
||||||
"https://example.com/start",
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
),
|
startUrl: "https://example.com/start",
|
||||||
|
}).entry.start_url,
|
||||||
).toBe("https://example.com/start");
|
).toBe("https://example.com/start");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
describe("trace reducer v3", () => {
|
||||||
|
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/final",
|
||||||
|
transitionQualifiers: ["server_redirect"],
|
||||||
|
preStateId: "s2",
|
||||||
|
postStateId: "s3",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const before = structuredClone(drafts);
|
||||||
|
const reduced = reduceTraceStepsV3(drafts);
|
||||||
|
|
||||||
|
expect(drafts).toEqual(before);
|
||||||
|
expect(reduced.steps).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
id: 1,
|
||||||
|
op: "navigate",
|
||||||
|
state: "s1",
|
||||||
|
to: "https://example.com/final",
|
||||||
|
result: { state: "s3" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds the wire model from protocol constants", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||||
|
preStateId: state.id,
|
||||||
|
postStateId: state.id,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(trace.version).toBe(TRACE_VERSION_V3);
|
||||||
|
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
||||||
|
expect(trace.steps[0]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { role: "button", name: "Save", unmatched: true },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,12 +1,13 @@
|
|||||||
/**
|
/**
|
||||||
* Build a semantic TargetDescriptor for an interacted element.
|
* Build a semantic capture descriptor for an interacted element.
|
||||||
*
|
*
|
||||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||||
* on screen (usually a short visible name). Tag-only noise like
|
* on screen (usually a short visible name). Tag-only noise like
|
||||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
/** Content-script capture descriptor before VOM geometric matching. */
|
||||||
|
export interface CaptureTargetDescriptor {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
|||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
* Recording “点击div” with no name fails this bar.
|
* Recording “点击div” with no name fails this bar.
|
||||||
*/
|
*/
|
||||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
const name = target.name?.trim();
|
const name = target.name?.trim();
|
||||||
if (name && isActionableLabel(name)) return true;
|
if (name && isActionableLabel(name)) return true;
|
||||||
if (
|
if (
|
||||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeTarget(el: Element): TargetDescriptor {
|
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||||
const tag = el.tagName.toLowerCase();
|
const tag = el.tagName.toLowerCase();
|
||||||
const role = inferRole(el);
|
const role = inferRole(el);
|
||||||
const name = accessibleName(el);
|
const name = accessibleName(el);
|
||||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const clickable = resolveClickableElement(target);
|
const clickable = resolveClickableElement(target);
|
||||||
if (!clickable) return null;
|
if (!clickable) return null;
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
* service worker and a tab's content script.
|
* service worker and a tab's content script.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -42,7 +42,7 @@ export interface RecordStartMessage {
|
|||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||||
target?: TargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
||||||
|
|
||||||
|
const SETTLE_MIN_MS = 150;
|
||||||
|
const SETTLE_QUIET_MS = 250;
|
||||||
|
const SETTLE_MAX_MS = 2_000;
|
||||||
|
const SETTLE_POLL_MS = 60;
|
||||||
|
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
||||||
|
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export interface DocumentSettleScope {
|
||||||
|
target: CdpTarget;
|
||||||
|
/** Omit only for the target's root document. */
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProbeContext {
|
||||||
|
executionContextId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executionContextForScope(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
): Promise<number | undefined> {
|
||||||
|
if (!scope.frameId) return undefined;
|
||||||
|
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
{
|
||||||
|
frameId: scope.frameId,
|
||||||
|
worldName: SETTLE_WORLD_NAME,
|
||||||
|
grantUniveralAccess: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return result.executionContextId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
context: ProbeContext,
|
||||||
|
): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
if (scope.frameId && context.executionContextId === undefined) {
|
||||||
|
context.executionContextId = await executionContextForScope(cdp, scope);
|
||||||
|
if (context.executionContextId === undefined) return null;
|
||||||
|
}
|
||||||
|
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
...(context.executionContextId !== undefined
|
||||||
|
? { contextId: context.executionContextId }
|
||||||
|
: {}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
context.executionContextId = undefined;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForDocumentSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
options: { signal?: AbortSignal } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
const context: ProbeContext = {};
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS, options.signal);
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, scope, context);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe || probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
|
|
||||||
|
export function shouldRecordPress(
|
||||||
|
key: string,
|
||||||
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||||
|
): boolean {
|
||||||
|
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||||
|
const mods = modifiers ?? [];
|
||||||
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
||||||
|
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveDraftStartUrl(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
explicitStartUrl?: string,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): string {
|
||||||
|
if (explicitStartUrl) return explicitStartUrl;
|
||||||
|
const navigation = drafts.find(
|
||||||
|
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
||||||
|
);
|
||||||
|
if (navigation) return navigation.url;
|
||||||
|
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import {
|
||||||
|
type CaptureVomMatchNode,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
|
||||||
|
export interface IndexedObservationNode {
|
||||||
|
frameId: string;
|
||||||
|
geometry: CaptureVomMatchNode;
|
||||||
|
ref?: RenderedRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CapturedRecordingObservation {
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RegisteredObservation {
|
||||||
|
stateId: string;
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||||
|
return `${frameId}:${backendNodeId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameTagKey(frameId: string, tag: string): string {
|
||||||
|
return `${frameId}:${tag.toLowerCase()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ObservationNodeIndex {
|
||||||
|
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||||
|
readonly #refById = new Map<string, RenderedRef>();
|
||||||
|
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||||
|
|
||||||
|
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
|
||||||
|
const refByNode = new Map<string, RenderedRef>();
|
||||||
|
for (const ref of input.refs) {
|
||||||
|
const frameId = ref.frameId ?? input.rootFrameId;
|
||||||
|
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
||||||
|
this.#refById.set(ref.ref, ref);
|
||||||
|
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
frameRefs.push(ref);
|
||||||
|
this.#refsByFrame.set(frameId, frameRefs);
|
||||||
|
}
|
||||||
|
for (const geometry of input.matchNodes) {
|
||||||
|
const { frameId } = geometry;
|
||||||
|
const entry = {
|
||||||
|
frameId,
|
||||||
|
geometry,
|
||||||
|
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
||||||
|
};
|
||||||
|
const key = frameTagKey(frameId, geometry.tag);
|
||||||
|
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
||||||
|
bucket.push(entry);
|
||||||
|
this.#nodesByFrameTag.set(key, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
||||||
|
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
ref(refId: string): RenderedRef | undefined {
|
||||||
|
return this.#refById.get(refId);
|
||||||
|
}
|
||||||
|
|
||||||
|
refs(frameId: string): readonly RenderedRef[] {
|
||||||
|
return this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureRecordingObservation(input: {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<CapturedRecordingObservation> {
|
||||||
|
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
||||||
|
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
||||||
|
maxTokens: input.maxTokens,
|
||||||
|
redactValues: input.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
signal: input.signal,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: new ObservationNodeIndex(captured),
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
vomText: captured.text,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
||||||
|
import { RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
||||||
|
|
||||||
|
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
||||||
|
const MIN_CAPTURE_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
export interface TabObservationCursor {
|
||||||
|
lastSettled: RegisteredObservation | null;
|
||||||
|
lastCaptureAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||||
|
return draft.op !== "navigate" && draft.op !== "scroll";
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingObservationSession {
|
||||||
|
readonly registry: RecordingStateRegistry;
|
||||||
|
readonly cursor: TabObservationCursor;
|
||||||
|
readonly annotations: StepAnnotation[];
|
||||||
|
readonly #maxTokens: number;
|
||||||
|
readonly #redactValues: boolean;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
options: {
|
||||||
|
registry?: RecordingStateRegistry;
|
||||||
|
cursor?: TabObservationCursor;
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
maxTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
this.registry = options.registry ?? new RecordingStateRegistry();
|
||||||
|
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
||||||
|
this.annotations = options.annotations ?? [];
|
||||||
|
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
||||||
|
this.#redactValues = options.redactValues ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async capture(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RegisteredObservation> {
|
||||||
|
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
||||||
|
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
||||||
|
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
|
const captured = await captureRecordingObservation({
|
||||||
|
cdp,
|
||||||
|
tabsApi,
|
||||||
|
tabId,
|
||||||
|
maxTokens: this.#maxTokens,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const state = this.registry.register({
|
||||||
|
url: captured.url,
|
||||||
|
title: captured.title,
|
||||||
|
rawVomText: captured.vomText,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
});
|
||||||
|
const observation: RegisteredObservation = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: captured.index,
|
||||||
|
url: captured.url,
|
||||||
|
};
|
||||||
|
this.cursor.lastSettled = observation;
|
||||||
|
this.cursor.lastCaptureAt = Date.now();
|
||||||
|
return observation;
|
||||||
|
}
|
||||||
|
|
||||||
|
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
||||||
|
const observation = this.cursor.lastSettled;
|
||||||
|
if (isTargeted(draft)) {
|
||||||
|
draft.matchedTarget = observation
|
||||||
|
? matchObservationTarget({
|
||||||
|
observation,
|
||||||
|
hint: draft.targetHint,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: unmatchedTarget(draft.captureTarget);
|
||||||
|
}
|
||||||
|
if (!observation) return;
|
||||||
|
|
||||||
|
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
||||||
|
if (previousActionPending && unmatched) return;
|
||||||
|
draft.preStateId = observation.stateId;
|
||||||
|
this.registry.markStep(observation.stateId, draftId);
|
||||||
|
|
||||||
|
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
||||||
|
const ref = observation.index.ref(draft.matchedTarget.ref);
|
||||||
|
if (!ref) return;
|
||||||
|
this.annotations.push({
|
||||||
|
draftId,
|
||||||
|
op: draft.op,
|
||||||
|
line: ref.line,
|
||||||
|
stateId: observation.stateId,
|
||||||
|
...(draft.op === "fill" && !this.#redactValues
|
||||||
|
? { detail: JSON.stringify(draft.value) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||||
|
import { RecordingObservationSession } from "./observation-session";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface PendingSettle {
|
||||||
|
abort: AbortController;
|
||||||
|
scope: DocumentSettleScope;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingRedirect {
|
||||||
|
url: string;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
(error as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
||||||
|
for (let index = 0; index < drafts.length - 1; index += 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
const next = drafts[index + 1];
|
||||||
|
if (draft && next && !draft.postStateId && next.preStateId) {
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SettleController {
|
||||||
|
readonly #session: RecordingObservationSession;
|
||||||
|
readonly #cdp: CdpRunner;
|
||||||
|
readonly #tabsApi: ChromeTabsApi;
|
||||||
|
readonly #tabId: number;
|
||||||
|
readonly #rootScope: DocumentSettleScope;
|
||||||
|
readonly #pending = new Map<number, PendingSettle>();
|
||||||
|
#queue = Promise.resolve();
|
||||||
|
#pendingRedirect: PendingRedirect | null = null;
|
||||||
|
#redirectQueue = Promise.resolve();
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
session: RecordingObservationSession;
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
}) {
|
||||||
|
this.#session = input.session;
|
||||||
|
this.#cdp = input.cdp;
|
||||||
|
this.#tabsApi = input.tabsApi;
|
||||||
|
this.#tabId = input.tabId;
|
||||||
|
this.#rootScope = { target: { tabId: input.tabId } };
|
||||||
|
}
|
||||||
|
|
||||||
|
get hasPending(): boolean {
|
||||||
|
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #captureWithRetry(signal?: AbortSignal) {
|
||||||
|
try {
|
||||||
|
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
||||||
|
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
schedule(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
scope: DocumentSettleScope = this.#rootScope,
|
||||||
|
): void {
|
||||||
|
if (scope.target.tabId !== this.#tabId) {
|
||||||
|
throw new Error(
|
||||||
|
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const landing = drafts[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of this.#pending) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.abort.abort();
|
||||||
|
this.#pending.delete(index);
|
||||||
|
if (landing && drafts[index] && !drafts[index].postStateId) {
|
||||||
|
drafts[index].postStateId = landing;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.#pending.get(draftIndex)?.abort.abort();
|
||||||
|
const pending = { abort: new AbortController(), scope };
|
||||||
|
this.#pending.set(draftIndex, pending);
|
||||||
|
const task = async () => {
|
||||||
|
if (pending.abort.signal.aborted) return;
|
||||||
|
try {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
const observation = await this.#captureWithRetry(pending.abort.signal);
|
||||||
|
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
||||||
|
drafts[draftIndex].postStateId = observation.stateId;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (!isAbortError(error)) {
|
||||||
|
console.warn(
|
||||||
|
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.#queue = this.#queue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
for (const pending of this.#pending.values()) pending.abort.abort();
|
||||||
|
this.#pending.clear();
|
||||||
|
this.clearRedirect();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#queue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#queue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
||||||
|
}
|
||||||
|
|
||||||
|
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
||||||
|
const trailing: RecordingDraftStep[] = [];
|
||||||
|
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
try {
|
||||||
|
const observation = await this.#captureWithRetry();
|
||||||
|
for (const draft of trailing) draft.postStateId = observation.stateId;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
clearRedirect(): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
const pending: PendingRedirect = { url, abort: new AbortController() };
|
||||||
|
this.#pendingRedirect = pending;
|
||||||
|
const task = () => this.#settleRedirect(drafts, pending);
|
||||||
|
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
||||||
|
|
||||||
|
let finalUrl = pending.url;
|
||||||
|
try {
|
||||||
|
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
||||||
|
} catch {
|
||||||
|
// The navigation event URL remains the best available destination.
|
||||||
|
}
|
||||||
|
if (this.#pendingRedirect !== pending) return;
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
if (
|
||||||
|
!finalUrl ||
|
||||||
|
finalUrl === "about:blank" ||
|
||||||
|
this.#session.cursor.lastSettled?.url === finalUrl
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const last = drafts[drafts.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
||||||
|
await this.flush();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
pageUrl: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
drafts.push(draft);
|
||||||
|
const draftIndex = drafts.length - 1;
|
||||||
|
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
||||||
|
this.schedule(drafts, draftIndex);
|
||||||
|
await this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flushRedirects(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#redirectQueue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#redirectQueue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
export interface RecordedStateEntry {
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
stepsHere: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function stateIdentity(url: string, body: string): string {
|
||||||
|
return `${url}\0${body}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingStateRegistry {
|
||||||
|
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
||||||
|
readonly #idByIdentity = new Map<string, string>();
|
||||||
|
#nextId = 1;
|
||||||
|
|
||||||
|
register(input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}): RecordedStateEntry {
|
||||||
|
const identity = stateIdentity(input.url, input.rawVomText);
|
||||||
|
const existingId = this.#idByIdentity.get(identity);
|
||||||
|
if (existingId) {
|
||||||
|
const existing = this.#entriesById.get(existingId)!;
|
||||||
|
if (!existing.title && input.title) existing.title = input.title;
|
||||||
|
if (input.truncated) existing.truncated = true;
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry: RecordedStateEntry = {
|
||||||
|
id: `s${this.#nextId}`,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
rawVomText: input.rawVomText,
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
stepsHere: [],
|
||||||
|
};
|
||||||
|
this.#nextId += 1;
|
||||||
|
this.#entriesById.set(entry.id, entry);
|
||||||
|
this.#idByIdentity.set(identity, entry.id);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
values(): RecordedStateEntry[] {
|
||||||
|
return [...this.#entriesById.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
markStep(stateId: string, draftId: number): void {
|
||||||
|
const entry = this.#entriesById.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
||||||
|
}
|
||||||
|
}
|
||||||
+17
-17
@@ -1,8 +1,8 @@
|
|||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { RecordStepPayload } from "../record-bridge";
|
||||||
import type { RecordStepPayload } from "./record-bridge";
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
export interface RecordingStepBuffer {
|
||||||
steps: DraftTraceStep[];
|
steps: RecordingDraftStep[];
|
||||||
currentUrl?: string;
|
currentUrl?: string;
|
||||||
pendingNavigation: boolean;
|
pendingNavigation: boolean;
|
||||||
pendingNavigationDeadline?: number;
|
pendingNavigationDeadline?: number;
|
||||||
@@ -10,33 +10,33 @@ export interface RecordingStepBuffer {
|
|||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||||
const pageUrl = payload.page_url;
|
const pageUrl = payload.page_url;
|
||||||
switch (payload.op) {
|
switch (payload.op) {
|
||||||
case "click":
|
case "click":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "click",
|
op: "click",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "hover":
|
case "hover":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "fill":
|
case "fill":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
value: payload.value ?? "",
|
value: payload.value ?? "",
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "press":
|
case "press":
|
||||||
@@ -44,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
? {
|
? {
|
||||||
op: "press",
|
op: "press",
|
||||||
key: payload.key,
|
key: payload.key,
|
||||||
...(payload.target ? { target: payload.target } : {}),
|
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "select":
|
case "select":
|
||||||
return payload.target && payload.values
|
return payload.target && payload.values
|
||||||
? {
|
? {
|
||||||
op: "select",
|
op: "select",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
values: payload.values,
|
values: payload.values,
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "navigate":
|
case "navigate":
|
||||||
@@ -69,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b
|
|||||||
const step = buffer.steps[i];
|
const step = buffer.steps[i];
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||||
buffer.steps[i] = { ...step, navigated_to: url };
|
buffer.steps[i] = { ...step, navigatedTo: url };
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
@@ -96,7 +96,7 @@ export function observeRecordedNavigation(
|
|||||||
buffer.steps.push({
|
buffer.steps.push({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url,
|
url,
|
||||||
page_url: url,
|
pageUrl: url,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -107,7 +107,7 @@ export function observeRecordedNavigation(
|
|||||||
buffer.steps.push({
|
buffer.steps.push({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url,
|
url,
|
||||||
page_url: url,
|
pageUrl: url,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { TargetDescriptorV3 } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "../describe-target";
|
||||||
|
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
||||||
|
import type { TargetGeometry, TargetMatchHint } from "./types";
|
||||||
|
|
||||||
|
const MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
function close(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
||||||
|
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
|
||||||
|
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
...(fallback?.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback?.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalized(value: string | undefined): string {
|
||||||
|
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
||||||
|
const role = normalized(fallback?.role);
|
||||||
|
const name = normalized(fallback?.name);
|
||||||
|
if (!role && !name) return false;
|
||||||
|
if (role && normalized(ref.role) !== role) return false;
|
||||||
|
if (name && normalized(ref.name) !== name) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
ref: ref.ref,
|
||||||
|
...(ref.role ? { role: ref.role } : {}),
|
||||||
|
...(ref.name ? { name: ref.name } : {}),
|
||||||
|
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function matchObservationTarget(input: {
|
||||||
|
observation: RegisteredObservation;
|
||||||
|
hint?: TargetMatchHint;
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}): TargetDescriptorV3 {
|
||||||
|
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||||
|
const geometry = input.hint?.geometry;
|
||||||
|
if (geometry) {
|
||||||
|
const matches = input.observation.index
|
||||||
|
.candidates(frameId, geometry.tag)
|
||||||
|
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
|
||||||
|
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||||
|
const semanticMatches = matches.filter(
|
||||||
|
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||||
|
);
|
||||||
|
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
||||||
|
return unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const semanticMatches = input.observation.index
|
||||||
|
.refs(frameId)
|
||||||
|
.filter((ref) => matchesSemantics(ref, input.fallback));
|
||||||
|
return semanticMatches.length === 1
|
||||||
|
? descriptor(semanticMatches[0]!)
|
||||||
|
: unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import {
|
||||||
|
type StepV3,
|
||||||
|
type StopReason,
|
||||||
|
TRACE_VERSION_V3,
|
||||||
|
type TraceStateV3,
|
||||||
|
type TraceV3,
|
||||||
|
VOM_FORMAT_VERSION,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl } from "./draft-policy";
|
||||||
|
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
||||||
|
import { formatTraceStateBody } from "./trace-state-body";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
||||||
|
const entries = registry.values();
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
return [
|
||||||
|
...new Set(
|
||||||
|
draftIds.flatMap((id) => {
|
||||||
|
const stepId = stepIdByDraftId.get(id);
|
||||||
|
return stepId === undefined ? [] : [stepId];
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
registry: RecordingStateRegistry;
|
||||||
|
drafts: RecordingDraftStep[];
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
}): TraceV3 {
|
||||||
|
const reduced = reduceTraceStepsV3(input.drafts);
|
||||||
|
const entries = publishedEntries(input.registry, reduced.steps);
|
||||||
|
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||||
|
for (const annotation of input.annotations ?? []) {
|
||||||
|
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
||||||
|
bucket.push(annotation);
|
||||||
|
annotationsByState.set(annotation.stateId, bucket);
|
||||||
|
}
|
||||||
|
const steps = reduced.steps.map((step) => ({
|
||||||
|
...step,
|
||||||
|
state: publishedId.get(step.state) ?? step.state,
|
||||||
|
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
||||||
|
}));
|
||||||
|
const states: TraceStateV3[] = entries.map((entry) => {
|
||||||
|
const id = publishedId.get(entry.id) ?? entry.id;
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body: formatTraceStateBody({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||||
|
vomText: entry.rawVomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||||
|
}),
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
version: TRACE_VERSION_V3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
+51
-56
@@ -1,30 +1,10 @@
|
|||||||
import type { DraftTraceStep, PageRefV2, SelectedOptionV2, StepV2 } from "@/transport/types";
|
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
import type { RecordingDraftStep } from "./types";
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
|
||||||
|
|
||||||
export function shouldRecordPress(
|
|
||||||
key: string,
|
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
|
||||||
): boolean {
|
|
||||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
|
||||||
const mods = modifiers ?? [];
|
|
||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
|
||||||
// Drop bare character typing — FillSession already records the value.
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
|
||||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
/** Collapse consecutive navigations to the last hop. */
|
||||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
||||||
const out: DraftTraceStep[] = [];
|
const out: RecordingDraftStep[] = [];
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
const prev = out[out.length - 1];
|
const prev = out[out.length - 1];
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||||
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
||||||
const urls: string[] = [];
|
const urls: string[] = [];
|
||||||
if (startUrl) urls.push(startUrl);
|
if (startUrl) urls.push(startUrl);
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
urls.push(step.url);
|
urls.push(step.url);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
if (step.pageUrl) urls.push(step.pageUrl);
|
||||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
||||||
}
|
}
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const unique: string[] = [];
|
const unique: string[] = [];
|
||||||
@@ -58,7 +38,7 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildPageRegistry(
|
function buildPageRegistry(
|
||||||
steps: DraftTraceStep[],
|
steps: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
||||||
const urls = collectUrls(steps, startUrl);
|
const urls = collectUrls(steps, startUrl);
|
||||||
@@ -81,9 +61,9 @@ function pageIdFor(
|
|||||||
return urlToId.values().next().value ?? "p1";
|
return urlToId.values().next().value ?? "p1";
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
||||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
||||||
if ("page_url" in step && step.page_url) return step.page_url;
|
if (step.pageUrl) return step.pageUrl;
|
||||||
return fallbackUrl;
|
return fallbackUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,7 +90,7 @@ function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function toV2Step(
|
function toV2Step(
|
||||||
step: DraftTraceStep,
|
step: RecordingDraftStep,
|
||||||
id: number,
|
id: number,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
fallbackUrl?: string,
|
fallbackUrl?: string,
|
||||||
@@ -129,28 +109,31 @@ function toV2Step(
|
|||||||
to: step.url,
|
to: step.url,
|
||||||
};
|
};
|
||||||
case "click":
|
case "click":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "hover":
|
case "hover":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return {
|
return {
|
||||||
op: "hover",
|
op: "hover",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
};
|
};
|
||||||
case "fill":
|
case "fill":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
value: step.value,
|
value: step.value,
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
...(step.redacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
@@ -161,26 +144,29 @@ function toV2Step(
|
|||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
key: step.key,
|
key: step.key,
|
||||||
...(step.target ? { target: step.target } : {}),
|
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "select":
|
case "select":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
selection: toSelection(step.values, step.labels),
|
selection: toSelection(step.values, step.labels),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
|
case "scroll":
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReducedTrace {
|
interface ReducedTrace {
|
||||||
pages: PageRefV2[];
|
pages: PageRefV2[];
|
||||||
steps: StepV2[];
|
steps: StepV2[];
|
||||||
}
|
}
|
||||||
@@ -189,7 +175,7 @@ export interface ReducedTrace {
|
|||||||
* Compile capture drafts into record-only trace v2 steps.
|
* Compile capture drafts into record-only trace v2 steps.
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||||
*/
|
*/
|
||||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
||||||
const collapsed = collapseNavigations(steps);
|
const collapsed = collapseNavigations(steps);
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||||
const out: StepV2[] = [];
|
const out: StepV2[] = [];
|
||||||
@@ -197,8 +183,8 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
|||||||
let lastUrl = startUrl;
|
let lastUrl = startUrl;
|
||||||
for (const draft of collapsed) {
|
for (const draft of collapsed) {
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
if (draft.op === "navigate") lastUrl = draft.url;
|
||||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
||||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
out.push(step);
|
out.push(step);
|
||||||
@@ -207,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
|||||||
return { pages, steps: out };
|
return { pages, steps: out };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTraceStartUrl(
|
function resolveTraceStartUrl(
|
||||||
drafts: DraftTraceStep[],
|
drafts: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
pages?: PageRefV2[],
|
pages?: PageRefV2[],
|
||||||
): string {
|
): string {
|
||||||
if (startUrl) return startUrl;
|
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
||||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
}
|
||||||
return step.op === "navigate";
|
|
||||||
});
|
export function buildTraceV2(input: {
|
||||||
if (navigate) return navigate.url;
|
steps: RecordingDraftStep[];
|
||||||
for (const draft of drafts) {
|
startedAt: string;
|
||||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
startUrl?: string;
|
||||||
}
|
purpose?: string;
|
||||||
return pages?.[0]?.url ?? "about:blank";
|
}): TraceV2 {
|
||||||
|
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
||||||
|
return {
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
||||||
|
pages,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||||
|
import { shouldIncludeDraft } from "./draft-policy";
|
||||||
|
import { unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface CollapsedDraft {
|
||||||
|
draft: RecordingDraftStep;
|
||||||
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||||
|
return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||||
|
}
|
||||||
|
|
||||||
|
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||||
|
const output: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const previous = output[output.length - 1];
|
||||||
|
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
||||||
|
previous.draft = {
|
||||||
|
...previous.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? previous.draft.postStateId,
|
||||||
|
};
|
||||||
|
previous.draftIds.push(index + 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
||||||
|
});
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
||||||
|
if (step.cause) return step.cause;
|
||||||
|
const qualifiers = step.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
||||||
|
return values.map((value, index) => ({
|
||||||
|
value,
|
||||||
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
||||||
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
const state = draft.preStateId ?? draft.postStateId;
|
||||||
|
const resultState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!state || !resultState) return null;
|
||||||
|
const common = { id, state, result: { state: resultState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
|
case "navigate":
|
||||||
|
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
||||||
|
case "click":
|
||||||
|
return {
|
||||||
|
op: "click",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "hover":
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "fill":
|
||||||
|
return {
|
||||||
|
op: "fill",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
value: draft.value,
|
||||||
|
commit: draft.commit ?? "blur",
|
||||||
|
...(draft.redacted ? { redacted: true } : {}),
|
||||||
|
};
|
||||||
|
case "press":
|
||||||
|
return {
|
||||||
|
op: "press",
|
||||||
|
...common,
|
||||||
|
key: draft.key,
|
||||||
|
...(draft.captureTarget || draft.matchedTarget
|
||||||
|
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
||||||
|
: {}),
|
||||||
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
|
};
|
||||||
|
case "select":
|
||||||
|
return {
|
||||||
|
op: "select",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
selection: selection(draft.values, draft.labels),
|
||||||
|
};
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...common };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReducedTraceV3 {
|
||||||
|
steps: StepV3[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
|
||||||
|
const output: StepV3[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
|
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||||
|
const step = reduceDraft(draft, output.length + 1);
|
||||||
|
if (!step) continue;
|
||||||
|
output.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||||
|
}
|
||||||
|
return { steps: output, stepIdByDraftId };
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import type { StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
||||||
|
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatTraceStateBody(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepIds: number[];
|
||||||
|
vomText: string;
|
||||||
|
annotations: StepAnnotation[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}): string {
|
||||||
|
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
||||||
|
lines.push(`url: ${JSON.stringify(input.url)}`);
|
||||||
|
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
||||||
|
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
||||||
|
for (const annotation of input.annotations) {
|
||||||
|
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byLine.get(annotation.line) ?? [];
|
||||||
|
bucket.push({ annotation, stepId });
|
||||||
|
byLine.set(annotation.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
||||||
|
let line = bodyLine;
|
||||||
|
for (const item of byLine.get(lineIndex) ?? []) {
|
||||||
|
line += annotationText(item.annotation, item.stepId);
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
});
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
import type {
|
||||||
|
FillCommit,
|
||||||
|
KeyModifier,
|
||||||
|
NavigationCause,
|
||||||
|
StepV3,
|
||||||
|
TargetDescriptorV3,
|
||||||
|
} from "@/transport/types";
|
||||||
|
|
||||||
|
export interface TargetGeometry {
|
||||||
|
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
tag: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TargetMatchHint {
|
||||||
|
geometry?: TargetGeometry;
|
||||||
|
/** Missing means the current top frame, never an unrestricted frame search. */
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepAnnotation {
|
||||||
|
draftId: number;
|
||||||
|
op: StepV3["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftStateLink {
|
||||||
|
pageUrl?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftTarget {
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
targetHint?: TargetMatchHint;
|
||||||
|
matchedTarget?: TargetDescriptorV3;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftNavigationEffect {
|
||||||
|
navigatedTo?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordingDraftStep =
|
||||||
|
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
||||||
|
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
||||||
|
| ({
|
||||||
|
op: "fill";
|
||||||
|
value: string;
|
||||||
|
commit?: FillCommit;
|
||||||
|
redacted?: boolean;
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "press";
|
||||||
|
key: string;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "select";
|
||||||
|
values: string[];
|
||||||
|
labels?: string[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({ op: "scroll" } & DraftStateLink)
|
||||||
|
| ({
|
||||||
|
op: "navigate";
|
||||||
|
url: string;
|
||||||
|
cause?: NavigationCause;
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
} & DraftStateLink);
|
||||||
|
|
||||||
|
export type TargetedRecordingDraft = Extract<
|
||||||
|
RecordingDraftStep,
|
||||||
|
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
||||||
|
>;
|
||||||
@@ -17,11 +17,11 @@ import {
|
|||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
||||||
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
DraftTraceStep,
|
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
RecordAwaitResult,
|
RecordAwaitResult,
|
||||||
RecordStartParams,
|
RecordStartParams,
|
||||||
@@ -47,7 +47,7 @@ interface ActiveRecording {
|
|||||||
agentWindowId: number;
|
agentWindowId: number;
|
||||||
startUrl?: string;
|
startUrl?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
steps: DraftTraceStep[];
|
steps: RecordingDraftStep[];
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
startedAtMs: number;
|
startedAtMs: number;
|
||||||
finishPromise: Promise<TraceV2>;
|
finishPromise: Promise<TraceV2>;
|
||||||
@@ -155,16 +155,12 @@ async function sendRecordStartWithAck(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): TraceV2 {
|
function buildTrace(recording: ActiveRecording): TraceV2 {
|
||||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
return buildTraceV2({
|
||||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
steps: recording.steps,
|
||||||
return {
|
startedAt: recording.startedAt,
|
||||||
|
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
|
||||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||||
recorded_at: new Date().toISOString(),
|
});
|
||||||
started_at: recording.startedAt,
|
|
||||||
entry: { start_url: startUrl },
|
|
||||||
pages,
|
|
||||||
steps,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordDeps {
|
export interface RecordDeps {
|
||||||
|
|||||||
@@ -744,48 +744,6 @@ export interface StepCommonV2 {
|
|||||||
effect?: StepEffectV2;
|
effect?: StepEffectV2;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Capture/buffer draft before v2 reduction. */
|
|
||||||
export type DraftTraceStep =
|
|
||||||
| {
|
|
||||||
op: "click";
|
|
||||||
target: TargetDescriptorV2;
|
|
||||||
navigated_to?: string;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "hover";
|
|
||||||
target: TargetDescriptorV2;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "fill";
|
|
||||||
target: TargetDescriptorV2;
|
|
||||||
value: string;
|
|
||||||
redacted?: boolean;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "press";
|
|
||||||
key: string;
|
|
||||||
target?: TargetDescriptorV2;
|
|
||||||
modifiers?: KeyModifier[];
|
|
||||||
navigated_to?: string;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "select";
|
|
||||||
target: TargetDescriptorV2;
|
|
||||||
values: string[];
|
|
||||||
labels?: string[];
|
|
||||||
navigated_to?: string;
|
|
||||||
page_url?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
op: "navigate";
|
|
||||||
url: string;
|
|
||||||
page_url?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Exported record-only step (trace v2). */
|
/** Exported record-only step (trace v2). */
|
||||||
export type StepV2 =
|
export type StepV2 =
|
||||||
| ({ op: "navigate" } & StepCommonV2 & { to: string })
|
| ({ op: "navigate" } & StepCommonV2 & { to: string })
|
||||||
|
|||||||
Reference in New Issue
Block a user