mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(record): capture Trace v3 states after pages settle
Wait for DOM quiet after actions, match targets to VOM refs, and reduce drafts into deduped state-linked Trace v3 while keeping the v2 reducer. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
“haonan”
@@ -0,0 +1,116 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import { matchTarget } from "../match-target";
|
||||
|
||||
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("matchTarget", () => {
|
||||
const refs: RenderedRef[] = [
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
|
||||
];
|
||||
|
||||
it("matches a unique node by viewport coordinates", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42)],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("matches a non-fixed node after the top frame scrolls", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 40,
|
||||
scrollY: 300,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [
|
||||
node(42, {
|
||||
documentRect: { x: 50, y: 320, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("matches a static node by document coordinates when scroll changed after observation", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 250,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [
|
||||
node(42, {
|
||||
documentRect: { x: 10, y: 300, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 200, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 200, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("returns unmatched when multiple candidates match", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42), node(43)],
|
||||
refs,
|
||||
});
|
||||
expect(target.unmatched).toBe(true);
|
||||
});
|
||||
|
||||
it("uses viewport coordinates for fixed elements", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 5, y: 5, w: 80, h: 24 },
|
||||
scrollX: 100,
|
||||
scrollY: 200,
|
||||
position: "fixed",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
|
||||
refs,
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { waitForPageSettled } from "../page-settled";
|
||||
|
||||
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||
const send = vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
}));
|
||||
return {
|
||||
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||
send,
|
||||
};
|
||||
}
|
||||
|
||||
describe("waitForPageSettled", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { cdp, send } = quietCdp();
|
||||
|
||||
const settled = waitForPageSettled(cdp, 7);
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("cancels before probing a superseded observation", async () => {
|
||||
vi.useFakeTimers();
|
||||
let cancelled = false;
|
||||
const { cdp, send } = quietCdp();
|
||||
|
||||
const settled = waitForPageSettled(cdp, 7, { cancelled: () => cancelled });
|
||||
cancelled = true;
|
||||
await vi.advanceTimersByTimeAsync(60);
|
||||
|
||||
await expect(settled).resolves.toBe("cancelled");
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,206 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { resetStateIdCounterForTests } from "../record-constants";
|
||||
import {
|
||||
applyTargetMatching,
|
||||
buildTraceV3,
|
||||
createObservationState,
|
||||
flushPendingRedirectLanding,
|
||||
rememberStepOnPage,
|
||||
scheduleRedirectLandingFlush,
|
||||
} from "../record-observation";
|
||||
import { registerObservation } from "../trace-reducer";
|
||||
|
||||
const URL = "https://example.com/login";
|
||||
|
||||
function capturedInput(): CapturedNode {
|
||||
return {
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: null,
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
const obs = createObservationState({ redactValues });
|
||||
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
|
||||
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
|
||||
obs.lastSettled = {
|
||||
stateId,
|
||||
captured: [capturedInput()],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||
url: URL,
|
||||
vomText: rawVomText,
|
||||
};
|
||||
|
||||
const draft: DraftTraceStep = {
|
||||
op: "fill",
|
||||
target: { unmatched: true },
|
||||
value,
|
||||
geometry: {
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "input",
|
||||
},
|
||||
};
|
||||
applyTargetMatching(obs, draft, 1);
|
||||
draft.postStateId = stateId;
|
||||
rememberStepOnPage(obs, draft, 1);
|
||||
|
||||
return (
|
||||
buildTraceV3({
|
||||
obs,
|
||||
steps: [draft],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
}).states[0]?.body ?? ""
|
||||
);
|
||||
}
|
||||
|
||||
describe("record observation annotations", () => {
|
||||
beforeEach(() => {
|
||||
resetStateIdCounterForTests();
|
||||
});
|
||||
|
||||
it("omits a fill literal from finalized state bodies when values are redacted", () => {
|
||||
const secret = "hunter2-private";
|
||||
const body = finalizedFillBody(secret, true);
|
||||
|
||||
expect(body).toContain("step 1: fill");
|
||||
expect(body).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps fill details in ordinary recording annotations", () => {
|
||||
const value = "ordinary text";
|
||||
const body = finalizedFillBody(value, false);
|
||||
|
||||
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyTargetMatching without a settled observation", () => {
|
||||
it("keeps the capture role/name instead of a bare unmatched target", () => {
|
||||
const obs = createObservationState();
|
||||
const draft: DraftTraceStep = {
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
};
|
||||
|
||||
applyTargetMatching(obs, draft, 1);
|
||||
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
expect(draft.target).toEqual({
|
||||
role: "button",
|
||||
name: "新建",
|
||||
unmatched: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyTargetMatching while the previous action is still settling", () => {
|
||||
it("does not bind a new unmatched control to the stale observation", () => {
|
||||
const obs = createObservationState();
|
||||
const stateId = registerObservation(obs.stateRegistry, {
|
||||
url: URL,
|
||||
rawVomText: '@vom 1\n@e1 textbox "Search"',
|
||||
});
|
||||
obs.lastSettled = {
|
||||
stateId,
|
||||
captured: [capturedInput()],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Search", line: 1 }],
|
||||
url: URL,
|
||||
vomText: '@vom 1\n@e1 textbox "Search"',
|
||||
};
|
||||
obs.settles.set(0, { draftIndex: 0, cancelled: false });
|
||||
const draft: DraftTraceStep = {
|
||||
op: "click",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||
geometry: {
|
||||
rect: { x: 400, y: 300, w: 80, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "fixed",
|
||||
tag: "button",
|
||||
},
|
||||
};
|
||||
|
||||
applyTargetMatching(obs, draft, 2);
|
||||
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
expect(draft.target).toEqual({ role: "button", name: "Confirm", unmatched: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("redirect landing coalescing", () => {
|
||||
it("does not consume a newer redirect hop while reading tab metadata", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const finalUrl = "https://example.com/final";
|
||||
const intermediateUrl = "https://example.com/intermediate";
|
||||
const obs = createObservationState();
|
||||
obs.lastSettled = {
|
||||
stateId: "s-final",
|
||||
captured: [],
|
||||
refs: [],
|
||||
url: finalUrl,
|
||||
vomText: "@vom 1\nFinal",
|
||||
};
|
||||
const steps: DraftTraceStep[] = [];
|
||||
|
||||
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveFirstTab = resolve;
|
||||
});
|
||||
const get = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => firstTab)
|
||||
.mockResolvedValue({ id: 7, url: finalUrl });
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get,
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||
|
||||
scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, intermediateUrl);
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
expect(get).toHaveBeenCalledTimes(1);
|
||||
|
||||
scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, finalUrl);
|
||||
resolveFirstTab({ id: 7, url: intermediateUrl } as chrome.tabs.Tab);
|
||||
await Promise.resolve();
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await flushPendingRedirectLanding(obs, steps, cdp, 7, tabsApi);
|
||||
|
||||
expect(steps).toEqual([]);
|
||||
expect(obs.pendingRedirect).toBeNull();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
|
||||
describe("trace-reducer-v2", () => {
|
||||
it("drops scroll and bare character press steps", () => {
|
||||
const steps: DraftTraceStep[] = [
|
||||
{
|
||||
op: "scroll",
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
];
|
||||
const trace = buildTraceV2({
|
||||
steps,
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/",
|
||||
});
|
||||
expect(trace.steps).toHaveLength(0);
|
||||
expect(trace.pages).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("preserves hover steps supported by protocol v2", () => {
|
||||
const trace = buildTraceV2({
|
||||
steps: [
|
||||
{
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "结束" },
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
],
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/",
|
||||
});
|
||||
|
||||
expect(trace.steps).toEqual([
|
||||
expect.objectContaining({
|
||||
op: "hover",
|
||||
target: expect.objectContaining({ tag: "button", name: "结束" }),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps captureTarget to v2 target with required tag", () => {
|
||||
const steps: DraftTraceStep[] = [
|
||||
{
|
||||
op: "click",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "Submit" },
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
];
|
||||
const trace = buildTraceV2({
|
||||
steps,
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/",
|
||||
});
|
||||
expect(trace.steps).toHaveLength(1);
|
||||
expect(trace.steps[0]?.op).toBe("click");
|
||||
if (trace.steps[0]?.op === "click") {
|
||||
expect(trace.steps[0].target.tag).toBe("button");
|
||||
expect(trace.steps[0].target.name).toBe("Submit");
|
||||
}
|
||||
expect("version" in trace).toBe(false);
|
||||
});
|
||||
|
||||
it("records Enter presses", () => {
|
||||
expect(shouldRecordPress("Enter")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,8 @@
|
||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||
*/
|
||||
|
||||
export interface TargetDescriptor {
|
||||
/** Content-script capture descriptor before VOM geometric matching. */
|
||||
export interface CaptureTargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
@@ -15,6 +16,9 @@ export interface TargetDescriptor {
|
||||
nearby_label?: string;
|
||||
}
|
||||
|
||||
/** @deprecated Capture-time alias retained until the recorder integration migrates. */
|
||||
export type TargetDescriptor = CaptureTargetDescriptor;
|
||||
|
||||
/** Max length for a label that is still a useful “find this on screen” hint. */
|
||||
const ACTIONABLE_LABEL_MAX = 48;
|
||||
|
||||
@@ -247,7 +251,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
* Recording “点击div” with no name fails this bar.
|
||||
*/
|
||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
||||
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||
const name = target.name?.trim();
|
||||
if (name && isActionableLabel(name)) return true;
|
||||
if (
|
||||
@@ -323,7 +327,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function describeTarget(el: Element): TargetDescriptor {
|
||||
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = inferRole(el);
|
||||
const name = accessibleName(el);
|
||||
@@ -350,7 +354,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
||||
};
|
||||
}
|
||||
|
||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
||||
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (!clickable) return null;
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { StepV3 } from "@/transport/types";
|
||||
import { OBSERVATION_FILE_VERSION } from "./record-constants";
|
||||
|
||||
export interface ObservationAnnotation {
|
||||
stepId: number;
|
||||
op: StepV3["op"];
|
||||
line: number;
|
||||
stateId: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
|
||||
const suffix = detail ? `: ${detail}` : "";
|
||||
return ` ⟵ step ${stepId}: ${op}${suffix}`;
|
||||
}
|
||||
|
||||
/** Serialize a page observation file (front matter + VOM body + step annotations). */
|
||||
export function formatObservationFile(input: {
|
||||
stateId: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
stepsHere: number[];
|
||||
body: string;
|
||||
annotations?: ObservationAnnotation[];
|
||||
}): string {
|
||||
const lines: string[] = [
|
||||
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
|
||||
`state: ${input.stateId}`,
|
||||
`url: ${input.url}`,
|
||||
];
|
||||
if (input.title) lines.push(`title: ${input.title}`);
|
||||
if (input.stepsHere.length > 0) {
|
||||
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
|
||||
}
|
||||
lines.push("---");
|
||||
|
||||
const bodyLines = input.body.split("\n");
|
||||
const annotationMap = new Map<number, ObservationAnnotation[]>();
|
||||
for (const ann of input.annotations ?? []) {
|
||||
const bucket = annotationMap.get(ann.line) ?? [];
|
||||
bucket.push(ann);
|
||||
annotationMap.set(ann.line, bucket);
|
||||
}
|
||||
|
||||
for (let i = 0; i < bodyLines.length; i += 1) {
|
||||
let line = bodyLines[i] ?? "";
|
||||
const anns = annotationMap.get(i);
|
||||
if (anns) {
|
||||
for (const ann of anns) {
|
||||
line += formatAnnotation(ann);
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
/** Hash input: VOM body **before** annotations are inserted. */
|
||||
export function observationBodyForHash(vomText: string): string {
|
||||
return vomText;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { CaptureGeometry, TargetDescriptorV3 } from "@/transport/types";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
|
||||
|
||||
export interface MatchTargetInput {
|
||||
geometry: CaptureGeometry;
|
||||
captured: CapturedNode[];
|
||||
refs: RenderedRef[];
|
||||
fallback?: CaptureTargetDescriptor;
|
||||
}
|
||||
|
||||
function withinTolerance(a: number, b: number): boolean {
|
||||
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
|
||||
}
|
||||
|
||||
function rectsMatch(
|
||||
a: { x: number; y: number; w: number; h: number },
|
||||
b: { x: number; y: number; w: number; h: number },
|
||||
): boolean {
|
||||
return (
|
||||
withinTolerance(a.x, b.x) &&
|
||||
withinTolerance(a.y, b.y) &&
|
||||
withinTolerance(a.w, b.w) &&
|
||||
withinTolerance(a.h, b.h)
|
||||
);
|
||||
}
|
||||
|
||||
function nodeViewportRect(
|
||||
node: CapturedNode,
|
||||
): { x: number; y: number; w: number; h: number } | null {
|
||||
return node.localRect ?? node.rect;
|
||||
}
|
||||
|
||||
function matchingRects(
|
||||
geometry: CaptureGeometry,
|
||||
node: CapturedNode,
|
||||
): {
|
||||
target: { x: number; y: number; w: number; h: number };
|
||||
node: { x: number; y: number; w: number; h: number } | null;
|
||||
} {
|
||||
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
|
||||
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||
if (!topFrame || viewportPosition || !node.documentRect) {
|
||||
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||
}
|
||||
return {
|
||||
target: {
|
||||
x: geometry.rect.x + geometry.scrollX,
|
||||
y: geometry.rect.y + geometry.scrollY,
|
||||
w: geometry.rect.w,
|
||||
h: geometry.rect.h,
|
||||
},
|
||||
node: node.documentRect,
|
||||
};
|
||||
}
|
||||
|
||||
function tagMatches(geometryTag: string, nodeTag: string): boolean {
|
||||
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
|
||||
}
|
||||
|
||||
/** Best description available when the element cannot be located in the VOM. */
|
||||
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||
if (!fallback) {
|
||||
return { unmatched: true };
|
||||
}
|
||||
return {
|
||||
...(fallback.role ? { role: fallback.role } : {}),
|
||||
...(fallback.name ? { name: fallback.name } : {}),
|
||||
unmatched: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Locate the interacted element in the last settled observation by geometry. */
|
||||
export function matchTarget(input: MatchTargetInput): TargetDescriptorV3 {
|
||||
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
|
||||
|
||||
const candidates = input.captured.filter((node) => {
|
||||
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
|
||||
if (!tagMatches(input.geometry.tag, node.tag)) return false;
|
||||
const rects = matchingRects(input.geometry, node);
|
||||
if (!rects.node) return false;
|
||||
return rectsMatch(rects.target, rects.node);
|
||||
});
|
||||
|
||||
if (candidates.length !== 1) {
|
||||
return fallbackDescriptor(input.fallback);
|
||||
}
|
||||
|
||||
const backendNodeId = candidates[0]!.backendNodeId;
|
||||
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
|
||||
if (!refEntry) {
|
||||
return fallbackDescriptor(input.fallback);
|
||||
}
|
||||
|
||||
return {
|
||||
ref: refEntry.ref,
|
||||
...(refEntry.role ? { role: refEntry.role } : {}),
|
||||
...(refEntry.name ? { name: refEntry.name } : {}),
|
||||
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// Deciding *when* a page is worth observing.
|
||||
//
|
||||
// A fixed delay cannot serve both a modal that toggles in one frame and a
|
||||
// route change that renders for a second. Instead of guessing, ask the page:
|
||||
// a document that has stopped mutating and is no longer loading is done
|
||||
// reacting to whatever the user just did.
|
||||
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
|
||||
|
||||
/**
|
||||
* Installed once per document. Records only the timestamp of the last DOM
|
||||
* change so each poll stays O(1) no matter how large the page is.
|
||||
*/
|
||||
const QUIET_PROBE = `(() => {
|
||||
const scope = window;
|
||||
let probe = scope.__bskRecordQuiet;
|
||||
if (!probe) {
|
||||
probe = { changedAt: Date.now() };
|
||||
const observer = new MutationObserver(() => {
|
||||
probe.changedAt = Date.now();
|
||||
});
|
||||
observer.observe(document.documentElement, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
characterData: true,
|
||||
});
|
||||
scope.__bskRecordQuiet = probe;
|
||||
}
|
||||
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||
})()`;
|
||||
|
||||
export type SettleOutcome =
|
||||
/** The page stopped changing on its own. */
|
||||
| "quiet"
|
||||
/** Still changing when the budget ran out; observe it as it is. */
|
||||
| "timeout"
|
||||
/** A newer action took over; this observation is no longer wanted. */
|
||||
| "cancelled";
|
||||
|
||||
interface QuietProbe {
|
||||
idleMs: number;
|
||||
readyState: string;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
|
||||
try {
|
||||
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||
expression: QUIET_PROBE,
|
||||
returnByValue: true,
|
||||
});
|
||||
const value = reply.result?.value;
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||
return { idleMs, readyState };
|
||||
} catch {
|
||||
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
|
||||
// not a reason to give up.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until the page has finished reacting, or until the budget runs out. */
|
||||
export async function waitForPageSettled(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): Promise<SettleOutcome> {
|
||||
const startedAt = Date.now();
|
||||
const floor = startedAt + SETTLE_MIN_MS;
|
||||
const deadline = startedAt + SETTLE_MAX_MS;
|
||||
|
||||
for (;;) {
|
||||
if (options.cancelled?.()) return "cancelled";
|
||||
await sleep(SETTLE_POLL_MS);
|
||||
if (options.cancelled?.()) return "cancelled";
|
||||
|
||||
const probe = await readQuietProbe(cdp, tabId);
|
||||
const now = Date.now();
|
||||
if (now < floor) continue;
|
||||
if (now >= deadline) return "timeout";
|
||||
if (!probe) continue;
|
||||
if (probe.readyState === "loading") continue;
|
||||
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { FillCommit, NavigationCause } from "@/transport/types";
|
||||
|
||||
/**
|
||||
* Floor before a post-action observation may be taken. A page usually has not
|
||||
* started reacting in the first frames after an action, and capturing then
|
||||
* would record the page as it was, not as the action left it.
|
||||
*/
|
||||
export const SETTLE_MIN_MS = 150;
|
||||
|
||||
/** How long the DOM must stop changing before a page counts as settled. */
|
||||
export const SETTLE_QUIET_MS = 250;
|
||||
|
||||
/** Upper bound on waiting for a page to settle; animations never stop. */
|
||||
export const SETTLE_MAX_MS = 2_000;
|
||||
|
||||
/** How often to ask the page whether it has gone quiet. */
|
||||
export const SETTLE_POLL_MS = 60;
|
||||
|
||||
/** Minimum interval between consecutive observations on the same tab. */
|
||||
export const OBSERVATION_MIN_INTERVAL_MS = 200;
|
||||
|
||||
/** Delay before retrying a capture that lost its execution context. */
|
||||
export const CAPTURE_RETRY_DELAY_MS = 250;
|
||||
|
||||
/** Document-coordinate matching tolerance in CSS pixels. */
|
||||
export const GEOM_MATCH_TOLERANCE_PX = 2;
|
||||
|
||||
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
|
||||
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
|
||||
|
||||
/** VOM observation file format version (front matter header). */
|
||||
export const OBSERVATION_FILE_VERSION = 1;
|
||||
|
||||
/** FNV-1a 64-bit offset basis. */
|
||||
const FNV_OFFSET = 0xcbf29ce484222325n;
|
||||
/** FNV-1a 64-bit prime. */
|
||||
const FNV_PRIME = 0x100000001b3n;
|
||||
|
||||
/** Content-hash for state deduplication (sync, non-cryptographic). */
|
||||
export function fnv1a64(text: string): string {
|
||||
let hash = FNV_OFFSET;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
hash ^= BigInt(text.charCodeAt(i));
|
||||
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
|
||||
}
|
||||
return hash.toString(16).padStart(16, "0");
|
||||
}
|
||||
|
||||
let nextStateSerial = 0;
|
||||
|
||||
/** Monotonic state id generator (`s1`, `s2`, …). */
|
||||
export function nextStateId(): string {
|
||||
nextStateSerial += 1;
|
||||
return `s${nextStateSerial}`;
|
||||
}
|
||||
|
||||
/** Test seam: reset the state id counter. */
|
||||
export function resetStateIdCounterForTests(): void {
|
||||
nextStateSerial = 0;
|
||||
}
|
||||
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
|
||||
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
|
||||
typed: "user_typed",
|
||||
generated: "user_typed",
|
||||
keyword: "user_typed",
|
||||
keyword_generated: "user_typed",
|
||||
link: "link",
|
||||
form_submit: "form_submit",
|
||||
reload: "reload",
|
||||
auto_bookmark: "browser",
|
||||
start_page: "browser",
|
||||
};
|
||||
|
||||
export interface NavigationTransitionMeta {
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
navigationActionPending?: boolean;
|
||||
}
|
||||
|
||||
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
|
||||
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
|
||||
const qualifiers = meta.transitionQualifiers ?? [];
|
||||
if (qualifiers.includes("forward_back")) return "history";
|
||||
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
|
||||
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||
|
||||
const type = meta.transitionType ?? "";
|
||||
const mapped = TRANSITION_TO_CAUSE[type];
|
||||
if (mapped) return mapped;
|
||||
|
||||
if (!type && meta.navigationActionPending === false) return "script";
|
||||
return "browser";
|
||||
}
|
||||
|
||||
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
|
||||
@@ -0,0 +1,681 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
|
||||
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
|
||||
import { waitForPageSettled } from "@/lib/page-settled";
|
||||
import {
|
||||
CAPTURE_RETRY_DELAY_MS,
|
||||
DEFAULT_MAX_PAGE_TOKENS,
|
||||
OBSERVATION_MIN_INTERVAL_MS,
|
||||
} from "@/lib/record-constants";
|
||||
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
|
||||
import { captureVomObservation } from "@/tools/capture-vom-observation";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { DraftTraceStep, StepV3, StopReason, TraceState, TraceV3 } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||
|
||||
export interface LastSettledObservation {
|
||||
stateId: string;
|
||||
captured: CapturedNode[];
|
||||
refs: RenderedRef[];
|
||||
url: string;
|
||||
title?: string;
|
||||
vomText: string;
|
||||
}
|
||||
|
||||
/** A post-action observation that has been queued but not yet written down. */
|
||||
interface PendingSettle {
|
||||
draftIndex: number;
|
||||
/** Set when a newer action has already decided where this step landed. */
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
|
||||
export interface PendingRedirectLanding {
|
||||
url: string;
|
||||
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export interface RecordingObservationState {
|
||||
stateRegistry: Map<string, StateRegistryEntry>;
|
||||
lastSettled: LastSettledObservation | null;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
lastCaptureAtMs: number;
|
||||
/**
|
||||
* Tail of the settle chain. Observations run one at a time so that a slow
|
||||
* capture can never land after a faster one and report the pages out of the
|
||||
* order the user visited them.
|
||||
*/
|
||||
settleQueue: Promise<void>;
|
||||
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
|
||||
settles: Map<number, PendingSettle>;
|
||||
stepAnnotations: Map<number, ObservationAnnotation[]>;
|
||||
/**
|
||||
* OAuth / server redirects are coalesced here: intermediate hops only update
|
||||
* `url`+`generation`, and one `navigate` is emitted after the page settles.
|
||||
*/
|
||||
pendingRedirect: PendingRedirectLanding | null;
|
||||
redirectFlushQueue: Promise<void>;
|
||||
}
|
||||
|
||||
export function createObservationState(options?: {
|
||||
maxPageTokens?: number;
|
||||
redactValues?: boolean;
|
||||
}): RecordingObservationState {
|
||||
return {
|
||||
stateRegistry: new Map(),
|
||||
lastSettled: null,
|
||||
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues: options?.redactValues ?? false,
|
||||
lastCaptureAtMs: 0,
|
||||
settleQueue: Promise.resolve(),
|
||||
settles: new Map(),
|
||||
stepAnnotations: new Map(),
|
||||
pendingRedirect: null,
|
||||
redirectFlushQueue: Promise.resolve(),
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function readTabMeta(
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
): Promise<{ url: string; title?: string }> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||
} catch {
|
||||
return { url: "about:blank" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureAndRegisterObservation(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
urlOverride?: string,
|
||||
): Promise<LastSettledObservation> {
|
||||
const now = Date.now();
|
||||
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
|
||||
if (waitMs > 0) await sleep(waitMs);
|
||||
|
||||
const { url, title } = urlOverride
|
||||
? { url: urlOverride, title: undefined }
|
||||
: await readTabMeta(tabsApi, tabId);
|
||||
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
|
||||
const resolvedTitle = title ?? meta.title;
|
||||
|
||||
const rendered = await captureVomObservation(cdp, tabId, url, {
|
||||
maxTokens: obs.maxPageTokens,
|
||||
redactValues: obs.redactValues,
|
||||
conditionalSurfaceProbe: false,
|
||||
});
|
||||
|
||||
const stateId = registerObservation(obs.stateRegistry, {
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
rawVomText: rendered.text,
|
||||
truncated: rendered.truncated,
|
||||
});
|
||||
|
||||
obs.lastCaptureAtMs = Date.now();
|
||||
const settled: LastSettledObservation = {
|
||||
stateId,
|
||||
captured: rendered.captured,
|
||||
refs: rendered.refs,
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
vomText: rendered.text,
|
||||
};
|
||||
obs.lastSettled = settled;
|
||||
return settled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind a draft to the page it was performed on: origin state, geometric target
|
||||
* match, and the inline annotation. All three read the *current* observation,
|
||||
* which is only the page the user acted on while the draft is still fresh —
|
||||
* once the action settles, `lastSettled` is the destination page instead.
|
||||
*/
|
||||
export function applyTargetMatching(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
stepId: number,
|
||||
): void {
|
||||
if (!obs.lastSettled) {
|
||||
// Initial observation can fail or race the first action. Keep whatever
|
||||
// the content script captured so the exported step is still teachable.
|
||||
if ("target" in draft && "captureTarget" in draft) {
|
||||
draft.target = fallbackDescriptor(draft.captureTarget);
|
||||
}
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
return;
|
||||
}
|
||||
|
||||
if ("target" in draft) {
|
||||
draft.target =
|
||||
"geometry" in draft && draft.geometry
|
||||
? matchTarget({
|
||||
geometry: draft.geometry,
|
||||
captured: obs.lastSettled.captured,
|
||||
refs: obs.lastSettled.refs,
|
||||
fallback: draft.captureTarget,
|
||||
})
|
||||
: // No geometry to match on, but the capture still knew what the user
|
||||
// touched — keep that instead of an anonymous unmatched target.
|
||||
fallbackDescriptor(draft.captureTarget);
|
||||
}
|
||||
|
||||
// A pending settle means `lastSettled` may still describe the page before
|
||||
// the previous action. If the new control cannot be found there, attaching
|
||||
// that stale state would be worse than leaving the origin unknown for the
|
||||
// reducer to repair from the observations that do exist.
|
||||
const originLooksStale =
|
||||
obs.settles.size > 0 && (!("target" in draft) || draft.target?.unmatched === true);
|
||||
if (originLooksStale) {
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
return;
|
||||
}
|
||||
|
||||
draft.preStateId = obs.lastSettled.stateId;
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
}
|
||||
|
||||
function fillDetailForDraft(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
): string | undefined {
|
||||
if (obs.redactValues) return undefined;
|
||||
if (draft.op === "fill") return JSON.stringify(draft.value);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function refLineForDraft(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
): number | undefined {
|
||||
if (!("target" in draft) || !obs.lastSettled) return undefined;
|
||||
const targetRef = draft.target?.ref;
|
||||
if (!targetRef) return undefined;
|
||||
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
|
||||
}
|
||||
|
||||
function rememberStepAnnotation(
|
||||
obs: RecordingObservationState,
|
||||
stepId: number,
|
||||
draft: DraftTraceStep,
|
||||
): void {
|
||||
const line = refLineForDraft(obs, draft);
|
||||
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
|
||||
if (!draft.preStateId) return;
|
||||
const ann: ObservationAnnotation = {
|
||||
stepId,
|
||||
op: draft.op,
|
||||
line,
|
||||
stateId: draft.preStateId,
|
||||
detail: fillDetailForDraft(obs, draft),
|
||||
};
|
||||
const bucket = obs.stepAnnotations.get(line) ?? [];
|
||||
bucket.push(ann);
|
||||
obs.stepAnnotations.set(line, bucket);
|
||||
}
|
||||
|
||||
/** Record the step under the page it was performed on, not the one it led to. */
|
||||
export function rememberStepOnPage(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
stepId: number,
|
||||
): void {
|
||||
const stateId = draft.preStateId;
|
||||
if (!stateId) return;
|
||||
const entry = obs.stateRegistry.get(stateId);
|
||||
if (entry && !entry.stepsHere.includes(stepId)) {
|
||||
entry.stepsHere.push(stepId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
|
||||
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
|
||||
obs.pendingRedirect = null;
|
||||
}
|
||||
|
||||
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
|
||||
* to the final URL. Later hops only bump `generation` / replace `url`.
|
||||
*/
|
||||
export function scheduleRedirectLandingFlush(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
url: string,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): void {
|
||||
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
|
||||
obs.pendingRedirect = { url, generation };
|
||||
enqueueRedirectFlush(obs, () =>
|
||||
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain any coalesced redirect so the next action matches against the real
|
||||
* landing page. Safe to call when nothing is pending.
|
||||
*/
|
||||
export async function flushPendingRedirectLanding(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner | undefined,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): Promise<void> {
|
||||
if (obs.pendingRedirect && cdp) {
|
||||
enqueueRedirectFlush(obs, () =>
|
||||
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||
);
|
||||
}
|
||||
await obs.redirectFlushQueue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until the redirect chain stops changing the document, then emit one
|
||||
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
|
||||
*/
|
||||
async function coalesceRedirectLanding(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
options: { cancelled?: () => boolean },
|
||||
): Promise<void> {
|
||||
while (obs.pendingRedirect) {
|
||||
if (options.cancelled?.()) {
|
||||
obs.pendingRedirect = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const snap = obs.pendingRedirect;
|
||||
const outcome = await waitForPageSettled(cdp, tabId, {
|
||||
cancelled: () =>
|
||||
!!options.cancelled?.() ||
|
||||
obs.pendingRedirect === null ||
|
||||
obs.pendingRedirect.generation !== snap.generation,
|
||||
});
|
||||
|
||||
if (options.cancelled?.()) {
|
||||
obs.pendingRedirect = null;
|
||||
return;
|
||||
}
|
||||
// A newer hop cancelled this wait — loop and settle against the latest URL.
|
||||
if (outcome === "cancelled") continue;
|
||||
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
|
||||
// `tabs.get` is asynchronous. A newer redirect hop may have arrived while
|
||||
// it was in flight; only the generation that initiated this read may
|
||||
// consume the pending landing.
|
||||
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||
continue;
|
||||
}
|
||||
obs.pendingRedirect = null;
|
||||
|
||||
if (!finalUrl || finalUrl === "about:blank") return;
|
||||
if (obs.lastSettled?.url === finalUrl) return;
|
||||
|
||||
const last = steps[steps.length - 1];
|
||||
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||
if (!last.postStateId) {
|
||||
const draftIndex = steps.length - 1;
|
||||
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
|
||||
await obs.settleQueue;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (outcome === "timeout") {
|
||||
console.debug(
|
||||
`[bsk record] redirect landing still changing after settle budget; ` +
|
||||
`recording navigate to ${finalUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const draft: DraftTraceStep = {
|
||||
op: "navigate",
|
||||
url: finalUrl,
|
||||
page_url: finalUrl,
|
||||
cause: "browser",
|
||||
preStateId: obs.lastSettled?.stateId,
|
||||
};
|
||||
steps.push(draft);
|
||||
const draftIndex = steps.length - 1;
|
||||
const stepId = draftIndex + 1;
|
||||
rememberStepOnPage(obs, draft, stepId);
|
||||
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
|
||||
// Callers that flush before the next action need `lastSettled` to already
|
||||
// be the landing page so target matching does not use the pre-redirect view.
|
||||
await obs.settleQueue;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A capture that lands mid-navigation fails on a destroyed execution context.
|
||||
* That is transient, so give it one more chance before the step has to fall
|
||||
* back to a stale state.
|
||||
*/
|
||||
async function captureWithRetry(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
urlOverride?: string,
|
||||
): Promise<LastSettledObservation> {
|
||||
try {
|
||||
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||
} catch (err) {
|
||||
console.debug("[bsk record] observation failed, retrying once", err);
|
||||
await sleep(CAPTURE_RETRY_DELAY_MS);
|
||||
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||
}
|
||||
}
|
||||
|
||||
export async function settleDraftObservation(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
draftIndex: number,
|
||||
pending?: PendingSettle,
|
||||
): Promise<void> {
|
||||
const cancelled = () => pending?.cancelled === true;
|
||||
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
|
||||
if (outcome === "cancelled") return;
|
||||
if (outcome === "timeout") {
|
||||
console.debug(
|
||||
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
|
||||
"budget; observing it as it stands",
|
||||
);
|
||||
}
|
||||
|
||||
const started = steps[draftIndex];
|
||||
if (!started) return;
|
||||
|
||||
// The URL attached to the draft may only be an intermediate redirect hop.
|
||||
// Once the page is settled, register the capture against live tab metadata.
|
||||
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||
// The observation still counts as the latest view of the page even when this
|
||||
// step no longer wants it — the next action will start from it.
|
||||
if (cancelled()) return;
|
||||
// Re-read the slot: a navigation observed while the capture ran may have
|
||||
// rewritten this draft, and the observation belongs to whatever occupies the
|
||||
// slot now — writing to the object we started with would strand it.
|
||||
const draft = steps[draftIndex] ?? started;
|
||||
draft.postStateId = settled.stateId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover landing states from the shape of the recording itself: wherever the
|
||||
* next action was performed is, by definition, where the previous one landed.
|
||||
* Only the immediately following draft counts — a later one would claim a page
|
||||
* that several unobserved actions away, which is worse than admitting we saw
|
||||
* no change.
|
||||
*/
|
||||
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
|
||||
for (let i = 0; i < steps.length - 1; i += 1) {
|
||||
const draft = steps[i];
|
||||
const next = steps[i + 1];
|
||||
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
|
||||
draft.postStateId = next.preStateId;
|
||||
console.debug(
|
||||
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
|
||||
`using where step ${i + 2} started (${next.preStateId})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Last chance for the closing actions of a recording. Stopping right after the
|
||||
* final action is normal, and so is that action navigating away, so take one
|
||||
* final look at the page. Only trailing drafts may claim it: an earlier step
|
||||
* did not land on whatever the user happens to be looking at when they stop.
|
||||
*/
|
||||
export async function settleUnsettledDrafts(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
): Promise<void> {
|
||||
const trailing: DraftTraceStep[] = [];
|
||||
for (let i = steps.length - 1; i >= 0; i -= 1) {
|
||||
const draft = steps[i];
|
||||
if (!draft || draft.postStateId) break;
|
||||
trailing.push(draft);
|
||||
}
|
||||
if (trailing.length === 0) return;
|
||||
|
||||
let settled: LastSettledObservation;
|
||||
try {
|
||||
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||
} catch (err) {
|
||||
console.warn("[bsk record] final observation at stop failed", err);
|
||||
return;
|
||||
}
|
||||
for (const draft of trailing) draft.postStateId = settled.stateId;
|
||||
console.debug(
|
||||
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
|
||||
`(${settled.stateId} ${settled.url})`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Performing a new action ends the previous one: the page the user reached for
|
||||
* is, by definition, where the earlier step left them. Taking the landing from
|
||||
* the newer step also keeps the trace monotonic — a capture that finished late
|
||||
* would otherwise credit an earlier step with a page that only exists because
|
||||
* of the newer action.
|
||||
*/
|
||||
function supersedeEarlierSettles(
|
||||
obs: RecordingObservationState,
|
||||
draftIndex: number,
|
||||
steps: DraftTraceStep[],
|
||||
): void {
|
||||
const landing = steps[draftIndex]?.preStateId;
|
||||
for (const [index, pending] of obs.settles) {
|
||||
if (index >= draftIndex) continue;
|
||||
pending.cancelled = true;
|
||||
obs.settles.delete(index);
|
||||
|
||||
const draft = steps[index];
|
||||
if (!draft || draft.postStateId) continue;
|
||||
if (!landing) continue;
|
||||
draft.postStateId = landing;
|
||||
console.debug(
|
||||
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
|
||||
`${draftIndex + 1} started; landing it on ${landing}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleDraftSettle(
|
||||
obs: RecordingObservationState,
|
||||
draftIndex: number,
|
||||
stepId: number,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
): void {
|
||||
supersedeEarlierSettles(obs, draftIndex, steps);
|
||||
|
||||
// Rescheduling the same step (a navigation showed up after the action)
|
||||
// replaces the pending observation rather than racing it.
|
||||
const superseded = obs.settles.get(draftIndex);
|
||||
if (superseded) superseded.cancelled = true;
|
||||
|
||||
const pending: PendingSettle = { draftIndex, cancelled: false };
|
||||
obs.settles.set(draftIndex, pending);
|
||||
|
||||
enqueueSettle(obs, async () => {
|
||||
if (pending.cancelled) return;
|
||||
try {
|
||||
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
|
||||
} catch (err) {
|
||||
// Recoverable: stop-time repair still gives the step a landing page.
|
||||
const op = steps[draftIndex]?.op ?? "?";
|
||||
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
|
||||
} finally {
|
||||
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
export function cancelPendingSettles(obs: RecordingObservationState): void {
|
||||
for (const pending of obs.settles.values()) pending.cancelled = true;
|
||||
obs.settles.clear();
|
||||
}
|
||||
|
||||
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
|
||||
const MAX_SETTLE_FLUSH_ROUNDS = 10;
|
||||
|
||||
/**
|
||||
* Drain settle work before exporting, including work queued while draining —
|
||||
* a navigation observed during the last capture schedules another one, and
|
||||
* the trace would drop it if stop did not wait.
|
||||
*/
|
||||
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
|
||||
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
|
||||
const drained = obs.settleQueue;
|
||||
await drained;
|
||||
if (obs.settleQueue === drained) return;
|
||||
}
|
||||
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
|
||||
}
|
||||
|
||||
function finalizeStateBodies(
|
||||
entries: StateRegistryEntry[],
|
||||
annotationsByState: Map<string, ObservationAnnotation[]>,
|
||||
stepIdByDraftId: Map<number, number>,
|
||||
idByOldId: Map<string, string>,
|
||||
): TraceState[] {
|
||||
return entries.map((entry) => {
|
||||
const id = idByOldId.get(entry.id) ?? entry.id;
|
||||
const body = formatObservationFile({
|
||||
stateId: id,
|
||||
url: entry.url,
|
||||
title: entry.title,
|
||||
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
|
||||
body: entry.rawVomText,
|
||||
annotations: annotationsByState.get(entry.id) ?? [],
|
||||
});
|
||||
return {
|
||||
id,
|
||||
url: entry.url,
|
||||
...(entry.title ? { title: entry.title } : {}),
|
||||
body,
|
||||
...(entry.truncated ? { truncated: true } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Draft ids only become step ids after collapsing and filtering. */
|
||||
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||
const mapped = new Set<number>();
|
||||
for (const draftId of draftIds) {
|
||||
const stepId = stepIdByDraftId.get(draftId);
|
||||
if (stepId !== undefined) mapped.add(stepId);
|
||||
}
|
||||
return [...mapped].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function collectAnnotationsByState(
|
||||
obs: RecordingObservationState,
|
||||
stepIdByDraftId: Map<number, number>,
|
||||
): Map<string, ObservationAnnotation[]> {
|
||||
const byState = new Map<string, ObservationAnnotation[]>();
|
||||
for (const anns of obs.stepAnnotations.values()) {
|
||||
for (const ann of anns) {
|
||||
const stepId = stepIdByDraftId.get(ann.stepId);
|
||||
if (stepId === undefined) continue;
|
||||
const bucket = byState.get(ann.stateId) ?? [];
|
||||
bucket.push({ ...ann, stepId });
|
||||
byState.set(ann.stateId, bucket);
|
||||
}
|
||||
}
|
||||
return byState;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the pages the published steps point at. Redirect hops and
|
||||
* mid-load captures land in the registry too, and shipping them would invite
|
||||
* a reader to treat a page the flow merely passed through as a real stop.
|
||||
* With no steps at all the first observation is the whole artifact, so it
|
||||
* stays.
|
||||
*/
|
||||
function selectPublishedStates(
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
steps: StepV3[],
|
||||
): StateRegistryEntry[] {
|
||||
const entries = [...registry.values()];
|
||||
if (steps.length === 0) return entries.slice(0, 1);
|
||||
const referenced = new Set<string>();
|
||||
for (const step of steps) {
|
||||
referenced.add(step.state);
|
||||
referenced.add(step.result.state);
|
||||
}
|
||||
return entries.filter((entry) => referenced.has(entry.id));
|
||||
}
|
||||
|
||||
export function buildTraceV3(input: {
|
||||
obs: RecordingObservationState;
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
}): TraceV3 {
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||
// Renumber so the shipped dictionary reads s1..sN without holes where the
|
||||
// dropped captures used to be.
|
||||
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||
for (const step of steps) {
|
||||
step.state = idByOldId.get(step.state) ?? step.state;
|
||||
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
|
||||
}
|
||||
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
|
||||
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
|
||||
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
|
||||
return {
|
||||
version: 3,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
stopped_by: input.stoppedBy,
|
||||
entry: { start_url: startUrl },
|
||||
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||
states,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||
import type { DraftTraceStep, KeyModifier } from "@/transport/types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export interface TargetDescriptorV2 {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface SelectedOptionV2 {
|
||||
value: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface StepEffectV2 {
|
||||
navigated_to: string;
|
||||
}
|
||||
|
||||
export type StepV2 =
|
||||
| { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 }
|
||||
| { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 }
|
||||
| { op: "hover"; id: number; page: string; target: TargetDescriptorV2 }
|
||||
| {
|
||||
op: "fill";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
effect?: StepEffectV2;
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
selection: SelectedOptionV2[];
|
||||
effect?: StepEffectV2;
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
id: number;
|
||||
page: string;
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptorV2;
|
||||
effect?: StepEffectV2;
|
||||
};
|
||||
|
||||
export interface TraceV2 {
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
entry: { start_url: string };
|
||||
pages: PageRef[];
|
||||
steps: StepV2[];
|
||||
}
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "scroll") return false;
|
||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
for (const step of steps) {
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
out[out.length - 1] = step;
|
||||
continue;
|
||||
}
|
||||
out.push(step);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
if (step.op === "navigate") {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
for (const url of urls) {
|
||||
if (!url || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
unique.push(url);
|
||||
}
|
||||
return unique;
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
const id = `p${index + 1}`;
|
||||
urlToId.set(url, id);
|
||||
return { id, url };
|
||||
});
|
||||
return { pages, urlToId };
|
||||
}
|
||||
|
||||
function pageIdFor(
|
||||
url: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): string {
|
||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): StepEffectV2 | undefined {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 {
|
||||
return {
|
||||
tag: capture?.tag ?? "unknown",
|
||||
...(capture?.role ? { role: capture.role } : {}),
|
||||
...(capture?.name ? { name: capture.name } : {}),
|
||||
...(capture?.name_attr ? { name_attr: capture.name_attr } : {}),
|
||||
...(capture?.placeholder ? { placeholder: capture.placeholder } : {}),
|
||||
...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined {
|
||||
if (!("target" in step) && !("captureTarget" in step)) return undefined;
|
||||
const capture = "captureTarget" in step ? step.captureTarget : undefined;
|
||||
if (capture) return captureTargetToV2Target(capture);
|
||||
if ("target" in step && step.target && "tag" in (step.target as object)) {
|
||||
const legacy = step.target as TargetDescriptorV2 & { tag?: string };
|
||||
if (legacy.tag) return legacy;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): StepV2 | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
||||
const effect =
|
||||
"navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined;
|
||||
|
||||
switch (step.op) {
|
||||
case "navigate":
|
||||
return {
|
||||
op: "navigate",
|
||||
id,
|
||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
||||
to: step.url,
|
||||
};
|
||||
case "click": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return { op: "click", id, page, target, ...(effect ? { effect } : {}) };
|
||||
}
|
||||
case "hover": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return { op: "hover", id, page, target };
|
||||
}
|
||||
case "fill": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
}
|
||||
case "press":
|
||||
return {
|
||||
op: "press",
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: targetForDraft(step) } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
...(effect ? { effect } : {}),
|
||||
};
|
||||
case "select": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return {
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
...(effect ? { effect } : {}),
|
||||
};
|
||||
}
|
||||
case "scroll":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface BuildTraceV2Input {
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
}
|
||||
|
||||
export function buildTraceV2(input: BuildTraceV2Input): TraceV2 {
|
||||
const collapsed = collapseNavigations(input.steps);
|
||||
const startUrl =
|
||||
input.startUrl ??
|
||||
collapsed.find(
|
||||
(step): step is Extract<DraftTraceStep, { op: "navigate" }> => step.op === "navigate",
|
||||
)?.url ??
|
||||
collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ??
|
||||
"about:blank";
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: StepV2[] = [];
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
id += 1;
|
||||
}
|
||||
return {
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps: out,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user