mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): record user flows as trace v3 page-state observations
Capture settled page observations into a states/steps trace, match targets by geometry with capture fallbacks, and reuse VOM rendering with richer refs and optional value redaction for record bundles.
This commit is contained in:
1 parent
1508839bf9
commit
13a61b2688
28 files changed
+3993
-486
No files matched your search
@@ -1,5 +1,5 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||||
|
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
@@ -96,6 +96,65 @@ describe("handleRecordContentMessage stop/cancel", () => {
|
|||||||
expect(dispose).not.toHaveBeenCalled();
|
expect(dispose).not.toHaveBeenCalled();
|
||||||
expect(onStop).not.toHaveBeenCalled();
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
|
||||||
|
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
|
||||||
|
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
|
||||||
|
sendMessage.mockReset();
|
||||||
|
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
|
||||||
|
sendMessage.mockResolvedValueOnce(undefined);
|
||||||
|
|
||||||
|
let activeRequestId: string | null = null;
|
||||||
|
let capture: ReturnType<typeof startRecordCapture> | null = null;
|
||||||
|
const onStop = vi.fn();
|
||||||
|
const dispatch = (
|
||||||
|
message:
|
||||||
|
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
|
||||||
|
| { type: typeof RECORD_STOP; requestId: string },
|
||||||
|
sendResponse?: (response: unknown) => void,
|
||||||
|
) =>
|
||||||
|
handleRecordContentMessage(
|
||||||
|
message,
|
||||||
|
{
|
||||||
|
activeRequestId,
|
||||||
|
capture,
|
||||||
|
setActiveRequestId: (id) => {
|
||||||
|
activeRequestId = id;
|
||||||
|
},
|
||||||
|
setCapture: (next) => {
|
||||||
|
capture = next;
|
||||||
|
},
|
||||||
|
onStart: vi.fn(),
|
||||||
|
onStop,
|
||||||
|
},
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
dispatch({ type: RECORD_START, requestId: "rec-retry" });
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "dirty final value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
|
||||||
|
const firstResponse = vi.fn();
|
||||||
|
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(firstResponse).toHaveBeenCalledWith({
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(activeRequestId).toBe("rec-retry");
|
||||||
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const retryResponse = vi.fn();
|
||||||
|
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
|
||||||
|
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
|
||||||
|
|
||||||
|
expect(sendMessage).toHaveBeenCalledTimes(2);
|
||||||
|
expect(activeRequestId).toBeNull();
|
||||||
|
expect(onStop).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("record-capture semantic", () => {
|
describe("record-capture semantic", () => {
|
||||||
@@ -228,6 +287,7 @@ describe("record-capture semantic", () => {
|
|||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { role: "button", name: "Open user navigation menu" },
|
target: { role: "button", name: "Open user navigation menu" },
|
||||||
|
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||||
});
|
});
|
||||||
expect(steps[1]).toMatchObject({
|
expect(steps[1]).toMatchObject({
|
||||||
op: "click",
|
op: "click",
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import {
|
import {
|
||||||
|
type CaptureTargetDescriptor,
|
||||||
describeEventTarget,
|
describeEventTarget,
|
||||||
describeTarget,
|
describeTarget,
|
||||||
resolveClickableElement,
|
resolveClickableElement,
|
||||||
resolveHoverElement,
|
resolveHoverElement,
|
||||||
type TargetDescriptor,
|
|
||||||
} from "@/lib/describe-target";
|
} from "@/lib/describe-target";
|
||||||
import {
|
import {
|
||||||
evaluateHoverTrigger,
|
evaluateHoverTrigger,
|
||||||
@@ -39,17 +39,26 @@ import {
|
|||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
const failedStepDeliveries = new Set<string>();
|
const failedStepDeliveries = new Set<string>();
|
||||||
|
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
|
||||||
|
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
|
||||||
const knownRecordRequests = new Set<string>();
|
const knownRecordRequests = new Set<string>();
|
||||||
|
|
||||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
|
||||||
const payload = { type: RECORD_STEP, requestId, step };
|
const payload = { type: RECORD_STEP, requestId, step };
|
||||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||||
() => true,
|
() => true,
|
||||||
() => {
|
() => {
|
||||||
failedStepDeliveries.add(requestId);
|
failedStepDeliveries.add(requestId);
|
||||||
|
const failed = failedStepPayloads.get(requestId) ?? [];
|
||||||
|
failed.push(step);
|
||||||
|
failedStepPayloads.set(requestId, failed);
|
||||||
return false;
|
return false;
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||||
|
const pending = deliverRecordStep(requestId, step);
|
||||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||||
sends.add(pending);
|
sends.add(pending);
|
||||||
pendingStepSends.set(requestId, sends);
|
pendingStepSends.set(requestId, sends);
|
||||||
@@ -65,14 +74,15 @@ export interface RecordCaptureController {
|
|||||||
|
|
||||||
interface FillSession {
|
interface FillSession {
|
||||||
element: FillableElement;
|
element: FillableElement;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
baselineValue: string;
|
baselineValue: string;
|
||||||
lastValue: string;
|
lastValue: string;
|
||||||
|
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
element: Element;
|
element: Element;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
recordedAt: number;
|
recordedAt: number;
|
||||||
score: number;
|
score: number;
|
||||||
eligible: boolean;
|
eligible: boolean;
|
||||||
@@ -157,6 +167,28 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
const style = window.getComputedStyle(el);
|
||||||
|
return {
|
||||||
|
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||||
|
scrollX: window.scrollX,
|
||||||
|
scrollY: window.scrollY,
|
||||||
|
position: style.position,
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function geometryForEventTarget(
|
||||||
|
target: EventTarget | null,
|
||||||
|
): RecordStepPayload["geometry"] | undefined {
|
||||||
|
if (!(target instanceof Element)) return undefined;
|
||||||
|
const clickable = target.closest(
|
||||||
|
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||||
|
);
|
||||||
|
return captureGeometry(clickable ?? target);
|
||||||
|
}
|
||||||
|
|
||||||
function fillableValue(el: FillableElement): string {
|
function fillableValue(el: FillableElement): string {
|
||||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||||
return el.value;
|
return el.value;
|
||||||
@@ -232,7 +264,7 @@ function collectHoverTriggerLabelText(root: Element): string {
|
|||||||
return normalizeLabelText(text);
|
return normalizeLabelText(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
function compactHoverTargetName(el: Element, desc: CaptureTargetDescriptor): CaptureTargetDescriptor {
|
||||||
if (!desc.name) return desc;
|
if (!desc.name) return desc;
|
||||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
const compactName = collectHoverTriggerLabelText(el);
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
@@ -250,7 +282,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
return !target.role && !target.name && target.tag === "div";
|
return !target.role && !target.name && target.tag === "div";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,9 +293,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
|||||||
|
|
||||||
function normalizeHoverTarget(
|
function normalizeHoverTarget(
|
||||||
el: Element,
|
el: Element,
|
||||||
desc: TargetDescriptor,
|
desc: CaptureTargetDescriptor,
|
||||||
decision: HoverTriggerDecision,
|
decision: HoverTriggerDecision,
|
||||||
): TargetDescriptor {
|
): CaptureTargetDescriptor {
|
||||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
return { ...desc, name: "image" };
|
return { ...desc, name: "image" };
|
||||||
}
|
}
|
||||||
@@ -277,7 +309,7 @@ function normalizeHoverTarget(
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||||
if (!(el instanceof HTMLElement)) return null;
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
const style = hoverTriggerStyle(el);
|
const style = hoverTriggerStyle(el);
|
||||||
return {
|
return {
|
||||||
@@ -411,14 +443,17 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: session.target,
|
target: session.target,
|
||||||
|
geometry: captureGeometry(session.element),
|
||||||
value,
|
value,
|
||||||
|
commit: session.pendingCommit ?? "blur",
|
||||||
...(isPassword ? { redacted: true } : {}),
|
...(isPassword ? { redacted: true } : {}),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const commitFillSession = () => {
|
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||||
if (!fillSession || composing) return;
|
if (!fillSession || composing) return;
|
||||||
const session = fillSession;
|
const session = fillSession;
|
||||||
|
session.pendingCommit = commit;
|
||||||
fillSession = null;
|
fillSession = null;
|
||||||
emitFill(session);
|
emitFill(session);
|
||||||
committedValues.set(session.element, session.lastValue);
|
committedValues.set(session.element, session.lastValue);
|
||||||
@@ -624,6 +659,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: hover.target,
|
target: hover.target,
|
||||||
|
geometry: captureGeometry(hover.element),
|
||||||
});
|
});
|
||||||
emittedHoverElements.add(hover.element);
|
emittedHoverElements.add(hover.element);
|
||||||
};
|
};
|
||||||
@@ -678,6 +714,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "click",
|
op: "click",
|
||||||
target,
|
target,
|
||||||
|
geometry: geometryForEventTarget(eventTarget(event)),
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
@@ -779,7 +816,7 @@ export function startRecordCapture(
|
|||||||
scheduleInputCompletionCommit(
|
scheduleInputCompletionCommit(
|
||||||
sessionElement,
|
sessionElement,
|
||||||
syncFillSessionValue,
|
syncFillSessionValue,
|
||||||
commitFillSession,
|
() => commitFillSession("suggestion"),
|
||||||
(el) => fillSession?.element === el,
|
(el) => fillSession?.element === el,
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
@@ -839,6 +876,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "select",
|
op: "select",
|
||||||
target: desc,
|
target: desc,
|
||||||
|
geometry: captureGeometry(target),
|
||||||
values,
|
values,
|
||||||
labels,
|
labels,
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
@@ -879,7 +917,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
commitFillSession();
|
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
@@ -887,7 +925,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
...(desc ? { target: desc } : {}),
|
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||||
...(modifiers.length ? { modifiers } : {}),
|
...(modifiers.length ? { modifiers } : {}),
|
||||||
expects_navigation: event.key === "Enter",
|
expects_navigation: event.key === "Enter",
|
||||||
});
|
});
|
||||||
@@ -906,9 +944,12 @@ export function startRecordCapture(
|
|||||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||||
urlObserver.observe(document, { subtree: true, childList: true });
|
urlObserver.observe(document, { subtree: true, childList: true });
|
||||||
const onUrlEvent = () => emitNavigateIfChanged();
|
const onUrlEvent = () => emitNavigateIfChanged();
|
||||||
|
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||||
|
// the `commit` argument and stamp it onto the recorded fill step.
|
||||||
|
const onPageHide = () => commitFillSession();
|
||||||
window.addEventListener("hashchange", onUrlEvent);
|
window.addEventListener("hashchange", onUrlEvent);
|
||||||
window.addEventListener("popstate", onUrlEvent);
|
window.addEventListener("popstate", onUrlEvent);
|
||||||
window.addEventListener("pagehide", commitFillSession);
|
window.addEventListener("pagehide", onPageHide);
|
||||||
|
|
||||||
const originalPushState = history.pushState;
|
const originalPushState = history.pushState;
|
||||||
const originalReplaceState = history.replaceState;
|
const originalReplaceState = history.replaceState;
|
||||||
@@ -936,7 +977,7 @@ export function startRecordCapture(
|
|||||||
urlObserver.disconnect();
|
urlObserver.disconnect();
|
||||||
window.removeEventListener("hashchange", onUrlEvent);
|
window.removeEventListener("hashchange", onUrlEvent);
|
||||||
window.removeEventListener("popstate", onUrlEvent);
|
window.removeEventListener("popstate", onUrlEvent);
|
||||||
window.removeEventListener("pagehide", commitFillSession);
|
window.removeEventListener("pagehide", onPageHide);
|
||||||
history.pushState = originalPushState;
|
history.pushState = originalPushState;
|
||||||
history.replaceState = originalReplaceState;
|
history.replaceState = originalReplaceState;
|
||||||
},
|
},
|
||||||
@@ -965,6 +1006,7 @@ export function handleRecordContentMessage(
|
|||||||
if (!knownRecordRequests.has(message.requestId)) {
|
if (!knownRecordRequests.has(message.requestId)) {
|
||||||
knownRecordRequests.add(message.requestId);
|
knownRecordRequests.add(message.requestId);
|
||||||
failedStepDeliveries.delete(message.requestId);
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
}
|
}
|
||||||
state.capture?.dispose();
|
state.capture?.dispose();
|
||||||
state.setCapture(
|
state.setCapture(
|
||||||
@@ -992,27 +1034,51 @@ export function handleRecordContentMessage(
|
|||||||
state.setActiveRequestId(null);
|
state.setActiveRequestId(null);
|
||||||
};
|
};
|
||||||
if (isRecordStopMessage(message) && sendResponse) {
|
if (isRecordStopMessage(message) && sendResponse) {
|
||||||
|
const existingFlush = pendingStopFlushes.get(message.requestId);
|
||||||
|
if (existingFlush) {
|
||||||
|
void existingFlush.then(sendResponse);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
|
||||||
|
if (failedBeforeStop.length > 0) {
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
}
|
||||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||||
void Promise.all(pending).then((delivered) => {
|
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
|
||||||
finishStop();
|
const retried = await Promise.all(
|
||||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
|
||||||
|
);
|
||||||
|
const succeeded =
|
||||||
|
delivered.every(Boolean) &&
|
||||||
|
retried.every(Boolean) &&
|
||||||
|
!failedStepDeliveries.has(message.requestId);
|
||||||
if (succeeded) {
|
if (succeeded) {
|
||||||
failedStepDeliveries.delete(message.requestId);
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
knownRecordRequests.delete(message.requestId);
|
knownRecordRequests.delete(message.requestId);
|
||||||
|
finishStop();
|
||||||
|
}
|
||||||
|
return succeeded
|
||||||
|
? { ok: true }
|
||||||
|
: {
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
};
|
||||||
|
});
|
||||||
|
pendingStopFlushes.set(message.requestId, flush);
|
||||||
|
void flush.then(sendResponse).finally(() => {
|
||||||
|
if (pendingStopFlushes.get(message.requestId) === flush) {
|
||||||
|
pendingStopFlushes.delete(message.requestId);
|
||||||
}
|
}
|
||||||
sendResponse(
|
|
||||||
succeeded
|
|
||||||
? { ok: true }
|
|
||||||
: {
|
|
||||||
ok: false,
|
|
||||||
error: "failed to deliver one or more recorded steps",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (isRecordCancelMessage(message)) {
|
if (isRecordCancelMessage(message)) {
|
||||||
failedStepDeliveries.delete(message.requestId);
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
pendingStopFlushes.delete(message.requestId);
|
||||||
knownRecordRequests.delete(message.requestId);
|
knownRecordRequests.delete(message.requestId);
|
||||||
}
|
}
|
||||||
finishStop();
|
finishStop();
|
||||||
|
|||||||
@@ -172,6 +172,7 @@ export default defineBackground(() => {
|
|||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
const recordDeps = {
|
const recordDeps = {
|
||||||
tabsApi: chrome.tabs,
|
tabsApi: chrome.tabs,
|
||||||
|
cdp,
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
@@ -188,7 +189,7 @@ export default defineBackground(() => {
|
|||||||
// Message listeners stay up (cheap; fire only for record message types).
|
// Message listeners stay up (cheap; fire only for record message types).
|
||||||
// Tab / webNavigation observation attaches lazily while a recording is
|
// Tab / webNavigation observation attaches lazily while a recording is
|
||||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||||
attachRecordStepListener();
|
attachRecordStepListener(recordDeps);
|
||||||
attachRecordFinishListener(recordDeps);
|
attachRecordFinishListener(recordDeps);
|
||||||
attachRecordQueryListener(recordDeps);
|
attachRecordQueryListener(recordDeps);
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import { matchTarget } from "../match-target";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "button",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchTarget", () => {
|
||||||
|
const refs: RenderedRef[] = [
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
|
||||||
|
];
|
||||||
|
|
||||||
|
it("matches a unique node by viewport coordinates", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42)],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a non-fixed node after the top frame scrolls", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 40,
|
||||||
|
scrollY: 300,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, {
|
||||||
|
documentRect: { x: 50, y: 320, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a static node by document coordinates when scroll changed after observation", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 250,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, {
|
||||||
|
documentRect: { x: 10, y: 300, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 200, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 200, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched when multiple candidates match", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42), node(43)],
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses viewport coordinates for fixed elements", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 5, y: 5, w: 80, h: 24 },
|
||||||
|
scrollX: 100,
|
||||||
|
scrollY: 200,
|
||||||
|
position: "fixed",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
import { beforeEach, describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { DraftTraceStep } from "@/transport/types";
|
||||||
|
import { resetStateIdCounterForTests } from "../record-constants";
|
||||||
|
import {
|
||||||
|
applyTargetMatching,
|
||||||
|
buildTraceV3,
|
||||||
|
createObservationState,
|
||||||
|
rememberStepOnPage,
|
||||||
|
} from "../record-observation";
|
||||||
|
import { registerObservation } from "../trace-reducer";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function capturedInput(): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId: 42,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
const obs = createObservationState({ redactValues });
|
||||||
|
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
|
||||||
|
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
|
||||||
|
obs.lastSettled = {
|
||||||
|
stateId,
|
||||||
|
captured: [capturedInput()],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
url: URL,
|
||||||
|
vomText: rawVomText,
|
||||||
|
};
|
||||||
|
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "fill",
|
||||||
|
target: { unmatched: true },
|
||||||
|
value,
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "input",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
applyTargetMatching(obs, draft, 1);
|
||||||
|
draft.postStateId = stateId;
|
||||||
|
rememberStepOnPage(obs, draft, 1);
|
||||||
|
|
||||||
|
return (
|
||||||
|
buildTraceV3({
|
||||||
|
obs,
|
||||||
|
steps: [draft],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
}).states[0]?.body ?? ""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("omits a fill literal from finalized state bodies when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const body = finalizedFillBody(secret, true);
|
||||||
|
|
||||||
|
expect(body).toContain("step 1: fill");
|
||||||
|
expect(body).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps fill details in ordinary recording annotations", () => {
|
||||||
|
const value = "ordinary text";
|
||||||
|
const body = finalizedFillBody(value, false);
|
||||||
|
|
||||||
|
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("applyTargetMatching without a settled observation", () => {
|
||||||
|
it("keeps the capture role/name instead of a bare unmatched target", () => {
|
||||||
|
const obs = createObservationState();
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
applyTargetMatching(obs, draft, 1);
|
||||||
|
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect(draft.target).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "新建",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -12,12 +12,42 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(buffer.pendingNavigation).toBe(true);
|
expect(buffer.pendingNavigation).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps the hovered element description as a capture fallback", () => {
|
||||||
|
const buffer = { steps: [], pendingNavigation: false };
|
||||||
|
appendRecordedPayload(buffer, {
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||||
const buffer = {
|
const buffer = {
|
||||||
steps: [
|
steps: [
|
||||||
@@ -71,10 +101,41 @@ describe("recording-step-buffer", () => {
|
|||||||
currentUrl: "https://example.com/a",
|
currentUrl: "https://example.com/a",
|
||||||
pendingNavigation: false,
|
pendingNavigation: false,
|
||||||
};
|
};
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||||
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
expect(buffer.steps[0]).toMatchObject({
|
expect(buffer.steps[0]).toMatchObject({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url: "https://example.com/b",
|
url: "https://example.com/b",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
currentUrl: "https://passport.example/login",
|
||||||
|
pendingNavigation: false,
|
||||||
|
};
|
||||||
|
const hop1 = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
"https://passport.example/callback",
|
||||||
|
false,
|
||||||
|
"link",
|
||||||
|
["server_redirect"],
|
||||||
|
);
|
||||||
|
expect(hop1).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://passport.example/callback",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
|
||||||
|
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||||
|
"client_redirect",
|
||||||
|
]);
|
||||||
|
expect(hop2).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://app.example/dashboard",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -1,6 +1,11 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { DraftTraceStep } from "@/transport/types";
|
import { resetStateIdCounterForTests } from "@/lib/record-constants";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
import {
|
||||||
|
reduceTraceSteps,
|
||||||
|
registerObservation,
|
||||||
|
resolveTraceStartUrl,
|
||||||
|
shouldRecordPress,
|
||||||
|
} from "../trace-reducer";
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
describe("shouldRecordPress", () => {
|
||||||
it("keeps Enter and Escape", () => {
|
it("keeps Enter and Escape", () => {
|
||||||
@@ -17,86 +22,85 @@ describe("shouldRecordPress", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
describe("reduceTraceSteps v3", () => {
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
it("builds steps with state/result references and deduped states", () => {
|
||||||
const drafts: DraftTraceStep[] = [
|
resetStateIdCounterForTests();
|
||||||
{
|
const registry = new Map();
|
||||||
op: "navigate",
|
const body = "@vom 1\nL1 page";
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
const s1 = registerObservation(registry, {
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
url: "https://example.com/search",
|
||||||
},
|
rawVomText: body,
|
||||||
{
|
});
|
||||||
op: "fill",
|
const s2 = registerObservation(registry, {
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
url: "https://example.com/results/42",
|
||||||
value: "browser skill",
|
rawVomText: "@vom 1\nL1 results",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "press",
|
|
||||||
key: "Enter",
|
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
|
||||||
navigated_to: "https://example.com/results/42",
|
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
navigated_to: "https://example.com/p/99",
|
|
||||||
page_url: "https://example.com/results/42",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "press",
|
|
||||||
key: "a",
|
|
||||||
page_url: "https://example.com/p/99",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const { pages, steps } = reduceTraceSteps(
|
|
||||||
drafts,
|
|
||||||
"https://example.com/search?q=hello&utm_source=x",
|
|
||||||
);
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("parameters");
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("intent");
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("summary");
|
|
||||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
|
||||||
expect(pages.map((p) => p.url)).toEqual([
|
|
||||||
"https://example.com/search?q=hello&utm_source=x",
|
|
||||||
"https://example.com/results/42",
|
|
||||||
"https://example.com/p/99",
|
|
||||||
]);
|
|
||||||
|
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
id: 1,
|
|
||||||
op: "navigate",
|
|
||||||
page: "p1",
|
|
||||||
to: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const { states, steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/search",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||||
|
value: "browser skill",
|
||||||
|
commit: "enter",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "press",
|
||||||
|
key: "Enter",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s2,
|
||||||
|
navigated_to: "https://example.com/results/42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { ref: "e2", role: "button", name: "发布" },
|
||||||
|
preStateId: s2,
|
||||||
|
postStateId: s2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(states).toHaveLength(2);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||||
expect(steps[1]).toMatchObject({
|
expect(steps[1]).toMatchObject({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
page: "p1",
|
state: s1,
|
||||||
value: "browser skill",
|
result: { state: s1 },
|
||||||
|
commit: "enter",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(steps[2]).toMatchObject({
|
expect(steps[2]).toMatchObject({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "Enter",
|
state: s1,
|
||||||
page: "p1",
|
result: { state: s2 },
|
||||||
effect: { navigated_to: "p2" },
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(steps[3]).toMatchObject({
|
|
||||||
op: "click",
|
|
||||||
page: "p2",
|
|
||||||
effect: { navigated_to: "p3" },
|
|
||||||
});
|
});
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("page");
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("effect");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("collapses consecutive navigations", () => {
|
it("collapses consecutive navigations", () => {
|
||||||
const { steps } = reduceTraceSteps([
|
resetStateIdCounterForTests();
|
||||||
{ op: "navigate", url: "https://a.example/redirect1" },
|
const registry = new Map();
|
||||||
{ op: "navigate", url: "https://a.example/final" },
|
const s1 = registerObservation(registry, {
|
||||||
]);
|
url: "https://a.example/redirect1",
|
||||||
|
rawVomText: "a",
|
||||||
|
});
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
expect(steps).toHaveLength(1);
|
expect(steps).toHaveLength(1);
|
||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
@@ -104,52 +108,107 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const start = registerObservation(registry, {
|
||||||
|
url: "https://example.com/",
|
||||||
|
rawVomText: "start",
|
||||||
|
});
|
||||||
|
const hop = registerObservation(registry, {
|
||||||
|
url: "https://iwiki.woa.com/",
|
||||||
|
rawVomText: "hop",
|
||||||
|
});
|
||||||
|
const landing = registerObservation(registry, {
|
||||||
|
url: "https://iwiki.woa.com/dashboard",
|
||||||
|
rawVomText: "landing",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "navigate",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
url: "https://iwiki.woa.com/",
|
||||||
values: ["tech"],
|
preStateId: start,
|
||||||
labels: ["技术"],
|
postStateId: hop,
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
transitionType: "typed",
|
||||||
page_url: "https://example.com/list",
|
},
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://iwiki.woa.com/dashboard",
|
||||||
|
preStateId: hop,
|
||||||
|
postStateId: landing,
|
||||||
|
transitionType: "link",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/list",
|
registry,
|
||||||
);
|
);
|
||||||
expect(pages.map((p) => p.url)).toEqual([
|
|
||||||
"https://example.com/list",
|
expect(steps).toHaveLength(1);
|
||||||
"https://example.com/list?cat=tech",
|
|
||||||
]);
|
|
||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "select",
|
op: "navigate",
|
||||||
page: "p1",
|
id: 1,
|
||||||
selection: [{ value: "tech", label: "技术" }],
|
state: start,
|
||||||
effect: { navigated_to: "p2" },
|
result: { state: landing },
|
||||||
|
to: "https://iwiki.woa.com/dashboard",
|
||||||
|
cause: "user_typed",
|
||||||
});
|
});
|
||||||
|
// Both drafts fold into step 1, so anything tagged with either draft id
|
||||||
|
// must resolve to it.
|
||||||
|
expect(stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps draft ids to published step ids across dropped drafts", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||||
|
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
// Dropped: blank fill.
|
||||||
|
{ op: "fill", target: { ref: "e1" }, value: " ", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps.map((step) => step.op)).toEqual(["click"]);
|
||||||
|
expect(stepIdByDraftId.get(1)).toBeUndefined();
|
||||||
|
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps hover steps before menu clicks", () => {
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const state = registerObservation(registry, {
|
||||||
|
url: "https://example.com/app",
|
||||||
|
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
|
||||||
|
});
|
||||||
const { steps } = reduceTraceSteps(
|
const { steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { tag: "span", role: "button", name: "Account" },
|
target: { ref: "e1", role: "button", name: "Account" },
|
||||||
page_url: "https://example.com/app",
|
page_url: "https://example.com/app",
|
||||||
|
preStateId: state,
|
||||||
|
postStateId: state,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "a", role: "link", name: "Profile" },
|
target: { ref: "e2", role: "link", name: "Profile" },
|
||||||
page_url: "https://example.com/app",
|
page_url: "https://example.com/app",
|
||||||
|
preStateId: state,
|
||||||
|
postStateId: state,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/app",
|
registry,
|
||||||
);
|
);
|
||||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
page: "p1",
|
state,
|
||||||
|
result: { state },
|
||||||
target: { name: "Account" },
|
target: { name: "Account" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -157,7 +216,7 @@ describe("reduceTraceSteps", () => {
|
|||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
resolveTraceStartUrl(
|
||||||
[{ op: "navigate", url: "https://example.com/other" }],
|
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
|
||||||
"https://example.com/start",
|
"https://example.com/start",
|
||||||
),
|
),
|
||||||
).toBe("https://example.com/start");
|
).toBe("https://example.com/start");
|
||||||
|
|||||||
@@ -6,7 +6,8 @@
|
|||||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
/** Content-script capture descriptor before VOM geometric matching. */
|
||||||
|
export interface CaptureTargetDescriptor {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
|||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
* Recording “点击div” with no name fails this bar.
|
* Recording “点击div” with no name fails this bar.
|
||||||
*/
|
*/
|
||||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
const name = target.name?.trim();
|
const name = target.name?.trim();
|
||||||
if (name && isActionableLabel(name)) return true;
|
if (name && isActionableLabel(name)) return true;
|
||||||
if (
|
if (
|
||||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeTarget(el: Element): TargetDescriptor {
|
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||||
const tag = el.tagName.toLowerCase();
|
const tag = el.tagName.toLowerCase();
|
||||||
const role = inferRole(el);
|
const role = inferRole(el);
|
||||||
const name = accessibleName(el);
|
const name = accessibleName(el);
|
||||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const clickable = resolveClickableElement(target);
|
const clickable = resolveClickableElement(target);
|
||||||
if (!clickable) return null;
|
if (!clickable) return null;
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import type { Step } from "@/transport/types";
|
||||||
|
import { OBSERVATION_FILE_VERSION } from "./record-constants";
|
||||||
|
|
||||||
|
export interface ObservationAnnotation {
|
||||||
|
stepId: number;
|
||||||
|
op: Step["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
|
||||||
|
const suffix = detail ? `: ${detail}` : "";
|
||||||
|
return ` ⟵ step ${stepId}: ${op}${suffix}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Serialize a page observation file (front matter + VOM body + step annotations). */
|
||||||
|
export function formatObservationFile(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepsHere: number[];
|
||||||
|
body: string;
|
||||||
|
annotations?: ObservationAnnotation[];
|
||||||
|
}): string {
|
||||||
|
const lines: string[] = [
|
||||||
|
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
|
||||||
|
`state: ${input.stateId}`,
|
||||||
|
`url: ${input.url}`,
|
||||||
|
];
|
||||||
|
if (input.title) lines.push(`title: ${input.title}`);
|
||||||
|
if (input.stepsHere.length > 0) {
|
||||||
|
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
|
||||||
|
}
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const bodyLines = input.body.split("\n");
|
||||||
|
const annotationMap = new Map<number, ObservationAnnotation[]>();
|
||||||
|
for (const ann of input.annotations ?? []) {
|
||||||
|
const bucket = annotationMap.get(ann.line) ?? [];
|
||||||
|
bucket.push(ann);
|
||||||
|
annotationMap.set(ann.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < bodyLines.length; i += 1) {
|
||||||
|
let line = bodyLines[i] ?? "";
|
||||||
|
const anns = annotationMap.get(i);
|
||||||
|
if (anns) {
|
||||||
|
for (const ann of anns) {
|
||||||
|
line += formatAnnotation(ann);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Hash input: VOM body **before** annotations are inserted. */
|
||||||
|
export function observationBodyForHash(vomText: string): string {
|
||||||
|
return vomText;
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
|
||||||
|
|
||||||
|
export interface MatchTargetInput {
|
||||||
|
geometry: CaptureGeometry;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
refs: RenderedRef[];
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}
|
||||||
|
|
||||||
|
function withinTolerance(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectsMatch(
|
||||||
|
a: { x: number; y: number; w: number; h: number },
|
||||||
|
b: { x: number; y: number; w: number; h: number },
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
withinTolerance(a.x, b.x) &&
|
||||||
|
withinTolerance(a.y, b.y) &&
|
||||||
|
withinTolerance(a.w, b.w) &&
|
||||||
|
withinTolerance(a.h, b.h)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeViewportRect(
|
||||||
|
node: CapturedNode,
|
||||||
|
): { x: number; y: number; w: number; h: number } | null {
|
||||||
|
return node.localRect ?? node.rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchingRects(
|
||||||
|
geometry: CaptureGeometry,
|
||||||
|
node: CapturedNode,
|
||||||
|
): {
|
||||||
|
target: { x: number; y: number; w: number; h: number };
|
||||||
|
node: { x: number; y: number; w: number; h: number } | null;
|
||||||
|
} {
|
||||||
|
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
|
||||||
|
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||||
|
if (!topFrame || viewportPosition || !node.documentRect) {
|
||||||
|
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
target: {
|
||||||
|
x: geometry.rect.x + geometry.scrollX,
|
||||||
|
y: geometry.rect.y + geometry.scrollY,
|
||||||
|
w: geometry.rect.w,
|
||||||
|
h: geometry.rect.h,
|
||||||
|
},
|
||||||
|
node: node.documentRect,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function tagMatches(geometryTag: string, nodeTag: string): boolean {
|
||||||
|
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Best description available when the element cannot be located in the VOM. */
|
||||||
|
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
|
||||||
|
if (!fallback) {
|
||||||
|
return { unmatched: true };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...(fallback.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Locate the interacted element in the last settled observation by geometry. */
|
||||||
|
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
|
||||||
|
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
|
||||||
|
|
||||||
|
const candidates = input.captured.filter((node) => {
|
||||||
|
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
|
||||||
|
if (!tagMatches(input.geometry.tag, node.tag)) return false;
|
||||||
|
const rects = matchingRects(input.geometry, node);
|
||||||
|
if (!rects.node) return false;
|
||||||
|
return rectsMatch(rects.target, rects.node);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (candidates.length !== 1) {
|
||||||
|
return fallbackDescriptor(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const backendNodeId = candidates[0]!.backendNodeId;
|
||||||
|
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
|
||||||
|
if (!refEntry) {
|
||||||
|
return fallbackDescriptor(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ref: refEntry.ref,
|
||||||
|
...(refEntry.role ? { role: refEntry.role } : {}),
|
||||||
|
...(refEntry.name ? { name: refEntry.name } : {}),
|
||||||
|
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
// Deciding *when* a page is worth observing.
|
||||||
|
//
|
||||||
|
// A fixed delay cannot serve both a modal that toggles in one frame and a
|
||||||
|
// route change that renders for a second. Instead of guessing, ask the page:
|
||||||
|
// a document that has stopped mutating and is no longer loading is done
|
||||||
|
// reacting to whatever the user just did.
|
||||||
|
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Installed once per document. Records only the timestamp of the last DOM
|
||||||
|
* change so each poll stays O(1) no matter how large the page is.
|
||||||
|
*/
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export type SettleOutcome =
|
||||||
|
/** The page stopped changing on its own. */
|
||||||
|
| "quiet"
|
||||||
|
/** Still changing when the budget ran out; observe it as it is. */
|
||||||
|
| "timeout"
|
||||||
|
/** A newer action took over; this observation is no longer wanted. */
|
||||||
|
| "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
|
||||||
|
// not a reason to give up.
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wait until the page has finished reacting, or until the budget runs out. */
|
||||||
|
export async function waitForPageSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.cancelled?.()) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS);
|
||||||
|
if (options.cancelled?.()) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, tabId);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe) continue;
|
||||||
|
if (probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
* service worker and a tab's content script.
|
* service worker and a tab's content script.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -41,8 +41,8 @@ export interface RecordStartMessage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
|
||||||
target?: TargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||||
@@ -50,12 +50,25 @@ export interface RecordStepPayload {
|
|||||||
labels?: string[];
|
labels?: string[];
|
||||||
url?: string;
|
url?: string;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
|
commit?: "enter" | "suggestion" | "blur";
|
||||||
/** Page URL when the step was captured. */
|
/** Page URL when the step was captured. */
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
/** Document geometry at capture time for background geometric matching. */
|
||||||
|
geometry?: {
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
scrollX: number;
|
||||||
|
scrollY: number;
|
||||||
|
position: string;
|
||||||
|
tag: string;
|
||||||
|
ownerFrameBackendNodeId?: number | null;
|
||||||
|
};
|
||||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||||
expects_navigation?: boolean;
|
expects_navigation?: boolean;
|
||||||
/** Whether an observed URL change was synchronously caused by the action. */
|
/** Whether an observed URL change was synchronously caused by the action. */
|
||||||
navigation_caused_by_action?: boolean;
|
navigation_caused_by_action?: boolean;
|
||||||
|
/** Raw webNavigation transition metadata for cause mapping. */
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepMessage {
|
export interface RecordStepMessage {
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import type { FillCommit, NavigationCause } from "@/transport/types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Floor before a post-action observation may be taken. A page usually has not
|
||||||
|
* started reacting in the first frames after an action, and capturing then
|
||||||
|
* would record the page as it was, not as the action left it.
|
||||||
|
*/
|
||||||
|
export const SETTLE_MIN_MS = 150;
|
||||||
|
|
||||||
|
/** How long the DOM must stop changing before a page counts as settled. */
|
||||||
|
export const SETTLE_QUIET_MS = 250;
|
||||||
|
|
||||||
|
/** Upper bound on waiting for a page to settle; animations never stop. */
|
||||||
|
export const SETTLE_MAX_MS = 2_000;
|
||||||
|
|
||||||
|
/** How often to ask the page whether it has gone quiet. */
|
||||||
|
export const SETTLE_POLL_MS = 60;
|
||||||
|
|
||||||
|
/** Minimum interval between consecutive observations on the same tab. */
|
||||||
|
export const OBSERVATION_MIN_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
/** Delay before retrying a capture that lost its execution context. */
|
||||||
|
export const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
/** Document-coordinate matching tolerance in CSS pixels. */
|
||||||
|
export const GEOM_MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
|
||||||
|
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
|
||||||
|
|
||||||
|
/** VOM observation file format version (front matter header). */
|
||||||
|
export const OBSERVATION_FILE_VERSION = 1;
|
||||||
|
|
||||||
|
/** FNV-1a 64-bit offset basis. */
|
||||||
|
const FNV_OFFSET = 0xcbf29ce484222325n;
|
||||||
|
/** FNV-1a 64-bit prime. */
|
||||||
|
const FNV_PRIME = 0x100000001b3n;
|
||||||
|
|
||||||
|
/** Content-hash for state deduplication (sync, non-cryptographic). */
|
||||||
|
export function fnv1a64(text: string): string {
|
||||||
|
let hash = FNV_OFFSET;
|
||||||
|
for (let i = 0; i < text.length; i += 1) {
|
||||||
|
hash ^= BigInt(text.charCodeAt(i));
|
||||||
|
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
|
||||||
|
}
|
||||||
|
return hash.toString(16).padStart(16, "0");
|
||||||
|
}
|
||||||
|
|
||||||
|
let nextStateSerial = 0;
|
||||||
|
|
||||||
|
/** Monotonic state id generator (`s1`, `s2`, …). */
|
||||||
|
export function nextStateId(): string {
|
||||||
|
nextStateSerial += 1;
|
||||||
|
return `s${nextStateSerial}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test seam: reset the state id counter. */
|
||||||
|
export function resetStateIdCounterForTests(): void {
|
||||||
|
nextStateSerial = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
|
||||||
|
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface NavigationTransitionMeta {
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
navigationActionPending?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
|
||||||
|
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
|
||||||
|
const qualifiers = meta.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
|
||||||
|
const type = meta.transitionType ?? "";
|
||||||
|
const mapped = TRANSITION_TO_CAUSE[type];
|
||||||
|
if (mapped) return mapped;
|
||||||
|
|
||||||
|
if (!type && meta.navigationActionPending === false) return "script";
|
||||||
|
return "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
|
||||||
@@ -0,0 +1,663 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
|
||||||
|
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
|
||||||
|
import { waitForPageSettled } from "@/lib/page-settled";
|
||||||
|
import {
|
||||||
|
CAPTURE_RETRY_DELAY_MS,
|
||||||
|
DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
OBSERVATION_MIN_INTERVAL_MS,
|
||||||
|
} from "@/lib/record-constants";
|
||||||
|
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
|
||||||
|
import { captureVomObservation } from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
|
||||||
|
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||||
|
|
||||||
|
export interface LastSettledObservation {
|
||||||
|
stateId: string;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
refs: RenderedRef[];
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A post-action observation that has been queued but not yet written down. */
|
||||||
|
interface PendingSettle {
|
||||||
|
draftIndex: number;
|
||||||
|
/** Set when a newer action has already decided where this step landed. */
|
||||||
|
cancelled: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
|
||||||
|
export interface PendingRedirectLanding {
|
||||||
|
url: string;
|
||||||
|
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingObservationState {
|
||||||
|
stateRegistry: Map<string, StateRegistryEntry>;
|
||||||
|
lastSettled: LastSettledObservation | null;
|
||||||
|
maxPageTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
lastCaptureAtMs: number;
|
||||||
|
/**
|
||||||
|
* Tail of the settle chain. Observations run one at a time so that a slow
|
||||||
|
* capture can never land after a faster one and report the pages out of the
|
||||||
|
* order the user visited them.
|
||||||
|
*/
|
||||||
|
settleQueue: Promise<void>;
|
||||||
|
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
|
||||||
|
settles: Map<number, PendingSettle>;
|
||||||
|
stepAnnotations: Map<number, ObservationAnnotation[]>;
|
||||||
|
/**
|
||||||
|
* OAuth / server redirects are coalesced here: intermediate hops only update
|
||||||
|
* `url`+`generation`, and one `navigate` is emitted after the page settles.
|
||||||
|
*/
|
||||||
|
pendingRedirect: PendingRedirectLanding | null;
|
||||||
|
redirectFlushQueue: Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createObservationState(options?: {
|
||||||
|
maxPageTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
}): RecordingObservationState {
|
||||||
|
return {
|
||||||
|
stateRegistry: new Map(),
|
||||||
|
lastSettled: null,
|
||||||
|
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
redactValues: options?.redactValues ?? false,
|
||||||
|
lastCaptureAtMs: 0,
|
||||||
|
settleQueue: Promise.resolve(),
|
||||||
|
settles: new Map(),
|
||||||
|
stepAnnotations: new Map(),
|
||||||
|
pendingRedirect: null,
|
||||||
|
redirectFlushQueue: Promise.resolve(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureAndRegisterObservation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
urlOverride?: string,
|
||||||
|
): Promise<LastSettledObservation> {
|
||||||
|
const now = Date.now();
|
||||||
|
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
|
||||||
|
if (waitMs > 0) await sleep(waitMs);
|
||||||
|
|
||||||
|
const { url, title } = urlOverride
|
||||||
|
? { url: urlOverride, title: undefined }
|
||||||
|
: await readTabMeta(tabsApi, tabId);
|
||||||
|
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
|
||||||
|
const resolvedTitle = title ?? meta.title;
|
||||||
|
|
||||||
|
const rendered = await captureVomObservation(cdp, tabId, url, {
|
||||||
|
maxTokens: obs.maxPageTokens,
|
||||||
|
redactValues: obs.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const stateId = registerObservation(obs.stateRegistry, {
|
||||||
|
url,
|
||||||
|
title: resolvedTitle,
|
||||||
|
rawVomText: rendered.text,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
});
|
||||||
|
|
||||||
|
obs.lastCaptureAtMs = Date.now();
|
||||||
|
const settled: LastSettledObservation = {
|
||||||
|
stateId,
|
||||||
|
captured: rendered.captured,
|
||||||
|
refs: rendered.refs,
|
||||||
|
url,
|
||||||
|
title: resolvedTitle,
|
||||||
|
vomText: rendered.text,
|
||||||
|
};
|
||||||
|
obs.lastSettled = settled;
|
||||||
|
return settled;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bind a draft to the page it was performed on: origin state, geometric target
|
||||||
|
* match, and the inline annotation. All three read the *current* observation,
|
||||||
|
* which is only the page the user acted on while the draft is still fresh —
|
||||||
|
* once the action settles, `lastSettled` is the destination page instead.
|
||||||
|
*/
|
||||||
|
export function applyTargetMatching(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
stepId: number,
|
||||||
|
): void {
|
||||||
|
if (!obs.lastSettled) {
|
||||||
|
// Initial observation can fail or race the first action. Keep whatever
|
||||||
|
// the content script captured so the exported step is still teachable.
|
||||||
|
if ("target" in draft && "captureTarget" in draft) {
|
||||||
|
draft.target = fallbackDescriptor(draft.captureTarget);
|
||||||
|
}
|
||||||
|
rememberStepAnnotation(obs, stepId, draft);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
draft.preStateId = obs.lastSettled.stateId;
|
||||||
|
|
||||||
|
if ("target" in draft) {
|
||||||
|
draft.target =
|
||||||
|
"geometry" in draft && draft.geometry
|
||||||
|
? matchTarget({
|
||||||
|
geometry: draft.geometry,
|
||||||
|
captured: obs.lastSettled.captured,
|
||||||
|
refs: obs.lastSettled.refs,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: // No geometry to match on, but the capture still knew what the user
|
||||||
|
// touched — keep that instead of an anonymous unmatched target.
|
||||||
|
fallbackDescriptor(draft.captureTarget);
|
||||||
|
}
|
||||||
|
rememberStepAnnotation(obs, stepId, draft);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fillDetailForDraft(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): string | undefined {
|
||||||
|
if (obs.redactValues) return undefined;
|
||||||
|
if (draft.op === "fill") return JSON.stringify(draft.value);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function refLineForDraft(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): number | undefined {
|
||||||
|
if (!("target" in draft) || !obs.lastSettled) return undefined;
|
||||||
|
const targetRef = draft.target?.ref;
|
||||||
|
if (!targetRef) return undefined;
|
||||||
|
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rememberStepAnnotation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
stepId: number,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): void {
|
||||||
|
const line = refLineForDraft(obs, draft);
|
||||||
|
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
|
||||||
|
if (!draft.preStateId) return;
|
||||||
|
const ann: ObservationAnnotation = {
|
||||||
|
stepId,
|
||||||
|
op: draft.op,
|
||||||
|
line,
|
||||||
|
stateId: draft.preStateId,
|
||||||
|
detail: fillDetailForDraft(obs, draft),
|
||||||
|
};
|
||||||
|
const bucket = obs.stepAnnotations.get(line) ?? [];
|
||||||
|
bucket.push(ann);
|
||||||
|
obs.stepAnnotations.set(line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record the step under the page it was performed on, not the one it led to. */
|
||||||
|
export function rememberStepOnPage(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
stepId: number,
|
||||||
|
): void {
|
||||||
|
const stateId = draft.preStateId;
|
||||||
|
if (!stateId) return;
|
||||||
|
const entry = obs.stateRegistry.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(stepId)) {
|
||||||
|
entry.stepsHere.push(stepId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
|
||||||
|
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||||
|
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
|
||||||
|
* to the final URL. Later hops only bump `generation` / replace `url`.
|
||||||
|
*/
|
||||||
|
export function scheduleRedirectLandingFlush(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
url: string,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): void {
|
||||||
|
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
|
||||||
|
obs.pendingRedirect = { url, generation };
|
||||||
|
enqueueRedirectFlush(obs, () =>
|
||||||
|
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drain any coalesced redirect so the next action matches against the real
|
||||||
|
* landing page. Safe to call when nothing is pending.
|
||||||
|
*/
|
||||||
|
export async function flushPendingRedirectLanding(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner | undefined,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
if (obs.pendingRedirect && cdp) {
|
||||||
|
enqueueRedirectFlush(obs, () =>
|
||||||
|
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await obs.redirectFlushQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait until the redirect chain stops changing the document, then emit one
|
||||||
|
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
|
||||||
|
*/
|
||||||
|
async function coalesceRedirectLanding(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
options: { cancelled?: () => boolean },
|
||||||
|
): Promise<void> {
|
||||||
|
while (obs.pendingRedirect) {
|
||||||
|
if (options.cancelled?.()) {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const snap = obs.pendingRedirect;
|
||||||
|
const outcome = await waitForPageSettled(cdp, tabId, {
|
||||||
|
cancelled: () =>
|
||||||
|
!!options.cancelled?.() ||
|
||||||
|
obs.pendingRedirect === null ||
|
||||||
|
obs.pendingRedirect.generation !== snap.generation,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (options.cancelled?.()) {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A newer hop cancelled this wait — loop and settle against the latest URL.
|
||||||
|
if (outcome === "cancelled") continue;
|
||||||
|
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
|
||||||
|
if (!finalUrl || finalUrl === "about:blank") return;
|
||||||
|
if (obs.lastSettled?.url === finalUrl) return;
|
||||||
|
|
||||||
|
const last = steps[steps.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) {
|
||||||
|
const draftIndex = steps.length - 1;
|
||||||
|
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
|
||||||
|
await obs.settleQueue;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (outcome === "timeout") {
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] redirect landing still changing after settle budget; ` +
|
||||||
|
`recording navigate to ${finalUrl}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
page_url: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
preStateId: obs.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
steps.push(draft);
|
||||||
|
const draftIndex = steps.length - 1;
|
||||||
|
const stepId = draftIndex + 1;
|
||||||
|
rememberStepOnPage(obs, draft, stepId);
|
||||||
|
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
|
||||||
|
// Callers that flush before the next action need `lastSettled` to already
|
||||||
|
// be the landing page so target matching does not use the pre-redirect view.
|
||||||
|
await obs.settleQueue;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A capture that lands mid-navigation fails on a destroyed execution context.
|
||||||
|
* That is transient, so give it one more chance before the step has to fall
|
||||||
|
* back to a stale state.
|
||||||
|
*/
|
||||||
|
async function captureWithRetry(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
urlOverride?: string,
|
||||||
|
): Promise<LastSettledObservation> {
|
||||||
|
try {
|
||||||
|
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk record] observation failed, retrying once", err);
|
||||||
|
await sleep(CAPTURE_RETRY_DELAY_MS);
|
||||||
|
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function settleDraftObservation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
pending?: PendingSettle,
|
||||||
|
): Promise<void> {
|
||||||
|
const cancelled = () => pending?.cancelled === true;
|
||||||
|
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
if (outcome === "timeout") {
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
|
||||||
|
"budget; observing it as it stands",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const started = steps[draftIndex];
|
||||||
|
if (!started) return;
|
||||||
|
|
||||||
|
// The URL attached to the draft may only be an intermediate redirect hop.
|
||||||
|
// Once the page is settled, register the capture against live tab metadata.
|
||||||
|
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||||
|
// The observation still counts as the latest view of the page even when this
|
||||||
|
// step no longer wants it — the next action will start from it.
|
||||||
|
if (cancelled()) return;
|
||||||
|
// Re-read the slot: a navigation observed while the capture ran may have
|
||||||
|
// rewritten this draft, and the observation belongs to whatever occupies the
|
||||||
|
// slot now — writing to the object we started with would strand it.
|
||||||
|
const draft = steps[draftIndex] ?? started;
|
||||||
|
draft.postStateId = settled.stateId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recover landing states from the shape of the recording itself: wherever the
|
||||||
|
* next action was performed is, by definition, where the previous one landed.
|
||||||
|
* Only the immediately following draft counts — a later one would claim a page
|
||||||
|
* that several unobserved actions away, which is worse than admitting we saw
|
||||||
|
* no change.
|
||||||
|
*/
|
||||||
|
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
|
||||||
|
for (let i = 0; i < steps.length - 1; i += 1) {
|
||||||
|
const draft = steps[i];
|
||||||
|
const next = steps[i + 1];
|
||||||
|
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
|
||||||
|
`using where step ${i + 2} started (${next.preStateId})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Last chance for the closing actions of a recording. Stopping right after the
|
||||||
|
* final action is normal, and so is that action navigating away, so take one
|
||||||
|
* final look at the page. Only trailing drafts may claim it: an earlier step
|
||||||
|
* did not land on whatever the user happens to be looking at when they stop.
|
||||||
|
*/
|
||||||
|
export async function settleUnsettledDrafts(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): Promise<void> {
|
||||||
|
const trailing: DraftTraceStep[] = [];
|
||||||
|
for (let i = steps.length - 1; i >= 0; i -= 1) {
|
||||||
|
const draft = steps[i];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
|
||||||
|
let settled: LastSettledObservation;
|
||||||
|
try {
|
||||||
|
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const draft of trailing) draft.postStateId = settled.stateId;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
|
||||||
|
`(${settled.stateId} ${settled.url})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Performing a new action ends the previous one: the page the user reached for
|
||||||
|
* is, by definition, where the earlier step left them. Taking the landing from
|
||||||
|
* the newer step also keeps the trace monotonic — a capture that finished late
|
||||||
|
* would otherwise credit an earlier step with a page that only exists because
|
||||||
|
* of the newer action.
|
||||||
|
*/
|
||||||
|
function supersedeEarlierSettles(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draftIndex: number,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): void {
|
||||||
|
const landing = steps[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of obs.settles) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.cancelled = true;
|
||||||
|
obs.settles.delete(index);
|
||||||
|
|
||||||
|
const draft = steps[index];
|
||||||
|
if (!draft || draft.postStateId) continue;
|
||||||
|
if (!landing) continue;
|
||||||
|
draft.postStateId = landing;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
|
||||||
|
`${draftIndex + 1} started; landing it on ${landing}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function scheduleDraftSettle(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draftIndex: number,
|
||||||
|
stepId: number,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): void {
|
||||||
|
supersedeEarlierSettles(obs, draftIndex, steps);
|
||||||
|
|
||||||
|
// Rescheduling the same step (a navigation showed up after the action)
|
||||||
|
// replaces the pending observation rather than racing it.
|
||||||
|
const superseded = obs.settles.get(draftIndex);
|
||||||
|
if (superseded) superseded.cancelled = true;
|
||||||
|
|
||||||
|
const pending: PendingSettle = { draftIndex, cancelled: false };
|
||||||
|
obs.settles.set(draftIndex, pending);
|
||||||
|
|
||||||
|
enqueueSettle(obs, async () => {
|
||||||
|
if (pending.cancelled) return;
|
||||||
|
try {
|
||||||
|
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
|
||||||
|
} catch (err) {
|
||||||
|
// Recoverable: stop-time repair still gives the step a landing page.
|
||||||
|
const op = steps[draftIndex]?.op ?? "?";
|
||||||
|
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
|
||||||
|
} finally {
|
||||||
|
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||||
|
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cancelPendingSettles(obs: RecordingObservationState): void {
|
||||||
|
for (const pending of obs.settles.values()) pending.cancelled = true;
|
||||||
|
obs.settles.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
|
||||||
|
const MAX_SETTLE_FLUSH_ROUNDS = 10;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drain settle work before exporting, including work queued while draining —
|
||||||
|
* a navigation observed during the last capture schedules another one, and
|
||||||
|
* the trace would drop it if stop did not wait.
|
||||||
|
*/
|
||||||
|
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
|
||||||
|
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
|
||||||
|
const drained = obs.settleQueue;
|
||||||
|
await drained;
|
||||||
|
if (obs.settleQueue === drained) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizeStateBodies(
|
||||||
|
entries: StateRegistryEntry[],
|
||||||
|
annotationsByState: Map<string, ObservationAnnotation[]>,
|
||||||
|
stepIdByDraftId: Map<number, number>,
|
||||||
|
idByOldId: Map<string, string>,
|
||||||
|
): TraceState[] {
|
||||||
|
return entries.map((entry) => {
|
||||||
|
const id = idByOldId.get(entry.id) ?? entry.id;
|
||||||
|
const body = formatObservationFile({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
|
||||||
|
body: entry.rawVomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body,
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Draft ids only become step ids after collapsing and filtering. */
|
||||||
|
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
const mapped = new Set<number>();
|
||||||
|
for (const draftId of draftIds) {
|
||||||
|
const stepId = stepIdByDraftId.get(draftId);
|
||||||
|
if (stepId !== undefined) mapped.add(stepId);
|
||||||
|
}
|
||||||
|
return [...mapped].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectAnnotationsByState(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
stepIdByDraftId: Map<number, number>,
|
||||||
|
): Map<string, ObservationAnnotation[]> {
|
||||||
|
const byState = new Map<string, ObservationAnnotation[]>();
|
||||||
|
for (const anns of obs.stepAnnotations.values()) {
|
||||||
|
for (const ann of anns) {
|
||||||
|
const stepId = stepIdByDraftId.get(ann.stepId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byState.get(ann.stateId) ?? [];
|
||||||
|
bucket.push({ ...ann, stepId });
|
||||||
|
byState.set(ann.stateId, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return byState;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keep only the pages the published steps point at. Redirect hops and
|
||||||
|
* mid-load captures land in the registry too, and shipping them would invite
|
||||||
|
* a reader to treat a page the flow merely passed through as a real stop.
|
||||||
|
* With no steps at all the first observation is the whole artifact, so it
|
||||||
|
* stays.
|
||||||
|
*/
|
||||||
|
function selectPublishedStates(
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
steps: Step[],
|
||||||
|
): StateRegistryEntry[] {
|
||||||
|
const entries = [...registry.values()];
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set<string>();
|
||||||
|
for (const step of steps) {
|
||||||
|
referenced.add(step.state);
|
||||||
|
referenced.add(step.result.state);
|
||||||
|
}
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
obs: RecordingObservationState;
|
||||||
|
steps: DraftTraceStep[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
}): Trace {
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||||
|
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||||
|
// Renumber so the shipped dictionary reads s1..sN without holes where the
|
||||||
|
// dropped captures used to be.
|
||||||
|
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
for (const step of steps) {
|
||||||
|
step.state = idByOldId.get(step.state) ?? step.state;
|
||||||
|
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
|
||||||
|
}
|
||||||
|
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
|
||||||
|
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
|
||||||
|
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
|
||||||
|
return {
|
||||||
|
version: 3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: startUrl },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,23 +1,29 @@
|
|||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
|
||||||
import type { RecordStepPayload } from "./record-bridge";
|
import type { RecordStepPayload } from "./record-bridge";
|
||||||
|
import { mapNavigationCause } from "./record-constants";
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
export interface RecordingStepBuffer {
|
||||||
steps: DraftTraceStep[];
|
steps: DraftTraceStep[];
|
||||||
currentUrl?: string;
|
currentUrl?: string;
|
||||||
pendingNavigation: boolean;
|
pendingNavigation: boolean;
|
||||||
pendingNavigationDeadline?: number;
|
pendingNavigationDeadline?: number;
|
||||||
|
lastTransitionType?: string;
|
||||||
|
lastTransitionQualifiers?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||||
const pageUrl = payload.page_url;
|
const pageUrl = payload.page_url;
|
||||||
|
const geometry = payload.geometry;
|
||||||
switch (payload.op) {
|
switch (payload.op) {
|
||||||
case "click":
|
case "click":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "click",
|
op: "click",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
@@ -25,7 +31,9 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
@@ -33,8 +41,11 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
value: payload.value ?? "",
|
value: payload.value ?? "",
|
||||||
|
...(payload.commit ? { commit: payload.commit } : {}),
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
@@ -44,7 +55,10 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
? {
|
? {
|
||||||
op: "press",
|
op: "press",
|
||||||
key: payload.key,
|
key: payload.key,
|
||||||
...(payload.target ? { target: payload.target } : {}),
|
...(payload.target
|
||||||
|
? { target: { unmatched: true }, captureTarget: payload.target }
|
||||||
|
: {}),
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
@@ -53,37 +67,69 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
return payload.target && payload.values
|
return payload.target && payload.values
|
||||||
? {
|
? {
|
||||||
op: "select",
|
op: "select",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
values: payload.values,
|
values: payload.values,
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
|
||||||
case "navigate":
|
case "navigate":
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
|
||||||
|
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||||
const step = buffer.steps[i];
|
const step = buffer.steps[i];
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
buffer.steps[i] = { ...step, navigated_to: url };
|
// Mutate in place: an in-flight settle holds this object, and replacing
|
||||||
return true;
|
// it would strand that settle's observation on the discarded copy.
|
||||||
|
step.navigated_to = url;
|
||||||
|
return i;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
return false;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
|
||||||
|
return mapNavigationCause({
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Outcome of folding a main-frame URL change into the draft buffer. */
|
||||||
|
export type NavigationObserveResult =
|
||||||
|
| { kind: "noop" }
|
||||||
|
| { kind: "annotated"; index: number }
|
||||||
|
| { kind: "appended"; index: number }
|
||||||
|
/**
|
||||||
|
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
|
||||||
|
* emits one `navigate` after `waitForPageSettled` on the final URL.
|
||||||
|
*/
|
||||||
|
| { kind: "coalesce_redirect"; url: string };
|
||||||
|
|
||||||
export function observeRecordedNavigation(
|
export function observeRecordedNavigation(
|
||||||
buffer: RecordingStepBuffer,
|
buffer: RecordingStepBuffer,
|
||||||
url: string,
|
url: string,
|
||||||
causedByAction?: boolean,
|
causedByAction?: boolean,
|
||||||
): void {
|
transitionType?: string,
|
||||||
if (!url || url === buffer.currentUrl) return;
|
transitionQualifiers?: string[],
|
||||||
|
): NavigationObserveResult {
|
||||||
|
if (!url || url === buffer.currentUrl) return { kind: "noop" };
|
||||||
buffer.currentUrl = url;
|
buffer.currentUrl = url;
|
||||||
|
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
|
||||||
|
if (transitionQualifiers !== undefined) {
|
||||||
|
buffer.lastTransitionQualifiers = transitionQualifiers;
|
||||||
|
}
|
||||||
|
|
||||||
const pendingIsCurrent =
|
const pendingIsCurrent =
|
||||||
buffer.pendingNavigation &&
|
buffer.pendingNavigation &&
|
||||||
(buffer.pendingNavigationDeadline === undefined ||
|
(buffer.pendingNavigationDeadline === undefined ||
|
||||||
@@ -92,40 +138,69 @@ export function observeRecordedNavigation(
|
|||||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||||
buffer.pendingNavigation = false;
|
buffer.pendingNavigation = false;
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
buffer.pendingNavigationDeadline = undefined;
|
||||||
if (!annotateLastStepNavigation(buffer, url)) {
|
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||||
buffer.steps.push({
|
if (annotatedIndex >= 0) {
|
||||||
op: "navigate",
|
return { kind: "annotated", index: annotatedIndex };
|
||||||
url,
|
|
||||||
page_url: url,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
return;
|
const cause = causeFromBuffer(buffer);
|
||||||
|
if (cause === null) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
|
buffer.steps.push({
|
||||||
|
op: "navigate",
|
||||||
|
url,
|
||||||
|
page_url: url,
|
||||||
|
cause,
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
}
|
}
|
||||||
|
|
||||||
buffer.pendingNavigation = false;
|
buffer.pendingNavigation = false;
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
buffer.pendingNavigationDeadline = undefined;
|
||||||
|
const cause = causeFromBuffer(buffer);
|
||||||
|
if (cause === null) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
buffer.steps.push({
|
buffer.steps.push({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url,
|
url,
|
||||||
page_url: url,
|
page_url: url,
|
||||||
|
cause,
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
});
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Buffer one captured payload. Returns the index of the draft it appended, or
|
||||||
|
* `null` when the payload produced none (an unnamed click target, a navigation
|
||||||
|
* that only annotated the previous draft, …). Callers must not fall back to
|
||||||
|
* "the last draft" on `null`: that would re-match an already recorded step
|
||||||
|
* against a newer observation and overwrite its origin state.
|
||||||
|
*/
|
||||||
export function appendRecordedPayload(
|
export function appendRecordedPayload(
|
||||||
buffer: RecordingStepBuffer,
|
buffer: RecordingStepBuffer,
|
||||||
payload: RecordStepPayload,
|
payload: RecordStepPayload,
|
||||||
): void {
|
): number | null {
|
||||||
if (payload.op === "navigate") {
|
if (payload.op === "navigate") {
|
||||||
if (payload.url) {
|
if (!payload.url) return null;
|
||||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
const result = observeRecordedNavigation(
|
||||||
}
|
buffer,
|
||||||
return;
|
payload.url,
|
||||||
|
payload.navigation_caused_by_action,
|
||||||
|
payload.transitionType,
|
||||||
|
payload.transitionQualifiers,
|
||||||
|
);
|
||||||
|
return result.kind === "appended" ? result.index : null;
|
||||||
}
|
}
|
||||||
const step = toDraftStep({
|
const step = toDraftStep({
|
||||||
...payload,
|
...payload,
|
||||||
page_url: payload.page_url ?? buffer.currentUrl,
|
page_url: payload.page_url ?? buffer.currentUrl,
|
||||||
});
|
});
|
||||||
if (!step) return;
|
if (!step) return null;
|
||||||
buffer.steps.push(step);
|
buffer.steps.push(step);
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||||
@@ -133,4 +208,5 @@ export function appendRecordedPayload(
|
|||||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||||
: undefined;
|
: undefined;
|
||||||
}
|
}
|
||||||
|
return buffer.steps.length - 1;
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,17 @@
|
|||||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
import type {
|
||||||
|
DraftTraceStep,
|
||||||
|
FillCommit,
|
||||||
|
NavigationCause,
|
||||||
|
Step,
|
||||||
|
TraceState,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
DEFAULT_FILL_COMMIT,
|
||||||
|
fnv1a64,
|
||||||
|
mapNavigationCause,
|
||||||
|
nextStateId,
|
||||||
|
resetStateIdCounterForTests,
|
||||||
|
} from "./record-constants";
|
||||||
|
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
@@ -12,7 +25,6 @@ export function shouldRecordPress(
|
|||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
// Drop bare character typing — FillSession already records the value.
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -23,195 +35,191 @@ function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
/** A reduced draft plus the capture-order ids of every draft folded into it. */
|
||||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
interface CollapsedDraft {
|
||||||
const out: DraftTraceStep[] = [];
|
draft: DraftTraceStep;
|
||||||
for (const step of steps) {
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collapse a redirect chain into a single hop.
|
||||||
|
*
|
||||||
|
* The surviving step keeps the *first* hop's origin state and cause — that is
|
||||||
|
* where the user was and why they left — and the *last* hop's destination, so
|
||||||
|
* the intermediate URL the browser only passed through never becomes the
|
||||||
|
* step's `state` or `to`.
|
||||||
|
*/
|
||||||
|
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
|
||||||
|
const out: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const draftId = index + 1;
|
||||||
const prev = out[out.length - 1];
|
const prev = out[out.length - 1];
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
if (step.op === "navigate" && prev?.draft.op === "navigate") {
|
||||||
out[out.length - 1] = step;
|
prev.draft = {
|
||||||
continue;
|
...prev.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? prev.draft.postStateId,
|
||||||
|
};
|
||||||
|
prev.draftIds.push(draftId);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
out.push(step);
|
out.push({ draft: step, draftIds: [draftId] });
|
||||||
}
|
});
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
export interface StateRegistryEntry extends TraceState {
|
||||||
const urls: string[] = [];
|
contentHash: string;
|
||||||
if (startUrl) urls.push(startUrl);
|
rawVomText: string;
|
||||||
for (const step of steps) {
|
stepsHere: number[];
|
||||||
if (step.op === "navigate") {
|
|
||||||
urls.push(step.url);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
|
||||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
|
||||||
}
|
|
||||||
const seen = new Set<string>();
|
|
||||||
const unique: string[] = [];
|
|
||||||
for (const url of urls) {
|
|
||||||
if (!url || seen.has(url)) continue;
|
|
||||||
seen.add(url);
|
|
||||||
unique.push(url);
|
|
||||||
}
|
|
||||||
return unique;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildPageRegistry(
|
export function stateDedupKey(body: string, url: string): string {
|
||||||
steps: DraftTraceStep[],
|
return `${fnv1a64(body)}:${url}`;
|
||||||
startUrl?: string,
|
|
||||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
|
||||||
const urls = collectUrls(steps, startUrl);
|
|
||||||
const urlToId = new Map<string, string>();
|
|
||||||
const pages = urls.map((url, index) => {
|
|
||||||
const id = `p${index + 1}`;
|
|
||||||
urlToId.set(url, id);
|
|
||||||
return { id, url };
|
|
||||||
});
|
|
||||||
return { pages, urlToId };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageIdFor(
|
/** Register or reuse a page observation in the states dictionary. */
|
||||||
url: string | undefined,
|
export function registerObservation(
|
||||||
urlToId: Map<string, string>,
|
registry: Map<string, StateRegistryEntry>,
|
||||||
fallbackUrl?: string,
|
input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
},
|
||||||
): string {
|
): string {
|
||||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
const key = stateDedupKey(input.rawVomText, input.url);
|
||||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
for (const entry of registry.values()) {
|
||||||
return urlToId.values().next().value ?? "p1";
|
if (entry.contentHash === key) return entry.id;
|
||||||
|
}
|
||||||
|
const id = nextStateId();
|
||||||
|
registry.set(id, {
|
||||||
|
id,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
contentHash: key,
|
||||||
|
rawVomText: input.rawVomText,
|
||||||
|
stepsHere: [],
|
||||||
|
});
|
||||||
|
return id;
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
function toSelection(
|
||||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
values: string[],
|
||||||
if ("page_url" in step && step.page_url) return step.page_url;
|
labels?: string[],
|
||||||
return fallbackUrl;
|
): Array<{ value: string; label?: string }> {
|
||||||
}
|
|
||||||
|
|
||||||
function effectForNavigation(
|
|
||||||
navigatedTo: string | undefined,
|
|
||||||
urlToId: Map<string, string>,
|
|
||||||
): Step["effect"] {
|
|
||||||
if (!navigatedTo) return undefined;
|
|
||||||
const pageId = urlToId.get(navigatedTo);
|
|
||||||
if (!pageId) return undefined;
|
|
||||||
return { navigated_to: pageId };
|
|
||||||
}
|
|
||||||
|
|
||||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
|
||||||
if (!effect) return step;
|
|
||||||
return { ...step, effect };
|
|
||||||
}
|
|
||||||
|
|
||||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
|
||||||
return values.map((value, index) => ({
|
return values.map((value, index) => ({
|
||||||
value,
|
value,
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function toV2Step(
|
function navigateCauseForDraft(
|
||||||
step: DraftTraceStep,
|
draft: Extract<DraftTraceStep, { op: "navigate" }>,
|
||||||
id: number,
|
): NavigationCause {
|
||||||
urlToId: Map<string, string>,
|
if (draft.cause) return draft.cause;
|
||||||
fallbackUrl?: string,
|
const mapped = mapNavigationCause({
|
||||||
): Step | null {
|
transitionType: draft.transitionType,
|
||||||
if (!shouldIncludeDraft(step)) return null;
|
transitionQualifiers: draft.transitionQualifiers,
|
||||||
|
});
|
||||||
|
return mapped ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
|
||||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
// An action the user really performed must not disappear because one of its
|
||||||
|
// observations is missing — a capture racing a document swap is routine, and
|
||||||
|
// the last step of a recording is the most exposed to it. Fall back to the
|
||||||
|
// endpoint we do know; only a step with no observation at all has nothing to
|
||||||
|
// point at.
|
||||||
|
const preState = draft.preStateId ?? draft.postStateId;
|
||||||
|
const postState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!preState || !postState) return null;
|
||||||
|
|
||||||
switch (step.op) {
|
const common = { id, state: preState, result: { state: postState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
case "navigate":
|
case "navigate":
|
||||||
return {
|
return {
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
id,
|
...common,
|
||||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
to: draft.url,
|
||||||
to: step.url,
|
cause: navigateCauseForDraft(draft),
|
||||||
};
|
};
|
||||||
case "click":
|
case "click":
|
||||||
return withEffect(
|
return { op: "click", ...common, target: draft.target };
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.target,
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
|
||||||
);
|
|
||||||
case "hover":
|
case "hover":
|
||||||
return {
|
return { op: "hover", ...common, target: draft.target };
|
||||||
op: "hover",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.target,
|
|
||||||
};
|
|
||||||
case "fill":
|
case "fill":
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
id,
|
...common,
|
||||||
page,
|
target: draft.target,
|
||||||
target: step.target,
|
value: draft.value,
|
||||||
value: step.value,
|
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
...(draft.redacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
case "press":
|
case "press":
|
||||||
return withEffect(
|
return {
|
||||||
{
|
op: "press",
|
||||||
op: "press",
|
...common,
|
||||||
id,
|
key: draft.key,
|
||||||
page,
|
...(draft.target ? { target: draft.target } : {}),
|
||||||
key: step.key,
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
...(step.target ? { target: step.target } : {}),
|
};
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
|
||||||
);
|
|
||||||
case "select":
|
case "select":
|
||||||
return withEffect(
|
return {
|
||||||
{
|
op: "select",
|
||||||
op: "select",
|
...common,
|
||||||
id,
|
target: draft.target,
|
||||||
page,
|
selection: toSelection(draft.values, draft.labels),
|
||||||
target: step.target,
|
};
|
||||||
selection: toSelection(step.values, step.labels),
|
case "scroll":
|
||||||
},
|
return { op: "scroll", ...common };
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReducedTrace {
|
export interface ReducedTrace {
|
||||||
pages: PageRef[];
|
states: TraceState[];
|
||||||
steps: Step[];
|
steps: Step[];
|
||||||
|
/**
|
||||||
|
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
|
||||||
|
* and dropped drafts make the two diverge, so anything the recorder tagged
|
||||||
|
* with a draft id (`steps_here`, inline annotations) must be remapped.
|
||||||
|
*/
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compile capture drafts into record-only trace v2 steps.
|
* Compile capture drafts into record-only trace v3 steps.
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
|
||||||
*/
|
*/
|
||||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
export function reduceTraceSteps(
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
): ReducedTrace {
|
||||||
const collapsed = collapseNavigations(steps);
|
const collapsed = collapseNavigations(steps);
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
|
||||||
const out: Step[] = [];
|
const out: Step[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
let id = 1;
|
let id = 1;
|
||||||
let lastUrl = startUrl;
|
for (const { draft, draftIds } of collapsed) {
|
||||||
for (const draft of collapsed) {
|
const step = toV3Step(draft, id);
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
|
||||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
|
||||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
out.push(step);
|
out.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
|
||||||
id += 1;
|
id += 1;
|
||||||
}
|
}
|
||||||
return { pages, steps: out };
|
const states = [...registry.values()].map(
|
||||||
|
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
|
||||||
|
);
|
||||||
|
return { states, steps: out, stepIdByDraftId };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTraceStartUrl(
|
export function resolveTraceStartUrl(
|
||||||
drafts: DraftTraceStep[],
|
drafts: DraftTraceStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
pages?: PageRef[],
|
states?: TraceState[],
|
||||||
): string {
|
): string {
|
||||||
if (startUrl) return startUrl;
|
if (startUrl) return startUrl;
|
||||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||||
@@ -221,5 +229,9 @@ export function resolveTraceStartUrl(
|
|||||||
for (const draft of drafts) {
|
for (const draft of drafts) {
|
||||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||||
}
|
}
|
||||||
return pages?.[0]?.url ?? "about:blank";
|
return states?.[0]?.url ?? "about:blank";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type { FillCommit, NavigationCause };
|
||||||
|
/** Test seam */
|
||||||
|
export { resetStateIdCounterForTests };
|
||||||
@@ -1034,7 +1034,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||||
const axNodes: CdpAxNode[] = [
|
const axNodes: CdpAxNode[] = [
|
||||||
{
|
{
|
||||||
nodeId: "1",
|
nodeId: "1",
|
||||||
@@ -1069,7 +1069,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
id: 10,
|
id: 10,
|
||||||
sensitive: false,
|
sensitive: true,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -1364,7 +1364,9 @@ describe("buildVomScene", () => {
|
|||||||
);
|
);
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "close"');
|
expect(rendered.text).toContain('@e1 button "close"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||||
@@ -1508,7 +1510,9 @@ describe("buildVomScene", () => {
|
|||||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,101 @@
|
|||||||
|
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||||
|
import type { CdpAxNode } from "./observation";
|
||||||
|
import { buildVomScene } from "./observation";
|
||||||
|
import type { CdpRunner } from "./shared";
|
||||||
|
import {
|
||||||
|
type CapturedNode,
|
||||||
|
type CapturedSurfaceProbe,
|
||||||
|
type CapturedViewModel,
|
||||||
|
captureViewModel,
|
||||||
|
collectOverlayExcludedBackendIds,
|
||||||
|
} from "./vom/capture";
|
||||||
|
|
||||||
|
export interface CaptureVomObservationOptions extends VomOptions {
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureVomObservationResult {
|
||||||
|
text: string;
|
||||||
|
refs: RenderedRef[];
|
||||||
|
truncated: boolean;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||||
|
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LayoutMetricsViewportReply {
|
||||||
|
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fallbackCapturedViewModel(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<CapturedViewModel> {
|
||||||
|
let viewport = { width: 0, height: 0 };
|
||||||
|
try {
|
||||||
|
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
viewport = {
|
||||||
|
width: vpSrc.clientWidth ?? 0,
|
||||||
|
height: vpSrc.clientHeight ?? 0,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
// viewport stays zero-sized
|
||||||
|
}
|
||||||
|
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||||
|
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureForVom(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
conditionalSurfaceProbe: boolean,
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||||
|
): Promise<CapturedViewModel> {
|
||||||
|
try {
|
||||||
|
return await captureViewModel(cdp, tabId, {
|
||||||
|
conditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return fallbackCapturedViewModel(cdp, tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
|
||||||
|
export async function captureVomObservation(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
url: string | undefined,
|
||||||
|
opts: CaptureVomObservationOptions = {},
|
||||||
|
): Promise<CaptureVomObservationResult> {
|
||||||
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
|
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
|
||||||
|
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
|
||||||
|
const axNodes = result.nodes ?? [];
|
||||||
|
const captured = await captureForVom(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
opts.conditionalSurfaceProbe ?? false,
|
||||||
|
opts.hoverProbeBypassOverlay,
|
||||||
|
);
|
||||||
|
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
|
||||||
|
const rendered = renderVom(scene, {
|
||||||
|
maxDepth: opts.maxDepth,
|
||||||
|
maxTokens: opts.maxTokens,
|
||||||
|
redactValues: opts.redactValues,
|
||||||
|
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
text: rendered.text,
|
||||||
|
refs: rendered.refs,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
captured: captured.nodes,
|
||||||
|
surfaceProbes: captured.surfaceProbes,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -8,7 +8,6 @@ import {
|
|||||||
applyVomInteractionRecovery,
|
applyVomInteractionRecovery,
|
||||||
type CondSurface,
|
type CondSurface,
|
||||||
isVomReferenceNode,
|
isVomReferenceNode,
|
||||||
renderVom,
|
|
||||||
type VomNode,
|
type VomNode,
|
||||||
type VomScene,
|
type VomScene,
|
||||||
} from "@browser-skill/vom";
|
} from "@browser-skill/vom";
|
||||||
@@ -29,6 +28,7 @@ import type {
|
|||||||
SnapshotParams,
|
SnapshotParams,
|
||||||
SnapshotResult,
|
SnapshotResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import { captureVomObservation } from "./capture-vom-observation";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
@@ -44,12 +44,7 @@ import {
|
|||||||
type ToolEffect,
|
type ToolEffect,
|
||||||
} from "./shared";
|
} from "./shared";
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import {
|
import { type CapturedNode, type CapturedViewModel } from "./vom/capture";
|
||||||
type CapturedNode,
|
|
||||||
type CapturedViewModel,
|
|
||||||
captureViewModel,
|
|
||||||
collectOverlayExcludedBackendIds,
|
|
||||||
} from "./vom/capture";
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||||
@@ -350,7 +345,18 @@ function isModalSignal(
|
|||||||
|
|
||||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||||
const attrs = capturedNode?.attrs ?? {};
|
const attrs = capturedNode?.attrs ?? {};
|
||||||
return (attrs.type ?? "").toLowerCase() === "password";
|
const type = (attrs.type ?? "").toLowerCase();
|
||||||
|
if (type === "password") return true;
|
||||||
|
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||||
|
if (autocomplete.startsWith("cc-")) return true;
|
||||||
|
if (
|
||||||
|
autocomplete === "one-time-code" ||
|
||||||
|
autocomplete === "current-password" ||
|
||||||
|
autocomplete === "new-password"
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AxNodeSignals {
|
interface AxNodeSignals {
|
||||||
@@ -400,7 +406,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
|||||||
"columnheader",
|
"columnheader",
|
||||||
"rowheader",
|
"rowheader",
|
||||||
]);
|
]);
|
||||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||||
|
"password",
|
||||||
|
"credit-card",
|
||||||
|
"one-time-code",
|
||||||
|
"current-password",
|
||||||
|
"new-password",
|
||||||
|
]);
|
||||||
|
|
||||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||||
const prop = axNode.properties?.find((item) => item.name === name);
|
const prop = axNode.properties?.find((item) => item.name === name);
|
||||||
@@ -1314,50 +1326,6 @@ export async function handleGetHtml(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
|
||||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
|
||||||
}
|
|
||||||
|
|
||||||
interface LayoutMetricsViewportReply {
|
|
||||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fallbackCapturedViewModel(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
): Promise<CapturedViewModel> {
|
|
||||||
let viewport = { width: 0, height: 0 };
|
|
||||||
try {
|
|
||||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
|
||||||
viewport = {
|
|
||||||
width: vpSrc.clientWidth ?? 0,
|
|
||||||
height: vpSrc.clientHeight ?? 0,
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
// viewport stays zero-sized
|
|
||||||
}
|
|
||||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
|
||||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function captureForVom(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
conditionalSurfaceProbe: boolean,
|
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
|
||||||
): Promise<CapturedViewModel> {
|
|
||||||
try {
|
|
||||||
return await captureViewModel(cdp, tabId, {
|
|
||||||
conditionalSurfaceProbe,
|
|
||||||
hoverProbeBypassOverlay,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
return fallbackCapturedViewModel(cdp, tabId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleVomObservation(
|
async function handleVomObservation(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SnapshotParams | ObserveParams,
|
params: SnapshotParams | ObserveParams,
|
||||||
@@ -1383,27 +1351,14 @@ async function handleVomObservation(
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
|
||||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
|
||||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
|
||||||
target.tabId,
|
|
||||||
"Accessibility.getFullAXTree",
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
const axNodes = result.nodes ?? [];
|
|
||||||
const effectiveConditionalSurfaceProbe =
|
const effectiveConditionalSurfaceProbe =
|
||||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||||
const captured = await captureForVom(
|
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
effectiveConditionalSurfaceProbe,
|
|
||||||
deps.hoverProbeBypassOverlay,
|
|
||||||
);
|
|
||||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
|
||||||
const rendered = renderVom(scene, {
|
|
||||||
maxDepth: params.max_depth,
|
maxDepth: params.max_depth,
|
||||||
maxTokens: params.max_tokens,
|
maxTokens: params.max_tokens,
|
||||||
activeRegionPolicy: true,
|
activeRegionPolicy: true,
|
||||||
|
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||||
});
|
});
|
||||||
ctx.refStore.replace(
|
ctx.refStore.replace(
|
||||||
rendered.refs.map(
|
rendered.refs.map(
|
||||||
@@ -1418,7 +1373,7 @@ async function handleVomObservation(
|
|||||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||||
? {
|
? {
|
||||||
debug: {
|
debug: {
|
||||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
|
||||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||||
trigger_action: probe.triggerAction,
|
trigger_action: probe.triggerAction,
|
||||||
|
|||||||
@@ -17,9 +17,26 @@ import {
|
|||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||||
|
import {
|
||||||
|
applyTargetMatching,
|
||||||
|
buildTraceV3,
|
||||||
|
cancelPendingSettles,
|
||||||
|
captureAndRegisterObservation,
|
||||||
|
clearPendingRedirectLanding,
|
||||||
|
createObservationState,
|
||||||
|
flushPendingRedirectLanding,
|
||||||
|
flushPendingSettles,
|
||||||
|
inferMissingPostStates,
|
||||||
|
type RecordingObservationState,
|
||||||
|
rememberStepOnPage,
|
||||||
|
scheduleDraftSettle,
|
||||||
|
scheduleRedirectLandingFlush,
|
||||||
|
settleUnsettledDrafts,
|
||||||
|
} from "@/lib/record-observation";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import type {
|
import type {
|
||||||
DraftTraceStep,
|
DraftTraceStep,
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
@@ -29,6 +46,7 @@ import type {
|
|||||||
RecordStopParams,
|
RecordStopParams,
|
||||||
RecordStopResult,
|
RecordStopResult,
|
||||||
RpcError,
|
RpcError,
|
||||||
|
StopReason,
|
||||||
Trace,
|
Trace,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { handleNavigate } from "./navigation";
|
import { handleNavigate } from "./navigation";
|
||||||
@@ -58,6 +76,33 @@ interface ActiveRecording {
|
|||||||
currentUrl?: string;
|
currentUrl?: string;
|
||||||
pendingNavigation: boolean;
|
pendingNavigation: boolean;
|
||||||
pendingNavigationDeadline?: number;
|
pendingNavigationDeadline?: number;
|
||||||
|
observation: RecordingObservationState;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
maxPageTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
/** Tabs whose content capture acknowledged START and still need STOP. */
|
||||||
|
armedTabIds: Set<number>;
|
||||||
|
/** Rearms already sending START when finish begins. */
|
||||||
|
rearmCallbacks: Set<Promise<void>>;
|
||||||
|
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||||
|
navigationCallbacks: Set<Promise<void>>;
|
||||||
|
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||||
|
acceptingNavigation: boolean;
|
||||||
|
/**
|
||||||
|
* Serializes step appends with navigation observation so a click is always
|
||||||
|
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||||
|
*/
|
||||||
|
actionQueue: Promise<void>;
|
||||||
|
/**
|
||||||
|
* CDP runner captured at `record_start`. The message listeners are attached
|
||||||
|
* once at service-worker startup, where no runner exists yet, so observation
|
||||||
|
* capture must go through the recording instead of the listener deps.
|
||||||
|
*/
|
||||||
|
cdp?: CdpRunner;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
|
||||||
|
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
const recordings = new Map<string, ActiveRecording>();
|
const recordings = new Map<string, ActiveRecording>();
|
||||||
@@ -78,6 +123,9 @@ function sleep(ms: number): Promise<void> {
|
|||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||||
|
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||||
|
|
||||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||||
|
|
||||||
@@ -155,16 +203,44 @@ async function sendRecordStartWithAck(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): Trace {
|
function buildTrace(recording: ActiveRecording): Trace {
|
||||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
return buildTraceV3({
|
||||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
obs: recording.observation,
|
||||||
return {
|
steps: recording.steps,
|
||||||
|
startedAt: recording.startedAt,
|
||||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||||
recorded_at: new Date().toISOString(),
|
startUrl: recording.startUrl,
|
||||||
started_at: recording.startedAt,
|
stoppedBy: recording.stoppedBy,
|
||||||
entry: { start_url: startUrl },
|
bskVersion: BSK_TRACE_VERSION,
|
||||||
pages,
|
});
|
||||||
steps,
|
}
|
||||||
};
|
|
||||||
|
async function processRecordedStep(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
draftIndex: number,
|
||||||
|
): Promise<void> {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (!cdp) return;
|
||||||
|
const draft = recording.steps[draftIndex];
|
||||||
|
if (!draft) return;
|
||||||
|
const stepId = draftIndex + 1;
|
||||||
|
|
||||||
|
if (draft.op === "navigate") {
|
||||||
|
draft.preStateId = recording.observation.lastSettled?.stateId;
|
||||||
|
} else {
|
||||||
|
applyTargetMatching(recording.observation, draft, stepId);
|
||||||
|
}
|
||||||
|
rememberStepOnPage(recording.observation, draft, stepId);
|
||||||
|
|
||||||
|
scheduleDraftSettle(
|
||||||
|
recording.observation,
|
||||||
|
draftIndex,
|
||||||
|
stepId,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordDeps {
|
export interface RecordDeps {
|
||||||
@@ -241,7 +317,7 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
|||||||
detachBrowserObservation = null;
|
detachBrowserObservation = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function attachRecordStepListener(): () => void {
|
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
@@ -250,7 +326,22 @@ export function attachRecordStepListener(): () => void {
|
|||||||
if (!isRecordStepMessage(message)) return false;
|
if (!isRecordStepMessage(message)) return false;
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.requestId !== message.requestId) continue;
|
if (recording.requestId !== message.requestId) continue;
|
||||||
appendRecordedPayload(recording, message.step);
|
enqueueRecordingAction(recording, async () => {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
// Flush before appending so a coalesced OAuth navigate is ordered
|
||||||
|
// ahead of the click that landed on the destination page. The flush
|
||||||
|
// itself waits for that navigate's observation before returning.
|
||||||
|
await flushPendingRedirectLanding(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
const draftIndex = appendRecordedPayload(recording, message.step);
|
||||||
|
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||||
|
});
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@@ -277,7 +368,9 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
|
|
||||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) {
|
||||||
|
return recording;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -330,29 +423,24 @@ async function stopRecordingOnAllAgentTabs(
|
|||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||||
let tabIds = [recording.tabId];
|
let failed = false;
|
||||||
try {
|
for (const tabId of [...recording.armedTabIds]) {
|
||||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
try {
|
||||||
tabIds = [
|
await deps.tabsApi.get(tabId);
|
||||||
...new Set([
|
} catch {
|
||||||
recording.tabId,
|
// A closed tab no longer has content capture to flush.
|
||||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
recording.armedTabIds.delete(tabId);
|
||||||
]),
|
continue;
|
||||||
];
|
}
|
||||||
} catch {
|
|
||||||
// Fall back to the current recording tab.
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const tabId of tabIds) {
|
|
||||||
try {
|
try {
|
||||||
const response = await deps.sendToTab(tabId, stopMsg);
|
const response = await deps.sendToTab(tabId, stopMsg);
|
||||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
if (!isRecordStartAck(response)) {
|
||||||
throw new Error("content script did not confirm recorded steps");
|
throw new Error("content script did not confirm recorded steps");
|
||||||
}
|
}
|
||||||
|
recording.armedTabIds.delete(tabId);
|
||||||
} catch {
|
} catch {
|
||||||
if (tabId === recording.tabId) {
|
failed = true;
|
||||||
throw new Error("failed to flush recorded steps");
|
continue;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (deps.bypassOverlay) {
|
if (deps.bypassOverlay) {
|
||||||
try {
|
try {
|
||||||
@@ -362,6 +450,9 @@ async function stopRecordingOnAllAgentTabs(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (failed || recording.armedTabIds.size > 0) {
|
||||||
|
throw new Error("failed to flush recorded steps");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rearmRecording(
|
async function rearmRecording(
|
||||||
@@ -369,27 +460,40 @@ async function rearmRecording(
|
|||||||
targetTabId: number,
|
targetTabId: number,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
|
let resolveTracked!: () => void;
|
||||||
|
const tracked = new Promise<void>((resolve) => {
|
||||||
|
resolveTracked = resolve;
|
||||||
|
});
|
||||||
|
recording.rearmCallbacks.add(tracked);
|
||||||
|
|
||||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||||
// already hides the control chrome while activeRecord is set.
|
// already hides the control chrome while activeRecord is set.
|
||||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
try {
|
||||||
const startMsg: RecordStartMessage = {
|
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
type: RECORD_START,
|
if (recording.settled || recording.finishing) return false;
|
||||||
requestId: recording.requestId,
|
const startMsg: RecordStartMessage = {
|
||||||
startedAtMs: recording.startedAtMs,
|
type: RECORD_START,
|
||||||
};
|
requestId: recording.requestId,
|
||||||
try {
|
startedAtMs: recording.startedAtMs,
|
||||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
};
|
||||||
recording.tabId = targetTabId;
|
try {
|
||||||
return true;
|
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||||
} catch {
|
recording.armedTabIds.add(targetTabId);
|
||||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
recording.tabId = targetTabId;
|
||||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
return true;
|
||||||
|
} catch {
|
||||||
|
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||||
|
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
recording.rearmCallbacks.delete(tracked);
|
||||||
|
resolveTracked();
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||||
@@ -432,11 +536,86 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
const observeMainFrame = (
|
||||||
|
tabId: number,
|
||||||
|
url?: string,
|
||||||
|
causedByAction?: boolean,
|
||||||
|
transitionType?: string,
|
||||||
|
transitionQualifiers?: string[],
|
||||||
|
) => {
|
||||||
|
if (!url) return;
|
||||||
|
const direct = findRecordingByTabId(tabId);
|
||||||
|
const candidates = direct
|
||||||
|
? direct.acceptingNavigation
|
||||||
|
? [direct]
|
||||||
|
: []
|
||||||
|
: [...recordings.values()].filter(
|
||||||
|
(recording) => !recording.settled && recording.acceptingNavigation,
|
||||||
|
);
|
||||||
|
if (candidates.length === 0) return;
|
||||||
|
|
||||||
|
let resolveTracked!: () => void;
|
||||||
|
const tracked = new Promise<void>((resolve) => {
|
||||||
|
resolveTracked = resolve;
|
||||||
|
});
|
||||||
|
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
|
||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const recording = await findRecordingForTab(tabId, deps);
|
try {
|
||||||
if (recording && url) {
|
const recording = await findRecordingForTab(tabId, deps);
|
||||||
observeRecordedNavigation(recording, url, causedByAction);
|
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
enqueueRecordingAction(recording, async () => {
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
recording,
|
||||||
|
url,
|
||||||
|
causedByAction,
|
||||||
|
transitionType,
|
||||||
|
transitionQualifiers,
|
||||||
|
);
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (!cdp) return;
|
||||||
|
|
||||||
|
if (result.kind === "coalesce_redirect") {
|
||||||
|
scheduleRedirectLandingFlush(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
result.url,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.kind === "noop") return;
|
||||||
|
|
||||||
|
// A concrete navigation supersedes any in-flight redirect coalesce.
|
||||||
|
clearPendingRedirectLanding(recording.observation);
|
||||||
|
|
||||||
|
const draftIndex = result.index;
|
||||||
|
const lastDraft = recording.steps[draftIndex];
|
||||||
|
if (!lastDraft) return;
|
||||||
|
|
||||||
|
if (result.kind === "appended") {
|
||||||
|
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
|
||||||
|
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
|
||||||
|
}
|
||||||
|
scheduleDraftSettle(
|
||||||
|
recording.observation,
|
||||||
|
draftIndex,
|
||||||
|
draftIndex + 1,
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
|
||||||
|
resolveTracked();
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
};
|
};
|
||||||
@@ -458,18 +637,13 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||||
) => {
|
) => {
|
||||||
if (details.frameId !== 0) return;
|
if (details.frameId !== 0) return;
|
||||||
const causedByAction =
|
observeMainFrame(
|
||||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
details.tabId,
|
||||||
? true
|
details.url,
|
||||||
: details.transitionType === "typed" ||
|
undefined,
|
||||||
details.transitionType === "auto_bookmark" ||
|
details.transitionType,
|
||||||
details.transitionType === "generated" ||
|
details.transitionQualifiers,
|
||||||
details.transitionType === "keyword" ||
|
);
|
||||||
details.transitionType === "keyword_generated" ||
|
|
||||||
details.transitionType === "reload"
|
|
||||||
? false
|
|
||||||
: undefined;
|
|
||||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
|
||||||
};
|
};
|
||||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||||
@@ -487,6 +661,46 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||||
|
|
||||||
|
async function drainRecordingToStability(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||||
|
await Promise.all([...recording.navigationCallbacks]);
|
||||||
|
const actionTail = recording.actionQueue;
|
||||||
|
await actionTail;
|
||||||
|
await flushPendingRedirectLanding(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
await flushPendingSettles(recording.observation);
|
||||||
|
const redirectTail = recording.observation.redirectFlushQueue;
|
||||||
|
const settleTail = recording.observation.settleQueue;
|
||||||
|
await Promise.all([redirectTail, settleTail]);
|
||||||
|
|
||||||
|
if (
|
||||||
|
recording.navigationCallbacks.size === 0 &&
|
||||||
|
recording.actionQueue === actionTail &&
|
||||||
|
recording.observation.redirectFlushQueue === redirectTail &&
|
||||||
|
recording.observation.settleQueue === settleTail
|
||||||
|
) {
|
||||||
|
// No event or promise continuation can interleave with this synchronous
|
||||||
|
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||||
|
recording.acceptingNavigation = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
@@ -527,23 +741,49 @@ async function finishRecordingByRequest(
|
|||||||
if (recording.requestId !== requestId || recording.settled) continue;
|
if (recording.requestId !== requestId || recording.settled) continue;
|
||||||
const match = await findRecordingForTab(tabId, deps);
|
const match = await findRecordingForTab(tabId, deps);
|
||||||
if (match !== recording) continue;
|
if (match !== recording) continue;
|
||||||
await finishRecording(sessionId, deps);
|
await finishRecording(sessionId, deps, "user_finish");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
async function finishRecording(
|
||||||
|
sessionId: string,
|
||||||
|
deps: RecordDeps,
|
||||||
|
stoppedBy: StopReason,
|
||||||
|
): Promise<Trace | null> {
|
||||||
const recording = recordings.get(sessionId);
|
const recording = recordings.get(sessionId);
|
||||||
if (!recording || recording.settled || recording.finishing) return null;
|
if (!recording || recording.settled) return null;
|
||||||
|
if (recording.finishing) return recording.finishPromise;
|
||||||
recording.finishing = true;
|
recording.finishing = true;
|
||||||
|
recording.stoppedBy = stoppedBy;
|
||||||
|
|
||||||
await clearRearmTimersForRecording(recording, deps);
|
await clearRearmTimersForRecording(recording, deps);
|
||||||
|
await Promise.all([...recording.rearmCallbacks]);
|
||||||
try {
|
try {
|
||||||
|
// Disposing content capture may commit a final dirty fill. Flush content
|
||||||
|
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||||
|
// and the observation queues are drained.
|
||||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||||
} catch {
|
} catch {
|
||||||
recording.finishing = false;
|
recording.finishing = false;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
if (!(await drainRecordingToStability(recording, deps))) {
|
||||||
|
recording.finishing = false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
cancelPendingSettles(recording.observation);
|
||||||
|
inferMissingPostStates(recording.steps);
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (cdp) {
|
||||||
|
await settleUnsettledDrafts(
|
||||||
|
recording.observation,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
recording.settled = true;
|
recording.settled = true;
|
||||||
recordings.delete(sessionId);
|
recordings.delete(sessionId);
|
||||||
@@ -582,6 +822,8 @@ export async function handleRecordStart(
|
|||||||
});
|
});
|
||||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||||
const startedAtMs = Date.now();
|
const startedAtMs = Date.now();
|
||||||
|
const maxPageTokens = params.max_page_tokens;
|
||||||
|
const redactValues = params.redact_values ?? false;
|
||||||
recordings.set(params.session_id, {
|
recordings.set(params.session_id, {
|
||||||
requestId,
|
requestId,
|
||||||
tabId: target.tabId,
|
tabId: target.tabId,
|
||||||
@@ -599,6 +841,16 @@ export async function handleRecordStart(
|
|||||||
currentUrl: navigateUrl,
|
currentUrl: navigateUrl,
|
||||||
pendingNavigation: false,
|
pendingNavigation: false,
|
||||||
pendingNavigationDeadline: undefined,
|
pendingNavigationDeadline: undefined,
|
||||||
|
observation: createObservationState({ maxPageTokens, redactValues }),
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
redactValues,
|
||||||
|
armedTabIds: new Set(),
|
||||||
|
rearmCallbacks: new Set(),
|
||||||
|
navigationCallbacks: new Set(),
|
||||||
|
acceptingNavigation: true,
|
||||||
|
actionQueue: Promise.resolve(),
|
||||||
|
...(deps.cdp ? { cdp: deps.cdp } : {}),
|
||||||
});
|
});
|
||||||
// Observe navigations for the whole recording lifetime; attach before
|
// Observe navigations for the whole recording lifetime; attach before
|
||||||
// optional navigate so the destination load can rearm capture.
|
// optional navigate so the destination load can rearm capture.
|
||||||
@@ -722,6 +974,7 @@ export async function handleRecordStart(
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||||
|
active.armedTabIds.add(target.tabId);
|
||||||
} catch {
|
} catch {
|
||||||
await abortPending(true);
|
await abortPending(true);
|
||||||
return {
|
return {
|
||||||
@@ -731,6 +984,23 @@ export async function handleRecordStart(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deps.cdp) {
|
||||||
|
const activeRecording = recordings.get(params.session_id);
|
||||||
|
if (activeRecording) {
|
||||||
|
try {
|
||||||
|
await captureAndRegisterObservation(
|
||||||
|
activeRecording.observation,
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
startUrl,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Proceed without initial observation; steps may be dropped by reducer.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
const cancelled = await abortIfCancelled(true);
|
const cancelled = await abortIfCancelled(true);
|
||||||
if (cancelled) return cancelled;
|
if (cancelled) return cancelled;
|
||||||
@@ -755,7 +1025,7 @@ export async function handleRecordStop(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const trace = await finishRecording(params.session_id, deps);
|
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||||
if (!trace) {
|
if (!trace) {
|
||||||
return {
|
return {
|
||||||
code: "protocol_error",
|
code: "protocol_error",
|
||||||
|
|||||||
@@ -677,7 +677,7 @@ describe("captureViewModel", () => {
|
|||||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
it("preserves document rects while deriving viewport rects after scroll", async () => {
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_t: number, method: string) => {
|
send: vi.fn(async (_t: number, method: string) => {
|
||||||
if (method === "DOMSnapshot.enable") return {};
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
@@ -691,11 +691,10 @@ describe("captureViewModel", () => {
|
|||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
const { nodes } = await captureViewModel(cdp, 4);
|
const { nodes } = await captureViewModel(cdp, 4);
|
||||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||||
// local rect preserves viewport-relative scroll math.
|
expect(input?.documentRect).toEqual({ x: 400, y: 300, w: 200, h: 40 });
|
||||||
expect(div?.localRect?.y).toBe(-200);
|
expect(input?.localRect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
expect(input?.rect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ export interface CapturedNode {
|
|||||||
rect: Rect | null;
|
rect: Rect | null;
|
||||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||||
localRect?: Rect | null;
|
localRect?: Rect | null;
|
||||||
|
/** Frame-local document-relative CSS px as reported by DOMSnapshot. */
|
||||||
|
documentRect?: Rect | null;
|
||||||
paintOrder: number;
|
paintOrder: number;
|
||||||
position: string;
|
position: string;
|
||||||
pointerEvents: string;
|
pointerEvents: string;
|
||||||
@@ -788,6 +790,7 @@ function parseDocumentNodes(
|
|||||||
|
|
||||||
let rect: Rect | null = null;
|
let rect: Rect | null = null;
|
||||||
let localRect: Rect | null = null;
|
let localRect: Rect | null = null;
|
||||||
|
let documentRect: Rect | null = null;
|
||||||
let paintOrder = 0;
|
let paintOrder = 0;
|
||||||
let position = "static";
|
let position = "static";
|
||||||
let pointerEvents = "auto";
|
let pointerEvents = "auto";
|
||||||
@@ -796,12 +799,18 @@ function parseDocumentNodes(
|
|||||||
if (li !== undefined) {
|
if (li !== undefined) {
|
||||||
const b = dl?.bounds?.[li];
|
const b = dl?.bounds?.[li];
|
||||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||||
localRect = {
|
documentRect = {
|
||||||
x: b[0] / dpr - context.scrollX,
|
x: b[0] / dpr,
|
||||||
y: b[1] / dpr - context.scrollY,
|
y: b[1] / dpr,
|
||||||
w: b[2] / dpr,
|
w: b[2] / dpr,
|
||||||
h: b[3] / dpr,
|
h: b[3] / dpr,
|
||||||
};
|
};
|
||||||
|
localRect = {
|
||||||
|
x: documentRect.x - context.scrollX,
|
||||||
|
y: documentRect.y - context.scrollY,
|
||||||
|
w: documentRect.w,
|
||||||
|
h: documentRect.h,
|
||||||
|
};
|
||||||
rect = rectInTop(localRect, context);
|
rect = rectInTop(localRect, context);
|
||||||
}
|
}
|
||||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||||
@@ -825,6 +834,7 @@ function parseDocumentNodes(
|
|||||||
attrs,
|
attrs,
|
||||||
rect,
|
rect,
|
||||||
localRect,
|
localRect,
|
||||||
|
documentRect,
|
||||||
paintOrder,
|
paintOrder,
|
||||||
position,
|
position,
|
||||||
pointerEvents,
|
pointerEvents,
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
// stay structural (interface, not class) so the same JSON parses on both
|
// stay structural (interface, not class) so the same JSON parses on both
|
||||||
// sides without extra adapters.
|
// sides without extra adapters.
|
||||||
|
|
||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
|
||||||
export type RpcId = string;
|
export type RpcId = string;
|
||||||
|
|
||||||
export type ErrorCode =
|
export type ErrorCode =
|
||||||
@@ -656,26 +658,50 @@ export interface EmulateResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v3)
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export const TRACE_VERSION = 3;
|
||||||
|
export const VOM_FORMAT_VERSION = 1;
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
export interface TargetDescriptor {
|
||||||
|
ref?: string;
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
ctx?: string;
|
||||||
name_attr?: string;
|
unmatched?: boolean;
|
||||||
placeholder?: string;
|
|
||||||
nearby_label?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TraceEntry {
|
export interface TraceEntry {
|
||||||
start_url: string;
|
start_url: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PageRef {
|
export interface RecorderInfo {
|
||||||
|
bsk: string;
|
||||||
|
vom: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StopReason = "user_finish" | "cli_stop";
|
||||||
|
|
||||||
|
export interface TraceState {
|
||||||
id: string;
|
id: string;
|
||||||
url: string;
|
url: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Wire-only: full page observation text. */
|
||||||
|
body?: string;
|
||||||
|
/** Disk-only: filename under the bundle `pages/` directory. */
|
||||||
|
page?: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepResult {
|
||||||
|
state: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepCommon {
|
||||||
|
id: number;
|
||||||
|
state: string;
|
||||||
|
result: StepResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SelectedOption {
|
export interface SelectedOption {
|
||||||
@@ -683,66 +709,140 @@ export interface SelectedOption {
|
|||||||
label?: string;
|
label?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepEffect {
|
export type NavigationCause =
|
||||||
navigated_to: string;
|
| "user_typed"
|
||||||
|
| "link"
|
||||||
|
| "form_submit"
|
||||||
|
| "reload"
|
||||||
|
| "history"
|
||||||
|
| "script"
|
||||||
|
| "browser";
|
||||||
|
|
||||||
|
export type FillCommit = "enter" | "suggestion" | "blur";
|
||||||
|
|
||||||
|
/** Geometry reported by the content script at capture time. */
|
||||||
|
export interface CaptureGeometry {
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
scrollX: number;
|
||||||
|
scrollY: number;
|
||||||
|
position: string;
|
||||||
|
tag: string;
|
||||||
|
ownerFrameBackendNodeId?: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepCommon {
|
/** Capture/buffer draft before v3 reduction. */
|
||||||
id: number;
|
|
||||||
page: string;
|
|
||||||
effect?: StepEffect;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Capture/buffer draft before v2 reduction. */
|
|
||||||
export type DraftTraceStep =
|
export type DraftTraceStep =
|
||||||
| {
|
| {
|
||||||
op: "click";
|
op: "click";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "hover";
|
op: "hover";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "fill";
|
op: "fill";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
value: string;
|
value: string;
|
||||||
|
commit?: FillCommit;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "press";
|
op: "press";
|
||||||
key: string;
|
key: string;
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
modifiers?: KeyModifier[];
|
modifiers?: KeyModifier[];
|
||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "select";
|
op: "select";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
values: string[];
|
values: string[];
|
||||||
labels?: string[];
|
labels?: string[];
|
||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
op: "scroll";
|
||||||
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "navigate";
|
op: "navigate";
|
||||||
url: string;
|
url: string;
|
||||||
|
cause?: NavigationCause;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
/** Raw webNavigation transition metadata for cause mapping. */
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Exported record-only step (trace v2). */
|
/** Exported record-only step (trace v3). */
|
||||||
export type Step =
|
export type Step =
|
||||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
| ({ op: "navigate" } & StepCommon & { to: string; cause: NavigationCause })
|
||||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||||
| ({ op: "fill" } & StepCommon & {
|
| ({ op: "fill" } & StepCommon & {
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
value: string;
|
value: string;
|
||||||
|
commit: FillCommit;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
})
|
})
|
||||||
| ({ op: "select" } & StepCommon & {
|
| ({ op: "select" } & StepCommon & {
|
||||||
@@ -753,14 +853,18 @@ export type Step =
|
|||||||
key: string;
|
key: string;
|
||||||
modifiers?: KeyModifier[];
|
modifiers?: KeyModifier[];
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptor;
|
||||||
});
|
})
|
||||||
|
| ({ op: "scroll" } & StepCommon);
|
||||||
|
|
||||||
export interface Trace {
|
export interface Trace {
|
||||||
|
version: number;
|
||||||
recorded_at: string;
|
recorded_at: string;
|
||||||
started_at?: string;
|
started_at?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
|
stopped_by: StopReason;
|
||||||
entry: TraceEntry;
|
entry: TraceEntry;
|
||||||
pages: PageRef[];
|
recorder: RecorderInfo;
|
||||||
|
states: TraceState[];
|
||||||
steps: Step[];
|
steps: Step[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -769,6 +873,8 @@ export interface RecordStartParams {
|
|||||||
tab_id?: number;
|
tab_id?: number;
|
||||||
url?: string;
|
url?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
|
max_page_tokens?: number;
|
||||||
|
redact_values?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStartResult {
|
export interface RecordStartResult {
|
||||||
|
|||||||
@@ -24,6 +24,11 @@ function scene(nodes: VomNode[]): VomScene {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Compare refs without recording-metadata fields (role/name/ctx/line). */
|
||||||
|
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||||
|
return refs.map(({ ref: id, backendNodeId }) => ({ ref: id, backendNodeId }));
|
||||||
|
}
|
||||||
|
|
||||||
describe("renderVom single-layer page", () => {
|
describe("renderVom single-layer page", () => {
|
||||||
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
||||||
const out = renderVom(
|
const out = renderVom(
|
||||||
@@ -51,7 +56,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
' @e2 button "加入购物车"',
|
' @e2 button "加入购物车"',
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.refs).toEqual([
|
expect(coreRefs(out.refs)).toEqual([
|
||||||
{ ref: "e1", backendNodeId: 3 },
|
{ ref: "e1", backendNodeId: 3 },
|
||||||
{ ref: "e2", backendNodeId: 6 },
|
{ ref: "e2", backendNodeId: 6 },
|
||||||
]);
|
]);
|
||||||
@@ -72,7 +77,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(' heading "Title"');
|
expect(out.text).toContain(' heading "Title"');
|
||||||
expect(out.text).toContain(' img "Logo"');
|
expect(out.text).toContain(' img "Logo"');
|
||||||
expect(out.text).toContain(' @e1 button "Continue"');
|
expect(out.text).toContain(' @e1 button "Continue"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("assigns refs to listbox controls", () => {
|
it("assigns refs to listbox controls", () => {
|
||||||
@@ -84,7 +89,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("annotates external links regardless of role casing", () => {
|
it("annotates external links regardless of role casing", () => {
|
||||||
@@ -121,7 +126,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(" alertdialog");
|
expect(out.text).toContain(" alertdialog");
|
||||||
expect(out.text).toContain(" section");
|
expect(out.text).toContain(" section");
|
||||||
expect(out.text).toContain(' @e1 button "Close"');
|
expect(out.text).toContain(' @e1 button "Close"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("skips generic nodes without adding an extra indentation level", () => {
|
it("skips generic nodes without adding an extra indentation level", () => {
|
||||||
@@ -192,7 +197,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(" section");
|
expect(out.text).toContain(" section");
|
||||||
expect(out.text).not.toContain("Too deep");
|
expect(out.text).not.toContain("Too deep");
|
||||||
expect(out.text).toContain(' @e1 button "Later"');
|
expect(out.text).toContain(' @e1 button "Later"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||||
@@ -227,7 +232,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Open settings"');
|
expect(out.text).toContain('@e1 button "Open settings"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||||
@@ -257,7 +262,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).not.toContain('@e1 button "Card"');
|
expect(out.text).not.toContain('@e1 button "Card"');
|
||||||
expect(out.text).toContain('@e1 button "Details"');
|
expect(out.text).toContain('@e1 button "Details"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not recover clickable wrappers around native controls", () => {
|
it("does not recover clickable wrappers around native controls", () => {
|
||||||
@@ -280,7 +285,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).not.toContain('@e1 button "Checkout"');
|
expect(out.text).not.toContain('@e1 button "Checkout"');
|
||||||
expect(out.text).toContain('@e1 button "Pay"');
|
expect(out.text).toContain('@e1 button "Pay"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("adds context to duplicate weak action labels", () => {
|
it("adds context to duplicate weak action labels", () => {
|
||||||
@@ -343,7 +348,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).toContain("@layers 1 focus=L1");
|
expect(out.text).toContain("@layers 1 focus=L1");
|
||||||
expect(out.text).toContain('@e1 button "Background"');
|
expect(out.text).toContain('@e1 button "Background"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||||
@@ -386,7 +391,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).not.toContain("active-region");
|
expect(out.text).not.toContain("active-region");
|
||||||
expect(out.text).not.toContain("Background");
|
expect(out.text).not.toContain("Background");
|
||||||
expect(out.text).toContain('@e1 button "Foreground"');
|
expect(out.text).toContain('@e1 button "Foreground"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("renders conditional surface items inline on the trigger", () => {
|
it("renders conditional surface items inline on the trigger", () => {
|
||||||
@@ -405,7 +410,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||||
@@ -431,7 +436,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
" Bob - great sound",
|
" Bob - great sound",
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not render redundant children inside named ref controls", () => {
|
it("does not render redundant children inside named ref controls", () => {
|
||||||
@@ -445,7 +450,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Save"');
|
expect(out.text).toContain('@e1 button "Save"');
|
||||||
expect(out.text).not.toContain("disk icon");
|
expect(out.text).not.toContain("disk icon");
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not render redundant children inside native text inputs", () => {
|
it("does not render redundant children inside native text inputs", () => {
|
||||||
@@ -612,7 +617,7 @@ describe("renderVom double-layer page", () => {
|
|||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.text).not.toContain("底层按钮");
|
expect(out.text).not.toContain("底层按钮");
|
||||||
expect(out.refs).toEqual([
|
expect(coreRefs(out.refs)).toEqual([
|
||||||
{ ref: "e1", backendNodeId: 4 },
|
{ ref: "e1", backendNodeId: 4 },
|
||||||
{ ref: "e2", backendNodeId: 5 },
|
{ ref: "e2", backendNodeId: 5 },
|
||||||
]);
|
]);
|
||||||
@@ -664,4 +669,24 @@ describe("renderVom double-layer page", () => {
|
|||||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("redactValues omits literal form values", () => {
|
||||||
|
const out = renderVom(
|
||||||
|
scene([
|
||||||
|
node({ id: 1, role: "RootWebArea", name: "Form" }),
|
||||||
|
node({
|
||||||
|
id: 2,
|
||||||
|
parentId: 1,
|
||||||
|
role: "textbox",
|
||||||
|
name: "邮箱",
|
||||||
|
value: "user@example.com",
|
||||||
|
inputState: "filled",
|
||||||
|
tag: "input",
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
{ redactValues: true },
|
||||||
|
);
|
||||||
|
expect(out.text).toContain("[filled]");
|
||||||
|
expect(out.text).not.toContain("user@example.com");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -5,6 +5,7 @@ export type {
|
|||||||
CondSurface,
|
CondSurface,
|
||||||
LayerKind,
|
LayerKind,
|
||||||
Rect,
|
Rect,
|
||||||
|
RenderedRef,
|
||||||
Viewport,
|
Viewport,
|
||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import type {
|
|||||||
BlockingLayer,
|
BlockingLayer,
|
||||||
CondSurface,
|
CondSurface,
|
||||||
Rect,
|
Rect,
|
||||||
|
RenderedRef,
|
||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
VomResult,
|
VomResult,
|
||||||
@@ -606,6 +607,7 @@ function renderNodeLine(
|
|||||||
ref: string | undefined,
|
ref: string | undefined,
|
||||||
context: string[] = [],
|
context: string[] = [],
|
||||||
surface: CondSurface | undefined = undefined,
|
surface: CondSurface | undefined = undefined,
|
||||||
|
redactValues = false,
|
||||||
): string {
|
): string {
|
||||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||||
|
|
||||||
@@ -627,11 +629,12 @@ function renderNodeLine(
|
|||||||
}
|
}
|
||||||
const placeholder = cleaned(node.placeholder);
|
const placeholder = cleaned(node.placeholder);
|
||||||
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
||||||
const value = node.sensitive
|
const value =
|
||||||
? rawValue !== undefined
|
node.sensitive || redactValues
|
||||||
? SENSITIVE_MASK
|
? rawValue !== undefined
|
||||||
: undefined
|
? SENSITIVE_MASK
|
||||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
: undefined
|
||||||
|
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||||
|
|
||||||
if (surface && surface.subItems.length > 0) {
|
if (surface && surface.subItems.length > 0) {
|
||||||
@@ -684,11 +687,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
|
|||||||
|
|
||||||
interface RenderState {
|
interface RenderState {
|
||||||
lines: string[];
|
lines: string[];
|
||||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
refs: RenderedRef[];
|
||||||
nextRef: number;
|
nextRef: number;
|
||||||
tokens: number;
|
tokens: number;
|
||||||
maxDepth: number;
|
maxDepth: number;
|
||||||
maxTokens: number;
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
stopped: boolean;
|
stopped: boolean;
|
||||||
children: Map<number | null, VomNode[]>;
|
children: Map<number | null, VomNode[]>;
|
||||||
@@ -753,7 +757,7 @@ function renderTree(
|
|||||||
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||||
const context = ref ? handleContext(node, state) : [];
|
const context = ref ? handleContext(node, state) : [];
|
||||||
const surface = state.surfaceMap.get(node.id);
|
const surface = state.surfaceMap.get(node.id);
|
||||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
|
||||||
const nextTokens = state.tokens + estimateTokens(line);
|
const nextTokens = state.tokens + estimateTokens(line);
|
||||||
if (nextTokens > state.maxTokens) {
|
if (nextTokens > state.maxTokens) {
|
||||||
state.truncated = true;
|
state.truncated = true;
|
||||||
@@ -764,7 +768,16 @@ function renderTree(
|
|||||||
state.lines.push(line);
|
state.lines.push(line);
|
||||||
state.tokens = nextTokens;
|
state.tokens = nextTokens;
|
||||||
if (ref) {
|
if (ref) {
|
||||||
state.refs.push({ ref, backendNodeId: node.id });
|
const lineIndex = state.lines.length - 1;
|
||||||
|
const refName = cleaned(node.name);
|
||||||
|
state.refs.push({
|
||||||
|
ref,
|
||||||
|
backendNodeId: node.id,
|
||||||
|
role: node.role,
|
||||||
|
...(refName ? { name: refName } : {}),
|
||||||
|
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||||
|
line: lineIndex,
|
||||||
|
});
|
||||||
state.nextRef += 1;
|
state.nextRef += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -793,6 +806,7 @@ function renderNodes(
|
|||||||
tokens: estimateTokens(initialLines.join("\n")),
|
tokens: estimateTokens(initialLines.join("\n")),
|
||||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||||
|
redactValues: options.redactValues ?? false,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
stopped: false,
|
stopped: false,
|
||||||
children,
|
children,
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ export interface VomScene {
|
|||||||
export interface VomOptions {
|
export interface VomOptions {
|
||||||
maxDepth?: number;
|
maxDepth?: number;
|
||||||
maxTokens?: number;
|
maxTokens?: number;
|
||||||
|
/**
|
||||||
|
* When true, all form values render as [filled]/[empty] without literal values.
|
||||||
|
*/
|
||||||
|
redactValues?: boolean;
|
||||||
/**
|
/**
|
||||||
* Experimental: filter refs that are geometrically blocked by foreground
|
* Experimental: filter refs that are geometrically blocked by foreground
|
||||||
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
||||||
@@ -79,9 +83,19 @@ export interface ActiveScopeBlock {
|
|||||||
lines: string[];
|
lines: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RenderedRef {
|
||||||
|
ref: string;
|
||||||
|
backendNodeId: number;
|
||||||
|
role?: string;
|
||||||
|
name?: string;
|
||||||
|
ctx?: string;
|
||||||
|
/** Zero-based line index in `VomResult.text`. */
|
||||||
|
line: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface VomResult {
|
export interface VomResult {
|
||||||
text: string;
|
text: string;
|
||||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
refs: RenderedRef[];
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user