mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(record): safety problem and bug fix
This commit is contained in:
@@ -11,7 +11,7 @@ function sessionWithInput(redactValues = false): RecordingObservationSession {
|
|||||||
const session = new RecordingObservationSession({ redactValues });
|
const session = new RecordingObservationSession({ redactValues });
|
||||||
const state = session.registry.register({
|
const state = session.registry.register({
|
||||||
url: URL,
|
url: URL,
|
||||||
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||||
});
|
});
|
||||||
session.cursor.lastSettled = {
|
session.cursor.lastSettled = {
|
||||||
stateId: state.id,
|
stateId: state.id,
|
||||||
@@ -34,7 +34,7 @@ function sessionWithInput(redactValues = false): RecordingObservationSession {
|
|||||||
return session;
|
return session;
|
||||||
}
|
}
|
||||||
|
|
||||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
function finalizedFillTrace(value: string, redactValues: boolean) {
|
||||||
const session = sessionWithInput(redactValues);
|
const session = sessionWithInput(redactValues);
|
||||||
const draft: RecordingDraftStep = {
|
const draft: RecordingDraftStep = {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
@@ -53,15 +53,20 @@ function finalizedFillBody(value: string, redactValues: boolean): string {
|
|||||||
startedAt: "2026-08-12T00:00:00.000Z",
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
stoppedBy: "user_finish",
|
stoppedBy: "user_finish",
|
||||||
bskVersion: "test",
|
bskVersion: "test",
|
||||||
}).states[0]!.body;
|
redactValues,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
return finalizedFillTrace(value, redactValues).states[0]!.body;
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("record observation annotations", () => {
|
describe("record observation annotations", () => {
|
||||||
it("omits a fill literal when values are redacted", () => {
|
it("omits a fill literal when values are redacted", () => {
|
||||||
const secret = "hunter2-private";
|
const secret = "hunter2-private";
|
||||||
const body = finalizedFillBody(secret, true);
|
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
|
||||||
expect(body).toContain("step 1: fill");
|
expect(dumped).toContain("step 1: fill");
|
||||||
expect(body).not.toContain(secret);
|
expect(dumped).not.toContain(secret);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps ordinary fill details", () => {
|
it("keeps ordinary fill details", () => {
|
||||||
@@ -73,7 +78,7 @@ describe("record observation annotations", () => {
|
|||||||
const state = registry.register({
|
const state = registry.register({
|
||||||
url: "https://example.com/a\nb",
|
url: "https://example.com/a\nb",
|
||||||
title: "hello\nworld",
|
title: "hello\nworld",
|
||||||
rawVomText: "@vom 1",
|
vomText: "@vom 1",
|
||||||
});
|
});
|
||||||
const trace = buildTraceV3({
|
const trace = buildTraceV3({
|
||||||
registry,
|
registry,
|
||||||
@@ -91,18 +96,18 @@ describe("recording state ownership", () => {
|
|||||||
it("deduplicates within one recording and isolates ids between recordings", () => {
|
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||||
const first = new RecordingStateRegistry();
|
const first = new RecordingStateRegistry();
|
||||||
const second = new RecordingStateRegistry();
|
const second = new RecordingStateRegistry();
|
||||||
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||||
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||||
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
|
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||||
const registry = new RecordingStateRegistry();
|
const registry = new RecordingStateRegistry();
|
||||||
registry.register({ url: URL, rawVomText: "same" });
|
registry.register({ url: URL, vomText: "same" });
|
||||||
const state = registry.register({
|
const state = registry.register({
|
||||||
url: URL,
|
url: URL,
|
||||||
title: "Login",
|
title: "Login",
|
||||||
rawVomText: "same",
|
vomText: "same",
|
||||||
truncated: true,
|
truncated: true,
|
||||||
});
|
});
|
||||||
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
|||||||
function observation() {
|
function observation() {
|
||||||
return {
|
return {
|
||||||
rootFrameId: "root",
|
rootFrameId: "root",
|
||||||
index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: [], refs: [] }),
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
url: "https://example.com/",
|
url: "https://example.com/",
|
||||||
title: "Example",
|
title: "Example",
|
||||||
vomText: '@vom 1\nRootWebArea "Example"',
|
vomText: '@vom 1\nRootWebArea "Example"',
|
||||||
@@ -95,4 +95,39 @@ describe("RecordingObservationRuntime", () => {
|
|||||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
||||||
expect(trace.states).toHaveLength(1);
|
expect(trace.states).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps runtime matching data out of the serialized trace", async () => {
|
||||||
|
const captured = observation();
|
||||||
|
captured.index = new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
|
||||||
|
});
|
||||||
|
captureRecordingObservation.mockResolvedValueOnce(captured);
|
||||||
|
const recording = runtime();
|
||||||
|
|
||||||
|
await recording.captureInitial(7);
|
||||||
|
const dumped = JSON.stringify(
|
||||||
|
recording.buildTrace({
|
||||||
|
drafts: [],
|
||||||
|
startedAt: "2026-08-19T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "0.1.6",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(dumped).toContain("RootWebArea");
|
||||||
|
expect(dumped).not.toContain("matchNodes");
|
||||||
|
expect(dumped).not.toContain("localRect");
|
||||||
|
expect(dumped).not.toContain("backendNodeId");
|
||||||
|
expect(dumped).not.toContain("ObservationNodeIndex");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,6 +6,33 @@ import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
|||||||
import type { RecordingDraftStep } from "../recording/types";
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
describe("trace reducer v3", () => {
|
describe("trace reducer v3", () => {
|
||||||
|
it("removes form literals when value redaction is requested", () => {
|
||||||
|
const reduced = reduceTraceStepsV3(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
value: "private@example.com",
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "select",
|
||||||
|
values: ["private-account-id"],
|
||||||
|
labels: ["Private account"],
|
||||||
|
preStateId: "s1",
|
||||||
|
postStateId: "s1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{ redactValues: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
|
||||||
|
expect(reduced.steps[1]).toMatchObject({ op: "select" });
|
||||||
|
expect(reduced.steps[1]).not.toHaveProperty("selection");
|
||||||
|
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
|
||||||
|
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
|
||||||
|
});
|
||||||
|
|
||||||
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||||
const drafts: RecordingDraftStep[] = [
|
const drafts: RecordingDraftStep[] = [
|
||||||
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||||
@@ -36,7 +63,7 @@ describe("trace reducer v3", () => {
|
|||||||
|
|
||||||
it("builds the wire model from protocol constants", () => {
|
it("builds the wire model from protocol constants", () => {
|
||||||
const registry = new RecordingStateRegistry();
|
const registry = new RecordingStateRegistry();
|
||||||
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
|
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
|
||||||
const trace = buildTraceV3({
|
const trace = buildTraceV3({
|
||||||
registry,
|
registry,
|
||||||
drafts: [
|
drafts: [
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ export class RecordingObservationSession {
|
|||||||
const state = this.registry.register({
|
const state = this.registry.register({
|
||||||
url: captured.url,
|
url: captured.url,
|
||||||
title: captured.title,
|
title: captured.title,
|
||||||
rawVomText: captured.vomText,
|
vomText: captured.vomText,
|
||||||
truncated: captured.truncated,
|
truncated: captured.truncated,
|
||||||
});
|
});
|
||||||
const observation: RegisteredObservation = {
|
const observation: RegisteredObservation = {
|
||||||
|
|||||||
@@ -171,6 +171,7 @@ export class RecordingObservationRuntime {
|
|||||||
startUrl: input.startUrl,
|
startUrl: input.startUrl,
|
||||||
stoppedBy: input.stoppedBy,
|
stoppedBy: input.stoppedBy,
|
||||||
bskVersion: input.bskVersion,
|
bskVersion: input.bskVersion,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,8 @@ export interface RecordedStateEntry {
|
|||||||
id: string;
|
id: string;
|
||||||
url: string;
|
url: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
rawVomText: string;
|
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
|
||||||
|
vomText: string;
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
stepsHere: number[];
|
stepsHere: number[];
|
||||||
}
|
}
|
||||||
@@ -19,10 +20,10 @@ export class RecordingStateRegistry {
|
|||||||
register(input: {
|
register(input: {
|
||||||
url: string;
|
url: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
rawVomText: string;
|
vomText: string;
|
||||||
truncated?: boolean;
|
truncated?: boolean;
|
||||||
}): RecordedStateEntry {
|
}): RecordedStateEntry {
|
||||||
const identity = stateIdentity(input.url, input.rawVomText);
|
const identity = stateIdentity(input.url, input.vomText);
|
||||||
const existingId = this.#idByIdentity.get(identity);
|
const existingId = this.#idByIdentity.get(identity);
|
||||||
if (existingId) {
|
if (existingId) {
|
||||||
const existing = this.#entriesById.get(existingId)!;
|
const existing = this.#entriesById.get(existingId)!;
|
||||||
@@ -35,7 +36,7 @@ export class RecordingStateRegistry {
|
|||||||
id: `s${this.#nextId}`,
|
id: `s${this.#nextId}`,
|
||||||
url: input.url,
|
url: input.url,
|
||||||
...(input.title ? { title: input.title } : {}),
|
...(input.title ? { title: input.title } : {}),
|
||||||
rawVomText: input.rawVomText,
|
vomText: input.vomText,
|
||||||
truncated: input.truncated ?? false,
|
truncated: input.truncated ?? false,
|
||||||
stepsHere: [],
|
stepsHere: [],
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -39,8 +39,9 @@ export function buildTraceV3(input: {
|
|||||||
startUrl?: string;
|
startUrl?: string;
|
||||||
stoppedBy: StopReason;
|
stoppedBy: StopReason;
|
||||||
bskVersion: string;
|
bskVersion: string;
|
||||||
|
redactValues?: boolean;
|
||||||
}): TraceV3 {
|
}): TraceV3 {
|
||||||
const reduced = reduceTraceStepsV3(input.drafts);
|
const reduced = reduceTraceStepsV3(input.drafts, { redactValues: input.redactValues });
|
||||||
const entries = publishedEntries(input.registry, reduced.steps);
|
const entries = publishedEntries(input.registry, reduced.steps);
|
||||||
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
const annotationsByState = new Map<string, StepAnnotation[]>();
|
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||||
@@ -65,7 +66,7 @@ export function buildTraceV3(input: {
|
|||||||
url: entry.url,
|
url: entry.url,
|
||||||
title: entry.title,
|
title: entry.title,
|
||||||
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||||
vomText: entry.rawVomText,
|
vomText: entry.vomText,
|
||||||
annotations: annotationsByState.get(entry.id) ?? [],
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
stepIdByDraftId: reduced.stepIdByDraftId,
|
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ function selection(values: string[], labels?: string[]): Array<{ value: string;
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolean): StepV3 | null {
|
||||||
if (!shouldIncludeDraft(draft)) return null;
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
const state = draft.preStateId ?? draft.postStateId;
|
const state = draft.preStateId ?? draft.postStateId;
|
||||||
const resultState = draft.postStateId ?? draft.preStateId;
|
const resultState = draft.postStateId ?? draft.preStateId;
|
||||||
@@ -81,13 +81,14 @@ function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
|||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
};
|
};
|
||||||
case "fill":
|
case "fill":
|
||||||
|
const fillIsRedacted = redactValues || draft.redacted === true;
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
...common,
|
...common,
|
||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
value: draft.value,
|
value: fillIsRedacted ? "***" : draft.value,
|
||||||
commit: draft.commit ?? "blur",
|
commit: draft.commit ?? "blur",
|
||||||
...(draft.redacted ? { redacted: true } : {}),
|
...(fillIsRedacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
case "press":
|
case "press":
|
||||||
return {
|
return {
|
||||||
@@ -104,7 +105,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
|||||||
op: "select",
|
op: "select",
|
||||||
...common,
|
...common,
|
||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
selection: selection(draft.values, draft.labels),
|
...(!redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
|
||||||
};
|
};
|
||||||
case "scroll":
|
case "scroll":
|
||||||
return { op: "scroll", ...common };
|
return { op: "scroll", ...common };
|
||||||
@@ -116,11 +117,14 @@ export interface ReducedTraceV3 {
|
|||||||
stepIdByDraftId: Map<number, number>;
|
stepIdByDraftId: Map<number, number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
|
export function reduceTraceStepsV3(
|
||||||
|
steps: RecordingDraftStep[],
|
||||||
|
options: { redactValues?: boolean } = {},
|
||||||
|
): ReducedTraceV3 {
|
||||||
const output: StepV3[] = [];
|
const output: StepV3[] = [];
|
||||||
const stepIdByDraftId = new Map<number, number>();
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||||
const step = reduceDraft(draft, output.length + 1);
|
const step = reduceDraft(draft, output.length + 1, options.redactValues ?? false);
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
output.push(step);
|
output.push(step);
|
||||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||||
|
|||||||
Reference in New Issue
Block a user