mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): omit raw AX/DOM trees from record-safe observations
Return only frame identity and node geometry for matching so a serialized capture result cannot leak form values.
This commit is contained in:
1 parent
6aa931b60a
commit
20554508db
3 files changed
+214
-10
No files matched your search
@@ -2909,6 +2909,87 @@ describe("handleSnapshot", () => {
|
|||||||
],
|
],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function secretsSnapshotReply(secrets: {
|
||||||
|
email: string;
|
||||||
|
password: string;
|
||||||
|
otp: string;
|
||||||
|
card: string;
|
||||||
|
}) {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"input",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"type",
|
||||||
|
"text",
|
||||||
|
"password",
|
||||||
|
"autocomplete",
|
||||||
|
"one-time-code",
|
||||||
|
"cc-number",
|
||||||
|
"value",
|
||||||
|
secrets.email,
|
||||||
|
secrets.password,
|
||||||
|
secrets.otp,
|
||||||
|
secrets.card,
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
const style = [i("static"), i("auto"), i("auto")];
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||||
|
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||||
|
[
|
||||||
|
i("type"),
|
||||||
|
i("text"),
|
||||||
|
i("autocomplete"),
|
||||||
|
i("one-time-code"),
|
||||||
|
i("value"),
|
||||||
|
i(secrets.otp),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
i("type"),
|
||||||
|
i("text"),
|
||||||
|
i("autocomplete"),
|
||||||
|
i("cc-number"),
|
||||||
|
i("value"),
|
||||||
|
i(secrets.card),
|
||||||
|
],
|
||||||
|
],
|
||||||
|
inputValue: {
|
||||||
|
index: [2, 3, 4, 5],
|
||||||
|
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3, 4, 5],
|
||||||
|
styles: [style, style, style, style, style],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[10, 10, 200, 32],
|
||||||
|
[10, 50, 200, 32],
|
||||||
|
[10, 90, 200, 32],
|
||||||
|
[10, 130, 200, 32],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2, 3, 4],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const VP_METRICS = {
|
const VP_METRICS = {
|
||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
};
|
};
|
||||||
@@ -3063,20 +3144,96 @@ describe("handleSnapshot", () => {
|
|||||||
line: expect.any(Number),
|
line: expect.any(Number),
|
||||||
});
|
});
|
||||||
expect(result.rootFrameId).toBe("root");
|
expect(result.rootFrameId).toBe("root");
|
||||||
expect(result.frameDocuments).toHaveLength(1);
|
expect(result).not.toHaveProperty("frameDocuments");
|
||||||
expect(result.frameDocuments[0]?.domNodes.some((node) => node.backendNodeId === 13)).toBe(true);
|
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||||
|
expect(result.nodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 13,
|
||||||
|
tag: "input",
|
||||||
|
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps frame documents and rendered refs on the same frame identity", async () => {
|
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||||
|
const secrets = {
|
||||||
|
email: "user@example.com",
|
||||||
|
password: "hunter2",
|
||||||
|
otp: "847291",
|
||||||
|
card: "4111111111111111",
|
||||||
|
};
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
childIds: ["email", "password", "otp", "card"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "email",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Email" },
|
||||||
|
value: { value: secrets.email },
|
||||||
|
backendDOMNodeId: 12,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "password",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Password" },
|
||||||
|
value: { value: secrets.password },
|
||||||
|
backendDOMNodeId: 13,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "otp",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Code" },
|
||||||
|
value: { value: secrets.otp },
|
||||||
|
backendDOMNodeId: 14,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "card",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Card" },
|
||||||
|
value: { value: secrets.card },
|
||||||
|
backendDOMNodeId: 15,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||||
|
|
||||||
|
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||||
|
redactValues: true,
|
||||||
|
});
|
||||||
|
const dumped = JSON.stringify(result);
|
||||||
|
|
||||||
|
expect(result).not.toHaveProperty("frameDocuments");
|
||||||
|
expect(dumped).not.toContain(secrets.email);
|
||||||
|
expect(dumped).not.toContain(secrets.password);
|
||||||
|
expect(dumped).not.toContain(secrets.otp);
|
||||||
|
expect(dumped).not.toContain(secrets.card);
|
||||||
|
expect(dumped).not.toContain("formValue");
|
||||||
|
expect(dumped).not.toContain("axNodes");
|
||||||
|
expect(result.text).toContain("•••");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||||
const deps = makeFrameAwareDeps();
|
const deps = makeFrameAwareDeps();
|
||||||
|
|
||||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||||
|
|
||||||
expect(result.rootFrameId).toBe("main");
|
expect(result.rootFrameId).toBe("main");
|
||||||
expect(result.frameDocuments).toHaveLength(2);
|
expect(result.frames).toHaveLength(2);
|
||||||
expect(result.frameDocuments.find((document) => document.frameId === "child")?.target).toEqual({
|
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||||
tabId: 4,
|
frameId: "child",
|
||||||
sessionId: "child-session",
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
});
|
||||||
|
expect(result.nodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||||
|
frameId: "child",
|
||||||
|
tag: "button",
|
||||||
});
|
});
|
||||||
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||||
frameId: "child",
|
frameId: "child",
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
export {
|
export {
|
||||||
|
type CaptureVomFrame,
|
||||||
|
type CaptureVomNode,
|
||||||
type CaptureVomObservationOptions,
|
type CaptureVomObservationOptions,
|
||||||
type CaptureVomObservationResult,
|
type CaptureVomObservationResult,
|
||||||
captureVomObservation,
|
captureVomObservation,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
applyVomInteractionRecovery,
|
applyVomInteractionRecovery,
|
||||||
type CondSurface,
|
type CondSurface,
|
||||||
isVomReferenceNode,
|
isVomReferenceNode,
|
||||||
|
type Rect,
|
||||||
type RenderedRef,
|
type RenderedRef,
|
||||||
renderVom,
|
renderVom,
|
||||||
type VomNode,
|
type VomNode,
|
||||||
@@ -1590,15 +1591,58 @@ export interface CaptureVomObservationOptions extends VomOptions {
|
|||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Frame identity needed to click an `@eN` that lives in an iframe. */
|
||||||
|
export interface CaptureVomFrame {
|
||||||
|
frameId: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
parentFrameId?: string;
|
||||||
|
ownerBackendNodeId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Geometry index for matching a recorded click to a captured node. */
|
||||||
|
export interface CaptureVomNode {
|
||||||
|
frameId: string;
|
||||||
|
backendNodeId: number;
|
||||||
|
tag: string;
|
||||||
|
rect: Rect | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record-safe observation: rendered VOM text plus the matching index.
|
||||||
|
* Raw AX/DOM trees are not returned — they carry form values.
|
||||||
|
*/
|
||||||
export interface CaptureVomObservationResult {
|
export interface CaptureVomObservationResult {
|
||||||
text: string;
|
text: string;
|
||||||
refs: RenderedRef[];
|
refs: RenderedRef[];
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
rootFrameId: string;
|
rootFrameId: string;
|
||||||
frameDocuments: CapturedFrameDocument<CdpAxNode>[];
|
frames: CaptureVomFrame[];
|
||||||
|
nodes: CaptureVomNode[];
|
||||||
surfaceProbes?: CapturedSurfaceProbe[];
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordSafeFrames(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomFrame[] {
|
||||||
|
return documents.map((document) => ({
|
||||||
|
frameId: document.frameId,
|
||||||
|
target: document.target,
|
||||||
|
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||||
|
...(document.ownerBackendNodeId !== undefined
|
||||||
|
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||||
|
: {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordSafeNodes(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomNode[] {
|
||||||
|
return documents.flatMap((document) =>
|
||||||
|
document.domNodes.map((node) => ({
|
||||||
|
frameId: document.frameId,
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
tag: node.tag,
|
||||||
|
rect: node.rect,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function captureVomObservation(
|
export async function captureVomObservation(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -1628,7 +1672,8 @@ export async function captureVomObservation(
|
|||||||
refs: rendered.refs,
|
refs: rendered.refs,
|
||||||
truncated: rendered.truncated,
|
truncated: rendered.truncated,
|
||||||
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||||
frameDocuments: documents,
|
frames: recordSafeFrames(documents),
|
||||||
|
nodes: recordSafeNodes(documents),
|
||||||
surfaceProbes: captured.surfaceProbes,
|
surfaceProbes: captured.surfaceProbes,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -1674,7 +1719,7 @@ async function handleVomObservation(
|
|||||||
});
|
});
|
||||||
throwIfAborted(signal, toolName);
|
throwIfAborted(signal, toolName);
|
||||||
const targetByFrameId = new Map(
|
const targetByFrameId = new Map(
|
||||||
observation.frameDocuments.map((document) => [document.frameId, document.target]),
|
observation.frames.map((frame) => [frame.frameId, frame.target]),
|
||||||
);
|
);
|
||||||
ctx.refStore.replace(
|
ctx.refStore.replace(
|
||||||
observation.refs.map((ref) => {
|
observation.refs.map((ref) => {
|
||||||
|
|||||||
Reference in new issue
Block a user