mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): safety policy and rebase bug fix
This commit is contained in:
@@ -54,12 +54,11 @@ import {
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
import type { FrameDocument } from "./vom/frame-document";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
buildSemanticGraph,
|
||||
buildSemanticVomScene,
|
||||
@@ -812,7 +811,7 @@ export function buildFrameVomScene(
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const scene = buildSemanticVomScene({
|
||||
documents: documents as FrameDocument<SemanticAxNode>[],
|
||||
documents,
|
||||
viewport: captured.viewport,
|
||||
rootFrameId: captured.rootFrameId,
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
@@ -1038,7 +1037,7 @@ export async function captureVomObservation(
|
||||
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
|
||||
: documents;
|
||||
const semanticGraph = buildSemanticGraph({
|
||||
documents: normalizedDocuments as FrameDocument<SemanticAxNode>[],
|
||||
documents: normalizedDocuments,
|
||||
viewport: captured.viewport,
|
||||
rootFrameId: captured.rootFrameId,
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
|
||||
@@ -110,6 +110,28 @@ describe("tooltip name enrichment", () => {
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not collect supplemental names from form controls", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const inputDocument = document({ tabId: 8 }, 602);
|
||||
inputDocument.domNodes[0] = {
|
||||
...inputDocument.domNodes[0],
|
||||
tag: "input",
|
||||
attrs: { type: "email" },
|
||||
formValue: "user@example.com",
|
||||
formDefaultValue: "",
|
||||
formState: "filled",
|
||||
};
|
||||
|
||||
const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
|
||||
|
||||
expect(names.size).toBe(0);
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
|
||||
@@ -381,6 +381,58 @@ describe("semantic VOM graph", () => {
|
||||
expect(group?.name).toBeUndefined();
|
||||
expect(group && isVomReferenceNode(group)).toBe(false);
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
||||
|
||||
const rendered = renderVom(scene);
|
||||
const lines = rendered.text.split("\n");
|
||||
const groupLine = lines.findIndex((line) => line.trim() === "group");
|
||||
expect(groupLine).toBeGreaterThanOrEqual(0);
|
||||
for (const ref of rendered.refs) {
|
||||
expect(ref.line).toBeGreaterThan(groupLine);
|
||||
expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
|
||||
}
|
||||
});
|
||||
|
||||
it("never promotes a form value into its semantic name through supplemental evidence", () => {
|
||||
const secret = "user@example.com";
|
||||
const input = {
|
||||
...dom(2, 1, "input", { type: "email" }),
|
||||
formValue: secret,
|
||||
formDefaultValue: "",
|
||||
formState: "filled" as const,
|
||||
};
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
supplementalNames: new Map([["main\u00002", secret]]),
|
||||
identifierFallback: false,
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["email"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "textbox" },
|
||||
value: { value: secret },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), input],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const field = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||
expect(field?.name).toBeUndefined();
|
||||
const rendered = renderVom(scene, { redactValues: true });
|
||||
expect(rendered.text).toContain("•••");
|
||||
expect(JSON.stringify(rendered)).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps uncertain single-branch wrappers flat", () => {
|
||||
|
||||
@@ -10,6 +10,7 @@ const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||
const TOOLTIP_SETTLE_MS = 250;
|
||||
const CACHE_TTL_MS = 5 * 60_000;
|
||||
const MAX_CACHE_ENTRIES = 500;
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||
|
||||
interface TooltipCacheEntry {
|
||||
name: string;
|
||||
@@ -108,6 +109,7 @@ function candidates<T extends FrameOwnedAxNode>(
|
||||
for (const node of graph.nodes.values()) {
|
||||
if (
|
||||
node.vom.name ||
|
||||
FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") ||
|
||||
!node.referenceable ||
|
||||
node.backendNodeId === undefined ||
|
||||
!isVomReferenceNode({
|
||||
|
||||
@@ -399,7 +399,7 @@ function resolvedName(
|
||||
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
||||
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
||||
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
||||
!interactive || node.backendNodeId === undefined
|
||||
formControl || !interactive || node.backendNodeId === undefined
|
||||
? undefined
|
||||
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
|
||||
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
|
||||
|
||||
Reference in New Issue
Block a user