mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): safety policy and rebase bug fix
This commit is contained in:
@@ -54,12 +54,11 @@ import {
|
|||||||
collectOverlayExcludedBackendIds,
|
collectOverlayExcludedBackendIds,
|
||||||
} from "./vom/capture";
|
} from "./vom/capture";
|
||||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||||
|
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||||
import {
|
import {
|
||||||
type CaptureVomObservationResult,
|
type CaptureVomObservationResult,
|
||||||
projectRecordSafeObservation,
|
projectRecordSafeObservation,
|
||||||
} from "./vom/record-safe-observation";
|
} from "./vom/record-safe-observation";
|
||||||
import type { FrameDocument } from "./vom/frame-document";
|
|
||||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
|
||||||
import {
|
import {
|
||||||
buildSemanticGraph,
|
buildSemanticGraph,
|
||||||
buildSemanticVomScene,
|
buildSemanticVomScene,
|
||||||
@@ -812,7 +811,7 @@ export function buildFrameVomScene(
|
|||||||
options: BuildVomSceneOptions = {},
|
options: BuildVomSceneOptions = {},
|
||||||
): VomScene {
|
): VomScene {
|
||||||
const scene = buildSemanticVomScene({
|
const scene = buildSemanticVomScene({
|
||||||
documents: documents as FrameDocument<SemanticAxNode>[],
|
documents,
|
||||||
viewport: captured.viewport,
|
viewport: captured.viewport,
|
||||||
rootFrameId: captured.rootFrameId,
|
rootFrameId: captured.rootFrameId,
|
||||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||||
@@ -1038,7 +1037,7 @@ export async function captureVomObservation(
|
|||||||
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
|
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
|
||||||
: documents;
|
: documents;
|
||||||
const semanticGraph = buildSemanticGraph({
|
const semanticGraph = buildSemanticGraph({
|
||||||
documents: normalizedDocuments as FrameDocument<SemanticAxNode>[],
|
documents: normalizedDocuments,
|
||||||
viewport: captured.viewport,
|
viewport: captured.viewport,
|
||||||
rootFrameId: captured.rootFrameId,
|
rootFrameId: captured.rootFrameId,
|
||||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||||
|
|||||||
@@ -110,6 +110,28 @@ describe("tooltip name enrichment", () => {
|
|||||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not collect supplemental names from form controls", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
const inputDocument = document({ tabId: 8 }, 602);
|
||||||
|
inputDocument.domNodes[0] = {
|
||||||
|
...inputDocument.domNodes[0],
|
||||||
|
tag: "input",
|
||||||
|
attrs: { type: "email" },
|
||||||
|
formValue: "user@example.com",
|
||||||
|
formDefaultValue: "",
|
||||||
|
formState: "filled",
|
||||||
|
};
|
||||||
|
|
||||||
|
const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
|
||||||
|
|
||||||
|
expect(names.size).toBe(0);
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||||
let hoveredX = -10;
|
let hoveredX = -10;
|
||||||
const cdp: CdpRunner = {
|
const cdp: CdpRunner = {
|
||||||
|
|||||||
@@ -381,6 +381,58 @@ describe("semantic VOM graph", () => {
|
|||||||
expect(group?.name).toBeUndefined();
|
expect(group?.name).toBeUndefined();
|
||||||
expect(group && isVomReferenceNode(group)).toBe(false);
|
expect(group && isVomReferenceNode(group)).toBe(false);
|
||||||
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
||||||
|
|
||||||
|
const rendered = renderVom(scene);
|
||||||
|
const lines = rendered.text.split("\n");
|
||||||
|
const groupLine = lines.findIndex((line) => line.trim() === "group");
|
||||||
|
expect(groupLine).toBeGreaterThanOrEqual(0);
|
||||||
|
for (const ref of rendered.refs) {
|
||||||
|
expect(ref.line).toBeGreaterThan(groupLine);
|
||||||
|
expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never promotes a form value into its semantic name through supplemental evidence", () => {
|
||||||
|
const secret = "user@example.com";
|
||||||
|
const input = {
|
||||||
|
...dom(2, 1, "input", { type: "email" }),
|
||||||
|
formValue: secret,
|
||||||
|
formDefaultValue: "",
|
||||||
|
formState: "filled" as const,
|
||||||
|
};
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
supplementalNames: new Map([["main\u00002", secret]]),
|
||||||
|
identifierFallback: false,
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["email"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "email",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
value: { value: secret },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), input],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const field = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||||
|
expect(field?.name).toBeUndefined();
|
||||||
|
const rendered = renderVom(scene, { redactValues: true });
|
||||||
|
expect(rendered.text).toContain("•••");
|
||||||
|
expect(JSON.stringify(rendered)).not.toContain(secret);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps uncertain single-branch wrappers flat", () => {
|
it("keeps uncertain single-branch wrappers flat", () => {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ const MAX_TOOLTIP_PROBE_MS = 2_400;
|
|||||||
const TOOLTIP_SETTLE_MS = 250;
|
const TOOLTIP_SETTLE_MS = 250;
|
||||||
const CACHE_TTL_MS = 5 * 60_000;
|
const CACHE_TTL_MS = 5 * 60_000;
|
||||||
const MAX_CACHE_ENTRIES = 500;
|
const MAX_CACHE_ENTRIES = 500;
|
||||||
|
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||||
|
|
||||||
interface TooltipCacheEntry {
|
interface TooltipCacheEntry {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -108,6 +109,7 @@ function candidates<T extends FrameOwnedAxNode>(
|
|||||||
for (const node of graph.nodes.values()) {
|
for (const node of graph.nodes.values()) {
|
||||||
if (
|
if (
|
||||||
node.vom.name ||
|
node.vom.name ||
|
||||||
|
FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") ||
|
||||||
!node.referenceable ||
|
!node.referenceable ||
|
||||||
node.backendNodeId === undefined ||
|
node.backendNodeId === undefined ||
|
||||||
!isVomReferenceNode({
|
!isVomReferenceNode({
|
||||||
|
|||||||
@@ -399,7 +399,7 @@ function resolvedName(
|
|||||||
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
||||||
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
||||||
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
||||||
!interactive || node.backendNodeId === undefined
|
formControl || !interactive || node.backendNodeId === undefined
|
||||||
? undefined
|
? undefined
|
||||||
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
|
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
|
||||||
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
|
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
|
||||||
|
|||||||
Reference in New Issue
Block a user