mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): bug fix
This commit is contained in:
1 parent
38fd7d3f4c
commit
d1d3eb6a94
7 files changed
+244
-6
No files matched your search
@@ -3155,6 +3155,98 @@ describe("handleSnapshot", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("decorates live observations with active controlled content", async () => {
|
||||||
|
const strings = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"button",
|
||||||
|
"div",
|
||||||
|
"#text",
|
||||||
|
"role",
|
||||||
|
"tab",
|
||||||
|
"aria-selected",
|
||||||
|
"true",
|
||||||
|
"aria-controls",
|
||||||
|
"reviews-panel",
|
||||||
|
"id",
|
||||||
|
"static",
|
||||||
|
"auto",
|
||||||
|
"visible",
|
||||||
|
"1",
|
||||||
|
"A detailed review",
|
||||||
|
];
|
||||||
|
const index = (value: string) => strings.indexOf(value);
|
||||||
|
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
|
||||||
|
const snapshot = {
|
||||||
|
strings,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1, 3],
|
||||||
|
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
index("role"),
|
||||||
|
index("tab"),
|
||||||
|
index("aria-selected"),
|
||||||
|
index("true"),
|
||||||
|
index("aria-controls"),
|
||||||
|
index("reviews-panel"),
|
||||||
|
],
|
||||||
|
[index("id"), index("reviews-panel")],
|
||||||
|
[],
|
||||||
|
],
|
||||||
|
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1, 2, 3, 4],
|
||||||
|
styles: [style, style, style, style, style],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[20, 20, 120, 40],
|
||||||
|
[20, 80, 500, 200],
|
||||||
|
[20, 80, 200, 20],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 0, 1, 1, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["reviews"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "reviews",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 12,
|
||||||
|
role: { type: "role", value: "tab" },
|
||||||
|
name: { type: "computedString", value: "Reviews" },
|
||||||
|
properties: [
|
||||||
|
{ name: "selected", value: { value: true } },
|
||||||
|
{ name: "controls", value: { value: "reviews-panel" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await captureVomObservation(
|
||||||
|
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
|
||||||
|
4,
|
||||||
|
"https://example.com",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.text).toContain('@e1 tab "Reviews"');
|
||||||
|
expect(result.text).toContain("[§ active: Reviews]");
|
||||||
|
expect(result.text).toContain("A detailed review");
|
||||||
|
});
|
||||||
|
|
||||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||||
const secrets = {
|
const secrets = {
|
||||||
email: "user@example.com",
|
email: "user@example.com",
|
||||||
|
|||||||
@@ -817,6 +817,14 @@ export function buildFrameVomScene(
|
|||||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||||
supplementalNames: options.supplementalNames,
|
supplementalNames: options.supplementalNames,
|
||||||
});
|
});
|
||||||
|
return attachCapturedSceneAnnotations(scene, documents, captured);
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachCapturedSceneAnnotations(
|
||||||
|
scene: VomScene,
|
||||||
|
documents: VomFrameDocument[],
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
): VomScene {
|
||||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||||
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
||||||
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
||||||
@@ -1055,7 +1063,8 @@ export async function captureVomObservation(
|
|||||||
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
const rendered = renderVom(scene, {
|
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||||
|
const rendered = renderVom(decoratedScene, {
|
||||||
maxDepth: options.maxDepth,
|
maxDepth: options.maxDepth,
|
||||||
maxTokens: options.maxTokens,
|
maxTokens: options.maxTokens,
|
||||||
redactValues: options.redactValues,
|
redactValues: options.redactValues,
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ function fakeSnapshotReply() {
|
|||||||
},
|
},
|
||||||
layout: {
|
layout: {
|
||||||
nodeIndex: [1, 2, 3],
|
nodeIndex: [1, 2, 3],
|
||||||
// styles columns follow requested computedStyles order: [position, pointer-events]
|
// Legacy fixture: omitted style columns fall back to visible defaults.
|
||||||
styles: [
|
styles: [
|
||||||
[i("static"), i("auto")],
|
[i("static"), i("auto")],
|
||||||
[i("fixed"), i("auto")],
|
[i("fixed"), i("auto")],
|
||||||
@@ -682,6 +682,46 @@ describe("captureViewModel", () => {
|
|||||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("captures painted visibility as DOM fallback evidence", async () => {
|
||||||
|
const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
|
||||||
|
const i = (value: string) => S.indexOf(value);
|
||||||
|
const snapshot = {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14],
|
||||||
|
attributes: [[], [], [], [], []],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3, 4],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||||
|
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||||
|
[i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
|
||||||
|
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[10, 10, 100, 30],
|
||||||
|
[10, 50, 100, 30],
|
||||||
|
[10, 90, 100, 30],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2, 3],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||||
|
|
||||||
|
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
||||||
|
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
||||||
|
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_t: number, method: string) => {
|
send: vi.fn(async (_t: number, method: string) => {
|
||||||
|
|||||||
@@ -271,6 +271,63 @@ describe("semantic VOM graph", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("requires render evidence before promoting DOM fallback semantics", () => {
|
||||||
|
const hiddenOwner = {
|
||||||
|
...dom(10, 1, "iframe", { hidden: "" }),
|
||||||
|
rect: null,
|
||||||
|
localRect: null,
|
||||||
|
rendered: false,
|
||||||
|
};
|
||||||
|
const hiddenButton = {
|
||||||
|
...dom(2, 1, "button", { "aria-label": "Hidden action" }),
|
||||||
|
rendered: false,
|
||||||
|
};
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[dom(1, null, "body"), hiddenButton, hiddenOwner],
|
||||||
|
),
|
||||||
|
document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
|
||||||
|
expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
|
||||||
|
expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
|
||||||
|
expect(renderVom(scene).refs).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
|
||||||
|
const scene = buildSemanticVomScene({
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
rootFrameId: "main",
|
||||||
|
documents: [
|
||||||
|
document(
|
||||||
|
"main",
|
||||||
|
[],
|
||||||
|
[dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(renderVom(scene).refs).toEqual([
|
||||||
|
expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("lets explicit DOM structure override ignored AX metadata", () => {
|
it("lets explicit DOM structure override ignored AX metadata", () => {
|
||||||
const scene = buildSemanticVomScene({
|
const scene = buildSemanticVomScene({
|
||||||
viewport: { width: 800, height: 600 },
|
viewport: { width: 800, height: 600 },
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||||
// shared string table); we request exactly three computed styles so the
|
// shared string table); requested computed styles are decoded through
|
||||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||||
|
|
||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||||
@@ -11,7 +11,7 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
|
|||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
import { clearHover, waitForHover } from "./hover-perception";
|
import { clearHover, waitForHover } from "./hover-perception";
|
||||||
|
|
||||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||||
const STYLE_COL = Object.fromEntries(
|
const STYLE_COL = Object.fromEntries(
|
||||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||||
@@ -39,6 +39,12 @@ export interface CapturedNode {
|
|||||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||||
*/
|
*/
|
||||||
cursor?: string;
|
cursor?: string;
|
||||||
|
/**
|
||||||
|
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||||
|
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||||
|
* nodes remain authoritative even when they are outside the viewport.
|
||||||
|
*/
|
||||||
|
rendered?: boolean;
|
||||||
textContent?: string;
|
textContent?: string;
|
||||||
formState?: "empty" | "filled" | "default";
|
formState?: "empty" | "filled" | "default";
|
||||||
formValue?: string;
|
formValue?: string;
|
||||||
@@ -788,6 +794,8 @@ function parseDocumentNodes(
|
|||||||
const posCol = STYLE_COL.position;
|
const posCol = STYLE_COL.position;
|
||||||
const peCol = STYLE_COL["pointer-events"];
|
const peCol = STYLE_COL["pointer-events"];
|
||||||
const cursorCol = STYLE_COL.cursor;
|
const cursorCol = STYLE_COL.cursor;
|
||||||
|
const visibilityCol = STYLE_COL.visibility;
|
||||||
|
const opacityCol = STYLE_COL.opacity;
|
||||||
const inputValues = sparseIndexMap(dn.inputValue);
|
const inputValues = sparseIndexMap(dn.inputValue);
|
||||||
const textValues = sparseIndexMap(dn.textValue);
|
const textValues = sparseIndexMap(dn.textValue);
|
||||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||||
@@ -832,6 +840,7 @@ function parseDocumentNodes(
|
|||||||
let position = "static";
|
let position = "static";
|
||||||
let pointerEvents = "auto";
|
let pointerEvents = "auto";
|
||||||
let cursor = "auto";
|
let cursor = "auto";
|
||||||
|
let rendered = false;
|
||||||
const li = layoutByNode.get(n);
|
const li = layoutByNode.get(n);
|
||||||
if (li !== undefined) {
|
if (li !== undefined) {
|
||||||
const b = dl?.bounds?.[li];
|
const b = dl?.bounds?.[li];
|
||||||
@@ -852,6 +861,13 @@ function parseDocumentNodes(
|
|||||||
position = str(strings, styleRow[posCol]) || "static";
|
position = str(strings, styleRow[posCol]) || "static";
|
||||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||||
|
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||||
|
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||||
|
rendered =
|
||||||
|
localRect !== null &&
|
||||||
|
visibility !== "hidden" &&
|
||||||
|
visibility !== "collapse" &&
|
||||||
|
(Number.parseFloat(opacity) || 0) > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||||
@@ -887,6 +903,7 @@ function parseDocumentNodes(
|
|||||||
position,
|
position,
|
||||||
pointerEvents,
|
pointerEvents,
|
||||||
cursor,
|
cursor,
|
||||||
|
rendered,
|
||||||
textContent,
|
textContent,
|
||||||
...(formValue !== undefined ? { formValue } : {}),
|
...(formValue !== undefined ? { formValue } : {}),
|
||||||
...(tag === "input" || tag === "textarea"
|
...(tag === "input" || tag === "textarea"
|
||||||
|
|||||||
@@ -49,6 +49,23 @@ function explicitStructural(node: ResolvedSemanticNode): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean {
|
||||||
|
return node.ax !== undefined && node.ax.ignored !== true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRenderedDomFallback(node: ResolvedSemanticNode): boolean {
|
||||||
|
if (!node.dom) return false;
|
||||||
|
const attrs = node.dom.attrs;
|
||||||
|
if (
|
||||||
|
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
||||||
|
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
||||||
|
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null);
|
||||||
|
}
|
||||||
|
|
||||||
function initialParentId(
|
function initialParentId(
|
||||||
graph: ResolvedSemanticGraph,
|
graph: ResolvedSemanticGraph,
|
||||||
node: ResolvedSemanticNode,
|
node: ResolvedSemanticNode,
|
||||||
@@ -142,6 +159,9 @@ function baseDecision(
|
|||||||
): StructureDecision {
|
): StructureDecision {
|
||||||
if (node.excluded) return { kind: "excluded", reason: "excluded" };
|
if (node.excluded) return { kind: "excluded", reason: "excluded" };
|
||||||
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
|
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
|
||||||
|
if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) {
|
||||||
|
return { kind: "excluded", reason: "non-rendered-dom-fallback" };
|
||||||
|
}
|
||||||
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
|
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
|
||||||
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
|
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
|
||||||
return { kind: "transparent", reason: "child-document-root" };
|
return { kind: "transparent", reason: "child-document-root" };
|
||||||
|
|||||||
@@ -65,7 +65,10 @@ export type StructureDecision =
|
|||||||
| { kind: "keep"; reason: "semantic" | "frame-owner" }
|
| { kind: "keep"; reason: "semantic" | "frame-owner" }
|
||||||
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
|
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
|
||||||
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
|
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
|
||||||
| { kind: "excluded"; reason: "excluded" | "redundant-source" };
|
| {
|
||||||
|
kind: "excluded";
|
||||||
|
reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback";
|
||||||
|
};
|
||||||
|
|
||||||
export interface StructuredSemanticNode extends ResolvedSemanticNode {
|
export interface StructuredSemanticNode extends ResolvedSemanticNode {
|
||||||
structure: StructureDecision;
|
structure: StructureDecision;
|
||||||
|
|||||||
Reference in new issue
Block a user