figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(vom): bug fix

Browse Source
This commit is contained in:
Ljy-0827
2026-08-26 15:28:27 +08:00
committed by GitHub
parent 38fd7d3f4c
commit d1d3eb6a94
7 changed files with 244 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/observation.test.ts
+92
View File
@@ -3155,6 +3155,98 @@ describe("handleSnapshot", () => {
});
});
it("decorates live observations with active controlled content", async () => {
const strings = [
"html",
"body",
"button",
"div",
"#text",
"role",
"tab",
"aria-selected",
"true",
"aria-controls",
"reviews-panel",
"id",
"static",
"auto",
"visible",
"1",
"A detailed review",
];
const index = (value: string) => strings.indexOf(value);
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
const snapshot = {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
backendNodeId: [10, 11, 12, 13, 14],
attributes: [
[],
[],
[
index("role"),
index("tab"),
index("aria-selected"),
index("true"),
index("aria-controls"),
index("reviews-panel"),
],
[index("id"), index("reviews-panel")],
[],
],
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
},
layout: {
nodeIndex: [0, 1, 2, 3, 4],
styles: [style, style, style, style, style],
bounds: [
[0, 0, 1000, 800],
[0, 0, 1000, 800],
[20, 20, 120, 40],
[20, 80, 500, 200],
[20, 80, 200, 20],
],
paintOrders: [0, 0, 1, 1, 1],
},
},
],
};
const ax: CdpAxNode[] = [
{
nodeId: "root",
backendDOMNodeId: 11,
role: { type: "role", value: "RootWebArea" },
childIds: ["reviews"],
},
{
nodeId: "reviews",
parentId: "root",
backendDOMNodeId: 12,
role: { type: "role", value: "tab" },
name: { type: "computedString", value: "Reviews" },
properties: [
{ name: "selected", value: { value: true } },
{ name: "controls", value: { value: "reviews-panel" } },
],
},
];
const result = await captureVomObservation(
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
4,
"https://example.com",
);
expect(result.text).toContain('@e1 tab "Reviews"');
expect(result.text).toContain("[§ active: Reviews]");
expect(result.text).toContain("A detailed review");
});
it("does not leak form secrets through the record-safe observation payload", async () => {
const secrets = {
email: "user@example.com",
apps/extension/src/tools/observation.ts
+10 -1
View File
@@ -817,6 +817,14 @@ export function buildFrameVomScene(
excludedBackendNodeIds: captured.excludedBackendNodeIds,
supplementalNames: options.supplementalNames,
});
return attachCapturedSceneAnnotations(scene, documents, captured);
}
function attachCapturedSceneAnnotations(
scene: VomScene,
documents: VomFrameDocument[],
captured: CapturedViewModel,
): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
@@ -1055,7 +1063,8 @@ export async function captureVomObservation(
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
),
);
const rendered = renderVom(scene, {
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
redactValues: options.redactValues,
apps/extension/src/tools/vom/__tests__/capture.test.ts
+41 -1
View File
@@ -36,7 +36,7 @@ function fakeSnapshotReply() {
},
layout: {
nodeIndex: [1, 2, 3],
// styles columns follow requested computedStyles order: [position, pointer-events]
// Legacy fixture: omitted style columns fall back to visible defaults.
styles: [
[i("static"), i("auto")],
[i("fixed"), i("auto")],
@@ -682,6 +682,46 @@ describe("captureViewModel", () => {
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
});
it("captures painted visibility as DOM fallback evidence", async () => {
const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
const i = (value: string) => S.indexOf(value);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
backendNodeId: [10, 11, 12, 13, 14],
attributes: [[], [], [], [], []],
},
layout: {
nodeIndex: [1, 2, 3, 4],
styles: [
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
],
bounds: [
[0, 0, 1000, 800],
[10, 10, 100, 30],
[10, 50, 100, 30],
[10, 90, 100, 30],
],
paintOrders: [0, 1, 2, 3],
},
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
});
it("subtracts scroll offset to make rects viewport-relative", async () => {
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts
+57
View File
@@ -271,6 +271,63 @@ describe("semantic VOM graph", () => {
]);
});
it("requires render evidence before promoting DOM fallback semantics", () => {
const hiddenOwner = {
...dom(10, 1, "iframe", { hidden: "" }),
rect: null,
localRect: null,
rendered: false,
};
const hiddenButton = {
...dom(2, 1, "button", { "aria-label": "Hidden action" }),
rendered: false,
};
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
},
],
[dom(1, null, "body"), hiddenButton, hiddenOwner],
),
document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
parentFrameId: "main",
ownerBackendNodeId: 10,
}),
],
});
expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
expect(renderVom(scene).refs).toEqual([]);
});
it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[],
[dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
),
],
});
expect(renderVom(scene).refs).toEqual([
expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
]);
});
it("lets explicit DOM structure override ignored AX metadata", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
apps/extension/src/tools/vom/capture.ts
+20 -3
View File
@@ -1,8 +1,8 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); we request exactly three computed styles so the
// `styles` columns are [position, pointer-events, cursor] in that order.
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
@@ -11,7 +11,7 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -39,6 +39,12 @@ export interface CapturedNode {
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
@@ -788,6 +794,8 @@ function parseDocumentNodes(
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -832,6 +840,7 @@ function parseDocumentNodes(
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
@@ -852,6 +861,13 @@ function parseDocumentNodes(
position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
@@ -887,6 +903,7 @@ function parseDocumentNodes(
position,
pointerEvents,
cursor,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
apps/extension/src/tools/vom/semantic-graph/structure.ts
+20
View File
@@ -49,6 +49,23 @@ function explicitStructural(node: ResolvedSemanticNode): boolean {
);
}
function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean {
return node.ax !== undefined && node.ax.ignored !== true;
}
function isRenderedDomFallback(node: ResolvedSemanticNode): boolean {
if (!node.dom) return false;
const attrs = node.dom.attrs;
if (
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
) {
return false;
}
return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null);
}
function initialParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
@@ -142,6 +159,9 @@ function baseDecision(
): StructureDecision {
if (node.excluded) return { kind: "excluded", reason: "excluded" };
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) {
return { kind: "excluded", reason: "non-rendered-dom-fallback" };
}
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
return { kind: "transparent", reason: "child-document-root" };
apps/extension/src/tools/vom/semantic-graph/types.ts
+4 -1
View File
@@ -65,7 +65,10 @@ export type StructureDecision =
| { kind: "keep"; reason: "semantic" | "frame-owner" }
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
| { kind: "excluded"; reason: "excluded" | "redundant-source" };
| {
kind: "excluded";
reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback";
};
export interface StructuredSemanticNode extends ResolvedSemanticNode {
structure: StructureDecision;