figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(observation): defer visual facts until visual discovery

Browse Source
This commit is contained in:
Ljy-0827
2026-09-09 15:54:41 +08:00
committed by drakezhang
parent 34ccb96fd6
commit bcb9beb4f8
4 changed files with 31 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts
+1 -1
View File
@@ -126,7 +126,7 @@ describe("captureObservationFacts", () => {
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
expect(main.index.children.get(1)).toEqual([2]);
expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1);
expect(main.index.nodes.get(10001)?.nodeType).toBe(10);
expect(remote.axNodes[0].frameId).toBe("remote");
expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2));
apps/extension/src/tools/vom/__tests__/facts.test.ts
+14 -60
View File
@@ -23,6 +23,7 @@ describe("document facts", () => {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
@@ -33,14 +34,13 @@ describe("document facts", () => {
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.ancestry.get(count - 1)).toEqual({ complete: true, overlay: false });
expect(index.children.get(0)?.length).toBe(wide ? count - 1 : 1);
expect(index.excludedBackendNodeIds.size).toBe(count);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("marks cycles and orphans incomplete and propagates overlay through shadow roots", async () => {
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
@@ -53,7 +53,7 @@ describe("document facts", () => {
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
for (const id of [4, 5, 6, 7]) expect(index.ancestry.get(id)?.complete).toBe(false);
expect(index.nodes.size).toBe(input.length);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
@@ -62,74 +62,28 @@ describe("document facts", () => {
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("does not treat a missing snapshot parent as a complete root", async () => {
const { nodes } = await decodeDocument(
{ nodes: { backendNodeId: [1], nodeName: [0], parentIndex: [99] } },
["div"],
);
const index = await buildDocumentIndex(nodes);
expect(index.ancestry.get(1)?.complete).toBe(false);
});
it("retains raw box units and independent style/interaction evidence", async () => {
const strings = [
"html",
"canvas",
"aria-hidden",
"true",
"inert",
"",
"hidden",
"visible",
"0",
"auto",
"scroll",
"matrix(2,0,0,2,0,0)",
];
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility"
? 7
: style === "opacity"
? 8
: style === "overflow-x"
? 10
: style === "transform"
? 11
: 9,
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
);
const { nodes } = await decodeDocument(
{
nodes: {
backendNodeId: [1, 2],
parentIndex: [-1, 0],
nodeName: [0, 1],
attributes: [[2, 3, 4, 5], []],
},
layout: {
nodeIndex: [0, 1],
bounds: [
[0, 0, 200, 100],
[10, 20, 120, 40],
],
clientRects: [
[5, 5, 90, 40],
[0, 0, 120, 40],
],
styles: [styles, styles],
},
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toMatchObject({
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-css",
clientSpace: "unscaled-client-offset-css",
clientRect: [5, 5, 90, 40],
bounds: [10, 20, 120, 40],
styles: {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "0",
"overflow-x": "scroll",
transform: "matrix(2,0,0,2,0,0)",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
apps/extension/src/tools/vom/facts.ts
+16 -43
View File
@@ -56,29 +56,18 @@ export interface CapturedSceneInput {
excludedBackendNodeIds: ReadonlySet<number>;
}
/** Snapshot client/offset rectangles retain their protocol units. In particular,
* transformed bounds cannot be combined with unscaled client offsets. */
/** Bounds retain their snapshot document CSS units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
readonly clientSpace: "unscaled-client-offset-css";
bounds?: number[];
clientRect?: number[];
offsetRect?: number[];
scrollRect?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface AncestorState {
complete: boolean;
overlay: boolean;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
@@ -86,8 +75,6 @@ export interface NodeFacts extends CapturedNode {
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly children: ReadonlyMap<number, readonly number[]>;
readonly ancestry: ReadonlyMap<number, AncestorState>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
@@ -144,67 +131,53 @@ export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves ancestry evidence even when the semantic adapter omits those nodes. */
* preserves overlay propagation when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const nodes = new Map<number, T>();
const children = new Map<number, number[]>();
const ancestry = new Map<number, AncestorState>();
const overlayByNode = new Map<number, boolean>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = input[i];
nodes.set(node.backendNodeId, node);
if (node.parentBackendNodeId !== null) {
const siblings = children.get(node.parentBackendNodeId);
if (siblings) siblings.push(node.backendNodeId);
else children.set(node.parentBackendNodeId, [node.backendNodeId]);
}
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (ancestry.has(node.backendNodeId)) continue;
if (overlayByNode.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let state: AncestorState = { complete: true, overlay: false };
while (current && !ancestry.has(current.backendNodeId)) {
let overlay = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (visiting.has(current.backendNodeId)) {
// No member of a malformed cycle supplies complete ancestry evidence.
state = {
complete: false,
overlay: path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))),
};
// Only an overlay inside the cycle may mark the cycle and its descendants.
overlay = path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentMissing) state = { complete: false, overlay: false };
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
if (!current) state = { complete: false, overlay: false };
}
if (current && ancestry.has(current.backendNodeId))
state = ancestry.get(current.backendNodeId)!;
if (current && overlayByNode.has(current.backendNodeId))
overlay = overlayByNode.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const item = path[i];
state = {
complete: state.complete,
overlay: state.overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)),
};
ancestry.set(item.backendNodeId, state);
if (state.overlay) excludedBackendNodeIds.add(item.backendNodeId);
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, children, ancestry, excludedBackendNodeIds };
return { nodes, excludedBackendNodeIds };
}
apps/extension/src/tools/vom/snapshot.ts
-17
View File
@@ -6,13 +6,6 @@ export const REQUESTED_STYLES = [
"cursor",
"visibility",
"opacity",
"display",
"overflow-x",
"overflow-y",
"transform",
"zoom",
"clip-path",
"mask-image",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
@@ -54,9 +47,6 @@ export interface SnapshotDocument {
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
clientRects?: number[][];
offsetRects?: number[][];
scrollRects?: number[][];
};
}
@@ -173,11 +163,7 @@ export async function decodeDocument(
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
clientSpace: "unscaled-client-offset-css" as const,
bounds: dl?.bounds?.[li],
clientRect: dl?.clientRects?.[li],
offsetRect: dl?.offsetRects?.[li],
scrollRect: dl?.scrollRects?.[li],
styles,
};
@@ -201,9 +187,6 @@ export async function decodeDocument(
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(parentIdx >= 0 && dn.backendNodeId[parentIdx] === undefined
? { parentMissing: true }
: {}),
tag,
attrs,