figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(vom): add new texts and fix bugs

Browse Source
This commit is contained in:
Ljy-0827
2026-09-08 15:16:10 +08:00
parent f528f95bd6
commit 7469d3506a
7 changed files with 258 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/observation.test.ts
+144
View File
@@ -3179,6 +3179,150 @@ describe("handleSnapshot", () => {
expect(snapshot.ref_count).toBe(0);
});
it("preserves an overflow-clipped surface region through observation and screenshot", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Clipped canvas" },
backendDOMNodeId: 100,
};
const strings = [
"body",
"div",
"canvas",
"aria-label",
"clipped canvas",
"static",
"auto",
"visible",
"hidden",
"1",
];
const i = (value: string) => strings.indexOf(value);
const send = vi.fn(async (_tabId: number, method: string, _params?: object) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("body"), i("div"), i("canvas")],
backendNodeId: [100, 150, 200],
attributes: [[], [], [i("aria-label"), i("clipped canvas")]],
},
layout: {
nodeIndex: [0, 1, 2],
styles: [
[
i("static"),
i("auto"),
i("auto"),
i("visible"),
i("1"),
i("visible"),
i("visible"),
],
[
i("static"),
i("auto"),
i("auto"),
i("visible"),
i("1"),
i("hidden"),
i("hidden"),
],
[
i("static"),
i("auto"),
i("auto"),
i("visible"),
i("1"),
i("visible"),
i("visible"),
],
],
bounds: [
[0, 0, 1000, 800],
[10, 20, 100, 70],
[55, 30, 140, 60],
],
paintOrders: [0, 1, 2],
},
},
],
};
}
if (method === "DOM.getContentQuads") {
return { quads: [[55, 30, 195, 30, 195, 90, 55, 90]] };
}
if (method === "Runtime.evaluate") return { result: { value: 1 } };
if (method === "Page.captureScreenshot") {
return { data: pngWithIhdrDimensions(55, 60) };
}
throw new Error(`unexpected CDP method ${method}`);
});
const cdp = {
send: send as unknown as CdpRunner["send"],
trackSessionTab: vi.fn(),
getFrameGraph: vi.fn(async () => ({
rootFrameId: "root",
frames: [{ frameId: "root", target: { tabId: 4 } }],
})),
} as CdpRunner;
const deps = {
cdp,
tabsApi: {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
conditionalSurfaceProbe: false,
};
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
expect(observed.text).toContain(
'@e1 surface "clipped canvas" [bounds=55,30,55,60; rendering=canvas;',
);
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
backendNodeId: 200,
kind: "surface",
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
});
const screenshot = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e1", tab_id: 4 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 4, windowId: 100, active: true }) as chrome.tabs.Tab),
}),
);
if ("code" in screenshot) {
throw new Error(`unexpected error: ${JSON.stringify(screenshot)}`);
}
const captureCall = send.mock.calls.find(([, method]) => method === "Page.captureScreenshot");
expect((captureCall?.[2] as { clip?: object }).clip).toMatchObject({
x: 55,
y: 30,
width: 55,
height: 60,
});
expect(screenshot).toMatchObject({ width: 55, height: 60 });
});
it("allows passive snapshots of explicit user-window tabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
apps/extension/src/tools/vom/__tests__/capture.test.ts
+14 -7
View File
@@ -793,8 +793,10 @@ describe("captureViewModel", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
it.each([
1, 2,
])("keeps CSS-space bounds stable at devicePixelRatio=%i", async (deviceScaleFactor) => {
const S = ["html", "body", "canvas", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
strings: S,
@@ -802,7 +804,7 @@ describe("captureViewModel", () => {
{
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
nodeName: [i("html"), i("body"), i("canvas")],
backendNodeId: [10, 11, 12],
attributes: [[], [], []],
},
@@ -813,8 +815,8 @@ describe("captureViewModel", () => {
[i("fixed"), i("auto")],
],
bounds: [
[0, 0, 2000, 3200],
[0, 0, 2000, 1600],
[0, 0, 1000 * deviceScaleFactor, 1600 * deviceScaleFactor],
[0, 0, 1000 * deviceScaleFactor, 800 * deviceScaleFactor],
],
paintOrders: [0, 50],
},
@@ -828,14 +830,19 @@ describe("captureViewModel", () => {
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
layoutViewport: {
clientWidth: 1000 * deviceScaleFactor,
clientHeight: 800 * deviceScaleFactor,
},
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
const canvas = nodes.find((node) => node.backendNodeId === 12);
expect(canvas?.tag).toBe("canvas");
expect(canvas?.rect).toEqual({
x: 0,
y: 0,
w: 1000,
apps/extension/src/tools/vom/rendered-surfaces/cluster.ts
+2 -47
View File
@@ -1,4 +1,4 @@
import { compareVisualSurfacePriority, type Rect } from "@browser-skill/vom";
import { type Rect, selectHighestPriorityVisualSurfaces } from "@browser-skill/vom";
import type { ClusteredRenderedSurfaces, RenderedSurface, RenderedSurfaceGroup } from "./types";
const STACK_IOU_RATIO = 0.9;
@@ -97,51 +97,6 @@ function partitionKey(surface: RenderedSurface): string {
return `${surface.frameId}\u0000${surface.parentBackendNodeId ?? "root"}`;
}
function siftWorstUp(heap: RenderedSurface[], start: number): void {
let index = start;
while (index > 0) {
const parent = Math.floor((index - 1) / 2);
if (compareVisualSurfacePriority(heap[index], heap[parent]) <= 0) break;
[heap[index], heap[parent]] = [heap[parent], heap[index]];
index = parent;
}
}
function siftWorstDown(heap: RenderedSurface[], start: number): void {
let index = start;
while (true) {
const left = index * 2 + 1;
if (left >= heap.length) return;
const right = left + 1;
let worst = left;
if (right < heap.length && compareVisualSurfacePriority(heap[right], heap[left]) > 0) {
worst = right;
}
if (compareVisualSurfacePriority(heap[worst], heap[index]) <= 0) return;
[heap[index], heap[worst]] = [heap[worst], heap[index]];
index = worst;
}
}
function highestPrioritySurfaces(
surfaces: readonly RenderedSurface[],
limit: number,
): RenderedSurface[] {
if (limit <= 0) return [];
if (surfaces.length <= limit) return [...surfaces].sort(compareVisualSurfacePriority);
const heap: RenderedSurface[] = [];
for (const surface of surfaces) {
if (heap.length < limit) {
heap.push(surface);
siftWorstUp(heap, heap.length - 1);
} else if (compareVisualSurfacePriority(surface, heap[0]) < 0) {
heap[0] = surface;
siftWorstDown(heap, 0);
}
}
return heap.sort(compareVisualSurfacePriority);
}
function normalizedLimit(value: number | undefined, fallback: number): number {
const resolved = value ?? fallback;
return Number.isFinite(resolved) ? Math.max(0, Math.floor(resolved)) : fallback;
@@ -153,7 +108,7 @@ export function clusterRenderedSurfaces(
): ClusteredRenderedSurfaces {
const maxComparisons = normalizedLimit(options.maxComparisons, MAX_CLUSTER_COMPARISONS);
const maxSurfaces = normalizedLimit(options.maxSurfaces, MAX_CLUSTER_SURFACES);
const prioritized = highestPrioritySurfaces(surfaces, maxSurfaces);
const prioritized = selectHighestPriorityVisualSurfaces(surfaces, maxSurfaces);
const partitions = new Map<string, Map<string, Set<IndexedGroup>>>();
const groups: IndexedGroup[] = [];
let comparisons = 0;
packages/vom/src/__tests__/visual-surface-priority.test.ts
+35
View File
@@ -0,0 +1,35 @@
import { describe, expect, it } from "vitest";
import {
compareVisualSurfacePriority,
selectHighestPriorityVisualSurfaces,
} from "../visual-surface-priority";
function surface(backendNodeId: number, area: number, label?: string) {
return {
backendNodeId,
frameId: "main",
visibleRect: { x: backendNodeId, y: 0, w: area, h: 1 },
...(label ? { label } : {}),
};
}
describe("visual surface priority selection", () => {
it("matches a full priority sort when selecting a bounded prefix", () => {
const surfaces = Array.from({ length: 600 }, (_, index) => surface(index, index + 1));
surfaces[599] = surface(599, 1, "Important status");
const selected = selectHighestPriorityVisualSurfaces(surfaces, 512);
const expected = [...surfaces].sort(compareVisualSurfacePriority).slice(0, 512);
expect(selected).toEqual(expected);
expect(selected[0]?.backendNodeId).toBe(599);
});
it("normalizes empty, fractional, and unbounded limits", () => {
const surfaces = [surface(1, 1), surface(2, 2), surface(3, 3)];
expect(selectHighestPriorityVisualSurfaces(surfaces, 0)).toEqual([]);
expect(selectHighestPriorityVisualSurfaces(surfaces, 1.9)).toEqual([surface(3, 3)]);
expect(selectHighestPriorityVisualSurfaces(surfaces, Number.POSITIVE_INFINITY)).toHaveLength(3);
});
});
packages/vom/src/index.ts
+1
View File
@@ -22,5 +22,6 @@ export type {
} from "./types";
export {
compareVisualSurfacePriority,
selectHighestPriorityVisualSurfaces,
type VisualSurfacePriorityInput,
} from "./visual-surface-priority";
packages/vom/src/render.ts
+11 -10
View File
@@ -11,7 +11,7 @@ import type {
VomScene,
VomVisualSurface,
} from "./types";
import { compareVisualSurfacePriority } from "./visual-surface-priority";
import { selectHighestPriorityVisualSurfaces } from "./visual-surface-priority";
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -822,17 +822,18 @@ function selectVisualSurfaces(
surfaces: readonly VomVisualSurface[],
nodesById: ReadonlyMap<number, VomNode>,
): { selected: VomVisualSurface[]; eligibleCount: number } {
const selected: VomVisualSurface[] = [];
let eligibleCount = 0;
for (const surface of surfaces) {
if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
eligibleCount += 1;
selected.push(surface);
function* eligibleSurfaces(): Generator<VomVisualSurface> {
for (const surface of surfaces) {
if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
eligibleCount += 1;
yield surface;
}
}
selected.sort(compareVisualSurfacePriority);
return { selected, eligibleCount };
return {
selected: selectHighestPriorityVisualSurfaces(eligibleSurfaces(), MAX_VISUAL_SURFACE_REFS),
eligibleCount,
};
}
function emitVisualAppendix(state: RenderState): void {
packages/vom/src/visual-surface-priority.ts
+51
View File
@@ -27,3 +27,54 @@ export function compareVisualSurfacePriority(
a.backendNodeId - b.backendNodeId
);
}
function siftWorstUp<T extends VisualSurfacePriorityInput>(heap: T[], start: number): void {
let index = start;
while (index > 0) {
const parent = Math.floor((index - 1) / 2);
if (compareVisualSurfacePriority(heap[index], heap[parent]) <= 0) break;
[heap[index], heap[parent]] = [heap[parent], heap[index]];
index = parent;
}
}
function siftWorstDown<T extends VisualSurfacePriorityInput>(heap: T[], start: number): void {
let index = start;
while (true) {
const left = index * 2 + 1;
if (left >= heap.length) return;
const right = left + 1;
let worst = left;
if (right < heap.length && compareVisualSurfacePriority(heap[right], heap[left]) > 0) {
worst = right;
}
if (compareVisualSurfacePriority(heap[worst], heap[index]) <= 0) return;
[heap[index], heap[worst]] = [heap[worst], heap[index]];
index = worst;
}
}
/** Selects the best `limit` surfaces and returns them in priority order. */
export function selectHighestPriorityVisualSurfaces<T extends VisualSurfacePriorityInput>(
surfaces: Iterable<T>,
limit: number,
): T[] {
if (!Number.isFinite(limit)) return [...surfaces].sort(compareVisualSurfacePriority);
const normalizedLimit = Math.max(0, Math.floor(limit));
if (normalizedLimit === 0) return [];
if (Array.isArray(surfaces) && surfaces.length <= normalizedLimit) {
return [...surfaces].sort(compareVisualSurfacePriority);
}
const heap: T[] = [];
for (const surface of surfaces) {
if (heap.length < normalizedLimit) {
heap.push(surface);
siftWorstUp(heap, heap.length - 1);
} else if (compareVisualSurfacePriority(surface, heap[0]) < 0) {
heap[0] = surface;
siftWorstDown(heap, 0);
}
}
return heap.sort(compareVisualSurfacePriority);
}