From 7469d3506a32e318b3db8a37aaeabbefbc39e89d Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 8 Sep 2026 15:16:10 +0800 Subject: [PATCH] fix(vom): add new texts and fix bugs --- .../src/tools/__tests__/observation.test.ts | 144 ++++++++++++++++++ .../src/tools/vom/__tests__/capture.test.ts | 21 ++- .../tools/vom/rendered-surfaces/cluster.ts | 49 +----- .../__tests__/visual-surface-priority.test.ts | 35 +++++ packages/vom/src/index.ts | 1 + packages/vom/src/render.ts | 21 +-- packages/vom/src/visual-surface-priority.ts | 51 +++++++ 7 files changed, 258 insertions(+), 64 deletions(-) create mode 100644 packages/vom/src/__tests__/visual-surface-priority.test.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 352431f..904ff79 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -3179,6 +3179,150 @@ describe("handleSnapshot", () => { expect(snapshot.ref_count).toBe(0); }); + it("preserves an overflow-clipped surface region through observation and screenshot", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + const root: CdpAxNode = { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "Clipped canvas" }, + backendDOMNodeId: 100, + }; + const strings = [ + "body", + "div", + "canvas", + "aria-label", + "clipped canvas", + "static", + "auto", + "visible", + "hidden", + "1", + ]; + const i = (value: string) => strings.indexOf(value); + const send = vi.fn(async (_tabId: number, method: string, _params?: object) => { + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: [root] }; + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + layoutViewport: { clientWidth: 1000, clientHeight: 800 }, + }; + } + if (method === "DOMSnapshot.captureSnapshot") { + return { + strings, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1], + nodeName: [i("body"), i("div"), i("canvas")], + backendNodeId: [100, 150, 200], + attributes: [[], [], [i("aria-label"), i("clipped canvas")]], + }, + layout: { + nodeIndex: [0, 1, 2], + styles: [ + [ + i("static"), + i("auto"), + i("auto"), + i("visible"), + i("1"), + i("visible"), + i("visible"), + ], + [ + i("static"), + i("auto"), + i("auto"), + i("visible"), + i("1"), + i("hidden"), + i("hidden"), + ], + [ + i("static"), + i("auto"), + i("auto"), + i("visible"), + i("1"), + i("visible"), + i("visible"), + ], + ], + bounds: [ + [0, 0, 1000, 800], + [10, 20, 100, 70], + [55, 30, 140, 60], + ], + paintOrders: [0, 1, 2], + }, + }, + ], + }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[55, 30, 195, 30, 195, 90, 55, 90]] }; + } + if (method === "Runtime.evaluate") return { result: { value: 1 } }; + if (method === "Page.captureScreenshot") { + return { data: pngWithIhdrDimensions(55, 60) }; + } + throw new Error(`unexpected CDP method ${method}`); + }); + const cdp = { + send: send as unknown as CdpRunner["send"], + trackSessionTab: vi.fn(), + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "root", + frames: [{ frameId: "root", target: { tabId: 4 } }], + })), + } as CdpRunner; + const deps = { + cdp, + tabsApi: { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + conditionalSurfaceProbe: false, + }; + + const observed = await handleObserve(sm, { session_id: "aa11" }, deps); + if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`); + expect(observed.text).toContain( + '@e1 surface "clipped canvas" [bounds=55,30,55,60; rendering=canvas;', + ); + expect(ctx.refStore.resolveEntry("e1")).toMatchObject({ + backendNodeId: 200, + kind: "surface", + visibleRect: { x: 55, y: 30, w: 55, h: 60 }, + }); + + const screenshot = await handleScreenshot( + sm, + { session_id: "aa11", ref: "@e1", tab_id: 4 }, + makeScreenshotDeps({ + cdp, + get: vi.fn(async () => ({ id: 4, windowId: 100, active: true }) as chrome.tabs.Tab), + }), + ); + if ("code" in screenshot) { + throw new Error(`unexpected error: ${JSON.stringify(screenshot)}`); + } + const captureCall = send.mock.calls.find(([, method]) => method === "Page.captureScreenshot"); + expect((captureCall?.[2] as { clip?: object }).clip).toMatchObject({ + x: 55, + y: 30, + width: 55, + height: 60, + }); + expect(screenshot).toMatchObject({ width: 55, height: 60 }); + }); + it("allows passive snapshots of explicit user-window tabs", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index c47898d..738fccc 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -793,8 +793,10 @@ describe("captureViewModel", () => { expect(div?.rect).toMatchObject({ y: 0, h: 600 }); }); - it("normalizes device-pixel bounds by devicePixelRatio", async () => { - const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; + it.each([ + 1, 2, + ])("keeps CSS-space bounds stable at devicePixelRatio=%i", async (deviceScaleFactor) => { + const S = ["html", "body", "canvas", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { strings: S, @@ -802,7 +804,7 @@ describe("captureViewModel", () => { { nodes: { parentIndex: [-1, 0, 1], - nodeName: [i("html"), i("body"), i("div")], + nodeName: [i("html"), i("body"), i("canvas")], backendNodeId: [10, 11, 12], attributes: [[], [], []], }, @@ -813,8 +815,8 @@ describe("captureViewModel", () => { [i("fixed"), i("auto")], ], bounds: [ - [0, 0, 2000, 3200], - [0, 0, 2000, 1600], + [0, 0, 1000 * deviceScaleFactor, 1600 * deviceScaleFactor], + [0, 0, 1000 * deviceScaleFactor, 800 * deviceScaleFactor], ], paintOrders: [0, 50], }, @@ -828,14 +830,19 @@ describe("captureViewModel", () => { if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, - layoutViewport: { clientWidth: 2000, clientHeight: 1600 }, + layoutViewport: { + clientWidth: 1000 * deviceScaleFactor, + clientHeight: 800 * deviceScaleFactor, + }, }; } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; const { nodes } = await captureViewModel(cdp, 4); - expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ + const canvas = nodes.find((node) => node.backendNodeId === 12); + expect(canvas?.tag).toBe("canvas"); + expect(canvas?.rect).toEqual({ x: 0, y: 0, w: 1000, diff --git a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts index 8b2ecb8..06e009b 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts @@ -1,4 +1,4 @@ -import { compareVisualSurfacePriority, type Rect } from "@browser-skill/vom"; +import { type Rect, selectHighestPriorityVisualSurfaces } from "@browser-skill/vom"; import type { ClusteredRenderedSurfaces, RenderedSurface, RenderedSurfaceGroup } from "./types"; const STACK_IOU_RATIO = 0.9; @@ -97,51 +97,6 @@ function partitionKey(surface: RenderedSurface): string { return `${surface.frameId}\u0000${surface.parentBackendNodeId ?? "root"}`; } -function siftWorstUp(heap: RenderedSurface[], start: number): void { - let index = start; - while (index > 0) { - const parent = Math.floor((index - 1) / 2); - if (compareVisualSurfacePriority(heap[index], heap[parent]) <= 0) break; - [heap[index], heap[parent]] = [heap[parent], heap[index]]; - index = parent; - } -} - -function siftWorstDown(heap: RenderedSurface[], start: number): void { - let index = start; - while (true) { - const left = index * 2 + 1; - if (left >= heap.length) return; - const right = left + 1; - let worst = left; - if (right < heap.length && compareVisualSurfacePriority(heap[right], heap[left]) > 0) { - worst = right; - } - if (compareVisualSurfacePriority(heap[worst], heap[index]) <= 0) return; - [heap[index], heap[worst]] = [heap[worst], heap[index]]; - index = worst; - } -} - -function highestPrioritySurfaces( - surfaces: readonly RenderedSurface[], - limit: number, -): RenderedSurface[] { - if (limit <= 0) return []; - if (surfaces.length <= limit) return [...surfaces].sort(compareVisualSurfacePriority); - const heap: RenderedSurface[] = []; - for (const surface of surfaces) { - if (heap.length < limit) { - heap.push(surface); - siftWorstUp(heap, heap.length - 1); - } else if (compareVisualSurfacePriority(surface, heap[0]) < 0) { - heap[0] = surface; - siftWorstDown(heap, 0); - } - } - return heap.sort(compareVisualSurfacePriority); -} - function normalizedLimit(value: number | undefined, fallback: number): number { const resolved = value ?? fallback; return Number.isFinite(resolved) ? Math.max(0, Math.floor(resolved)) : fallback; @@ -153,7 +108,7 @@ export function clusterRenderedSurfaces( ): ClusteredRenderedSurfaces { const maxComparisons = normalizedLimit(options.maxComparisons, MAX_CLUSTER_COMPARISONS); const maxSurfaces = normalizedLimit(options.maxSurfaces, MAX_CLUSTER_SURFACES); - const prioritized = highestPrioritySurfaces(surfaces, maxSurfaces); + const prioritized = selectHighestPriorityVisualSurfaces(surfaces, maxSurfaces); const partitions = new Map>>(); const groups: IndexedGroup[] = []; let comparisons = 0; diff --git a/packages/vom/src/__tests__/visual-surface-priority.test.ts b/packages/vom/src/__tests__/visual-surface-priority.test.ts new file mode 100644 index 0000000..42bb19b --- /dev/null +++ b/packages/vom/src/__tests__/visual-surface-priority.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { + compareVisualSurfacePriority, + selectHighestPriorityVisualSurfaces, +} from "../visual-surface-priority"; + +function surface(backendNodeId: number, area: number, label?: string) { + return { + backendNodeId, + frameId: "main", + visibleRect: { x: backendNodeId, y: 0, w: area, h: 1 }, + ...(label ? { label } : {}), + }; +} + +describe("visual surface priority selection", () => { + it("matches a full priority sort when selecting a bounded prefix", () => { + const surfaces = Array.from({ length: 600 }, (_, index) => surface(index, index + 1)); + surfaces[599] = surface(599, 1, "Important status"); + + const selected = selectHighestPriorityVisualSurfaces(surfaces, 512); + const expected = [...surfaces].sort(compareVisualSurfacePriority).slice(0, 512); + + expect(selected).toEqual(expected); + expect(selected[0]?.backendNodeId).toBe(599); + }); + + it("normalizes empty, fractional, and unbounded limits", () => { + const surfaces = [surface(1, 1), surface(2, 2), surface(3, 3)]; + + expect(selectHighestPriorityVisualSurfaces(surfaces, 0)).toEqual([]); + expect(selectHighestPriorityVisualSurfaces(surfaces, 1.9)).toEqual([surface(3, 3)]); + expect(selectHighestPriorityVisualSurfaces(surfaces, Number.POSITIVE_INFINITY)).toHaveLength(3); + }); +}); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index 1a10f70..b277d66 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -22,5 +22,6 @@ export type { } from "./types"; export { compareVisualSurfacePriority, + selectHighestPriorityVisualSurfaces, type VisualSurfacePriorityInput, } from "./visual-surface-priority"; diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 8260159..86bdfa9 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -11,7 +11,7 @@ import type { VomScene, VomVisualSurface, } from "./types"; -import { compareVisualSurfacePriority } from "./visual-surface-priority"; +import { selectHighestPriorityVisualSurfaces } from "./visual-surface-priority"; const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]); @@ -822,17 +822,18 @@ function selectVisualSurfaces( surfaces: readonly VomVisualSurface[], nodesById: ReadonlyMap, ): { selected: VomVisualSurface[]; eligibleCount: number } { - const selected: VomVisualSurface[] = []; let eligibleCount = 0; - - for (const surface of surfaces) { - if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue; - eligibleCount += 1; - - selected.push(surface); + function* eligibleSurfaces(): Generator { + for (const surface of surfaces) { + if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue; + eligibleCount += 1; + yield surface; + } } - selected.sort(compareVisualSurfacePriority); - return { selected, eligibleCount }; + return { + selected: selectHighestPriorityVisualSurfaces(eligibleSurfaces(), MAX_VISUAL_SURFACE_REFS), + eligibleCount, + }; } function emitVisualAppendix(state: RenderState): void { diff --git a/packages/vom/src/visual-surface-priority.ts b/packages/vom/src/visual-surface-priority.ts index 97c2229..9bfdb43 100644 --- a/packages/vom/src/visual-surface-priority.ts +++ b/packages/vom/src/visual-surface-priority.ts @@ -27,3 +27,54 @@ export function compareVisualSurfacePriority( a.backendNodeId - b.backendNodeId ); } + +function siftWorstUp(heap: T[], start: number): void { + let index = start; + while (index > 0) { + const parent = Math.floor((index - 1) / 2); + if (compareVisualSurfacePriority(heap[index], heap[parent]) <= 0) break; + [heap[index], heap[parent]] = [heap[parent], heap[index]]; + index = parent; + } +} + +function siftWorstDown(heap: T[], start: number): void { + let index = start; + while (true) { + const left = index * 2 + 1; + if (left >= heap.length) return; + const right = left + 1; + let worst = left; + if (right < heap.length && compareVisualSurfacePriority(heap[right], heap[left]) > 0) { + worst = right; + } + if (compareVisualSurfacePriority(heap[worst], heap[index]) <= 0) return; + [heap[index], heap[worst]] = [heap[worst], heap[index]]; + index = worst; + } +} + +/** Selects the best `limit` surfaces and returns them in priority order. */ +export function selectHighestPriorityVisualSurfaces( + surfaces: Iterable, + limit: number, +): T[] { + if (!Number.isFinite(limit)) return [...surfaces].sort(compareVisualSurfacePriority); + const normalizedLimit = Math.max(0, Math.floor(limit)); + if (normalizedLimit === 0) return []; + if (Array.isArray(surfaces) && surfaces.length <= normalizedLimit) { + return [...surfaces].sort(compareVisualSurfacePriority); + } + + const heap: T[] = []; + for (const surface of surfaces) { + if (heap.length < normalizedLimit) { + heap.push(surface); + siftWorstUp(heap, heap.length - 1); + } else if (compareVisualSurfacePriority(surface, heap[0]) < 0) { + heap[0] = surface; + siftWorstDown(heap, 0); + } + } + return heap.sort(compareVisualSurfacePriority); +}