diff --git a/apps/extension/src/session-manager/__tests__/ref-store.test.ts b/apps/extension/src/session-manager/__tests__/ref-store.test.ts index 9cb540b..dfccd38 100644 --- a/apps/extension/src/session-manager/__tests__/ref-store.test.ts +++ b/apps/extension/src/session-manager/__tests__/ref-store.test.ts @@ -64,4 +64,17 @@ describe("RefStore", () => { capabilities: ["screenshot"], }); }); + + it("never grants ordinary interaction to a surface through defaults or caller input", () => { + const s = new RefStore(); + s.set("e1", 42, { tabId: 7, kind: "surface" }); + s.set("e2", 43, { + tabId: 7, + kind: "surface", + capabilities: ["interact", "screenshot"], + }); + + expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]); + expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]); + }); }); diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index 5f68c8b..661e997 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -36,6 +36,14 @@ export type RefInput = capabilities?: RefCapability[]; }; +function capabilitiesFor( + kind: RefTargetKind, + capabilities: RefCapability[] | undefined, +): RefCapability[] { + if (kind === "surface") return ["screenshot"]; + return capabilities ?? ["interact", "screenshot"]; +} + export class RefStore { private readonly map = new Map(); private generation = 0; @@ -80,13 +88,14 @@ export class RefStore { capabilities?: RefCapability[]; } = {}, ): void { + const kind = opts.kind ?? "dom"; this.map.set(normaliseRef(ref), { backendNodeId: id, tabId: opts.tabId ?? null, ...(opts.frameId ? { frameId: opts.frameId } : {}), ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}), - kind: opts.kind ?? "dom", - capabilities: opts.capabilities ?? ["interact", "screenshot"], + kind, + capabilities: capabilitiesFor(kind, opts.capabilities), generation: this.generation, }); } @@ -109,13 +118,14 @@ export class RefStore { generation: this.generation, }; } + const kind = input.kind ?? "dom"; return { backendNodeId: input.backendNodeId, tabId: input.tabId, ...(input.frameId ? { frameId: input.frameId } : {}), ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}), - kind: input.kind ?? "dom", - capabilities: input.capabilities ?? ["interact", "screenshot"], + kind, + capabilities: capabilitiesFor(kind, input.capabilities), generation: this.generation, }; } diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index aeee5d5..3420541 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1087,7 +1087,13 @@ export async function captureVomObservation( ? projectRenderedSurfaces( decoratedScene, normalizedDocuments, - clusterRenderedSurfaces(discoverRenderedSurfaces(normalizedDocuments, captured.viewport)), + clusterRenderedSurfaces( + discoverRenderedSurfaces( + normalizedDocuments, + captured.viewport, + captured.excludedBackendNodeIds, + ), + ), ) : decoratedScene; const rendered = renderVom(observedScene, { diff --git a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts index 40fddd1..05861d0 100644 --- a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts +++ b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts @@ -57,18 +57,18 @@ describe("rendered surface discovery", () => { ); expect(surfaces).toHaveLength(1); - expect(surfaces[0]).toMatchObject({ backendNodeId: 2, existenceConfidence: "high" }); + expect(surfaces[0]).toMatchObject({ backendNodeId: 2 }); }); - it("keeps tiny canvases only as low-confidence discoveries", () => { + it("keeps tiny canvases as individually addressable exact discoveries", () => { const surfaces = discoverRenderedSurfaces( [document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])], { width: 1280, height: 720 }, ); - expect(surfaces[0]?.existenceConfidence).toBe("low"); + expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]); }); - it("suppresses a canvas with a verified native accessibility mirror", () => { + it("keeps a surface when the canvas only has partial native accessibility content", () => { const axNodes = [ { nodeId: "canvas", @@ -86,6 +86,19 @@ describe("rendered surface discovery", () => { ] as SemanticAxNode[]; expect( discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }), + ).toEqual([expect.objectContaining({ backendNodeId: 2 })]); + }); + + it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => { + const owner = domNode(1, { tag: "div", rect: null, localRect: null }); + const canvas = domNode(2, { parentBackendNodeId: 1 }); + + expect( + discoverRenderedSurfaces( + [document([owner, canvas])], + { width: 1280, height: 720 }, + new Set([1]), + ), ).toEqual([]); }); }); @@ -134,4 +147,42 @@ describe("rendered surface clustering and projection", () => { visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })], }); }); + + it("does not merge a contained canvas with a larger independent canvas", () => { + const groups = clusterRenderedSurfaces([ + ...discoverRenderedSurfaces( + [ + document([ + domNode(1, { tag: "main", rect: null, localRect: null }), + domNode(2, { parentBackendNodeId: 1 }), + domNode(3, { + parentBackendNodeId: 1, + rect: { x: 30, y: 40, w: 200, h: 100 }, + }), + ]), + ], + { width: 1280, height: 720 }, + ), + ]); + + expect(groups).toHaveLength(2); + }); + + it("does not merge coincident canvases owned by different containers", () => { + const groups = clusterRenderedSurfaces( + discoverRenderedSurfaces( + [ + document([ + domNode(1, { tag: "section", rect: null, localRect: null }), + domNode(2, { tag: "section", rect: null, localRect: null }), + domNode(3, { parentBackendNodeId: 1 }), + domNode(4, { parentBackendNodeId: 2 }), + ]), + ], + { width: 1280, height: 720 }, + ), + ); + + expect(groups).toHaveLength(2); + }); }); diff --git a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts index 06f54e1..32db4a4 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts @@ -1,7 +1,8 @@ import type { Rect } from "@browser-skill/vom"; import type { RenderedSurface, RenderedSurfaceGroup } from "./types"; -const STACK_OVERLAP_RATIO = 0.9; +const STACK_IOU_RATIO = 0.9; +const STACK_SIZE_RATIO = 0.9; function area(rect: Rect): number { return Math.max(0, rect.w) * Math.max(0, rect.h); @@ -15,21 +16,23 @@ function intersectionArea(a: Rect, b: Rect): number { function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean { if (a.frameId !== b.frameId) return false; - const smallerArea = Math.min(area(a.visibleRect), area(b.visibleRect)); + if (a.parentBackendNodeId !== b.parentBackendNodeId) return false; + const aArea = area(a.visibleRect); + const bArea = area(b.visibleRect); + const intersection = intersectionArea(a.visibleRect, b.visibleRect); + const union = aArea + bArea - intersection; + const widthRatio = + Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w); + const heightRatio = + Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h); return ( - smallerArea > 0 && - intersectionArea(a.visibleRect, b.visibleRect) / smallerArea >= STACK_OVERLAP_RATIO + union > 0 && + intersection / union >= STACK_IOU_RATIO && + widthRatio >= STACK_SIZE_RATIO && + heightRatio >= STACK_SIZE_RATIO ); } -function unionRect(rects: Rect[]): Rect { - const x = Math.min(...rects.map((rect) => rect.x)); - const y = Math.min(...rects.map((rect) => rect.y)); - const right = Math.max(...rects.map((rect) => rect.x + rect.w)); - const bottom = Math.max(...rects.map((rect) => rect.y + rect.h)); - return { x, y, w: right - x, h: bottom - y }; -} - function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup { const representative = [...members].sort( (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder, @@ -38,12 +41,7 @@ function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup { frameId: representative.frameId, members, representative, - rect: unionRect(members.map((surface) => surface.rect)), - visibleRect: unionRect(members.map((surface) => surface.visibleRect)), label: members.map((surface) => surface.label).find((label) => label !== undefined), - existenceConfidence: members.some((surface) => surface.existenceConfidence === "high") - ? "high" - : "low", }; } @@ -61,7 +59,7 @@ export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSu .sort( (a, b) => a.frameId.localeCompare(b.frameId) || - a.visibleRect.y - b.visibleRect.y || - a.visibleRect.x - b.visibleRect.x, + a.representative.visibleRect.y - b.representative.visibleRect.y || + a.representative.visibleRect.x - b.representative.visibleRect.x, ); } diff --git a/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts b/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts index a1be11d..f4cb8e8 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts @@ -4,10 +4,6 @@ import type { CapturedFrameDocument } from "../frame-capture"; import type { SemanticAxNode } from "../semantic-graph"; import type { RenderedSurface } from "./types"; -const MIN_PROMINENT_EDGE = 32; -const MIN_PROMINENT_AREA = 4_096; -const NON_CONTENT_ROLES = new Set(["", "generic", "none", "presentation"]); - function clean(value: string | undefined): string | undefined { const normalized = value?.replace(/\s+/g, " ").trim(); return normalized || undefined; @@ -22,10 +18,15 @@ function intersectViewport(rect: Rect, viewport: Viewport): Rect | null { return { x, y, w: right - x, h: bottom - y }; } -function hiddenByDomPolicy(node: CapturedNode, nodesByBackend: Map): boolean { +function excludedByDomPolicy( + node: CapturedNode, + nodesByBackend: Map, + excludedBackendNodeIds: ReadonlySet, +): boolean { let current: CapturedNode | undefined = node; let guard = 0; while (current && guard <= nodesByBackend.size) { + if (excludedBackendNodeIds.has(current.backendNodeId)) return true; const attrs = current.attrs; if ( Object.prototype.hasOwnProperty.call(attrs, "hidden") || @@ -43,51 +44,17 @@ function hiddenByDomPolicy(node: CapturedNode, nodesByBackend: Map node.backendDOMNodeId === canvas.backendNodeId && node.ignored !== true, - ); - if (!owner?.childIds?.length) return false; - const byId = new Map(axNodes.map((node) => [node.nodeId, node])); - const queue = [...owner.childIds]; - const seen = new Set(); - while (queue.length > 0) { - const id = queue.shift() as string; - if (seen.has(id)) continue; - seen.add(id); - const node = byId.get(id); - if (!node || node.ignored === true) continue; - const role = String(node.role?.value ?? "").toLowerCase(); - if (!NON_CONTENT_ROLES.has(role)) return true; - queue.push(...(node.childIds ?? [])); - } - return false; -} - -function isProminent(node: CapturedNode, visibleRect: Rect, label: string | undefined): boolean { - const tabindex = Number.parseInt(node.attrs.tabindex ?? "", 10); - const interactionSignal = - node.cursor === "pointer" || (Number.isFinite(tabindex) && tabindex >= 0); - return ( - label !== undefined || - interactionSignal || - (visibleRect.w >= MIN_PROMINENT_EDGE && - visibleRect.h >= MIN_PROMINENT_EDGE && - visibleRect.w * visibleRect.h >= MIN_PROMINENT_AREA) - ); -} - export function discoverRenderedSurfaces( documents: CapturedFrameDocument[], viewport: Viewport, + excludedBackendNodeIds: ReadonlySet = new Set(), ): RenderedSurface[] { const surfaces: RenderedSurface[] = []; for (const document of documents) { const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node])); for (const node of document.domNodes) { if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue; - if (hiddenByDomPolicy(node, nodesByBackend)) continue; - if (hasVerifiedNativeMirror(node, document.axNodes)) continue; + if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue; const visibleRect = intersectViewport(node.rect, viewport); if (!visibleRect) continue; const label = clean(node.attrs["aria-label"] ?? node.attrs.title); @@ -96,12 +63,9 @@ export function discoverRenderedSurfaces( frameId: document.frameId, backendNodeId: node.backendNodeId, parentBackendNodeId: node.parentBackendNodeId, - rect: node.rect, - ...(node.localRect ? { localRect: node.localRect } : {}), visibleRect, paintOrder: node.paintOrder, ...(label ? { label } : {}), - existenceConfidence: isProminent(node, visibleRect, label) ? "high" : "low", }); } } diff --git a/apps/extension/src/tools/vom/rendered-surfaces/project.ts b/apps/extension/src/tools/vom/rendered-surfaces/project.ts index 64f3ad3..97e0b7c 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/project.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/project.ts @@ -1,4 +1,4 @@ -import type { VomScene, VomVisualSurface, VomVisualSurfaceSummary } from "@browser-skill/vom"; +import type { VomScene, VomVisualSurface } from "@browser-skill/vom"; import type { CapturedFrameDocument } from "../frame-capture"; import type { SemanticAxNode } from "../semantic-graph"; import type { RenderedSurfaceGroup } from "./types"; @@ -7,43 +7,69 @@ function frameBackendKey(frameId: string, backendNodeId: number): string { return `${frameId}\u0000${backendNodeId}`; } -function nearestSceneParent( - scene: VomScene, - documents: CapturedFrameDocument[], - group: RenderedSurfaceGroup, -): number | null { - const sceneByBackend = new Map( - scene.nodes.flatMap((node) => - node.backendNodeId === undefined - ? [] - : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const], - ), - ); - const document = documents.find((candidate) => candidate.frameId === group.frameId); - const domByBackend = new Map(document?.domNodes.map((node) => [node.backendNodeId, node]) ?? []); +interface ProjectionIndex { + sceneByBackend: ReadonlyMap; + documentByFrame: ReadonlyMap>; + domByFrame: ReadonlyMap< + string, + ReadonlyMap["domNodes"][number]> + >; + rootSceneIdByFrame: ReadonlyMap; +} + +function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null { + const document = index.documentByFrame.get(group.frameId); + const domByBackend = index.domByFrame.get(group.frameId) ?? new Map(); let parentBackendNodeId = group.representative.parentBackendNodeId; let guard = 0; while (parentBackendNodeId !== null && guard <= domByBackend.size) { - const sceneId = sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId)); + const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId)); if (sceneId !== undefined) return sceneId; parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null; guard += 1; } if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) { - const owner = sceneByBackend.get( + const owner = index.sceneByBackend.get( frameBackendKey(document.parentFrameId, document.ownerBackendNodeId), ); if (owner !== undefined) return owner; } - return ( - scene.nodes.find( - (node) => - node.frameId === group.frameId && - ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""), - )?.id ?? null - ); + return index.rootSceneIdByFrame.get(group.frameId) ?? null; +} + +function buildProjectionIndex( + scene: VomScene, + documents: CapturedFrameDocument[], +): ProjectionIndex { + const rootSceneIdByFrame = new Map(); + for (const node of scene.nodes) { + if ( + node.frameId && + !rootSceneIdByFrame.has(node.frameId) && + ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "") + ) { + rootSceneIdByFrame.set(node.frameId, node.id); + } + } + return { + sceneByBackend: new Map( + scene.nodes.flatMap((node) => + node.backendNodeId === undefined + ? [] + : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const], + ), + ), + documentByFrame: new Map(documents.map((document) => [document.frameId, document])), + domByFrame: new Map( + documents.map((document) => [ + document.frameId, + new Map(document.domNodes.map((node) => [node.backendNodeId, node])), + ]), + ), + rootSceneIdByFrame, + }; } export function projectRenderedSurfaces( @@ -51,37 +77,23 @@ export function projectRenderedSurfaces( documents: CapturedFrameDocument[], groups: RenderedSurfaceGroup[], ): VomScene { + const index = buildProjectionIndex(scene, documents); const visualSurfaces: VomVisualSurface[] = []; - const summaryCounts = new Map< - string, - { parentId: number | null; frameId: string; count: number } - >(); for (const group of groups) { - const parentId = nearestSceneParent(scene, documents, group); - if (group.existenceConfidence === "low") { - const key = `${group.frameId}\u0000${parentId ?? "root"}`; - const current = summaryCounts.get(key) ?? { parentId, frameId: group.frameId, count: 0 }; - current.count += group.members.length; - summaryCounts.set(key, current); - continue; - } + const parentId = nearestSceneParent(index, group); visualSurfaces.push({ parentId, backendNodeId: group.representative.backendNodeId, frameId: group.frameId, renderingKind: "canvas", - rect: group.visibleRect, - ...(group.representative.localRect ? { localRect: group.representative.localRect } : {}), ...(group.label ? { label: group.label } : {}), memberCount: group.members.length, }); } - const visualSurfaceSummaries: VomVisualSurfaceSummary[] = [...summaryCounts.values()]; return { ...scene, ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}), - ...(visualSurfaceSummaries.length > 0 ? { visualSurfaceSummaries } : {}), }; } diff --git a/apps/extension/src/tools/vom/rendered-surfaces/types.ts b/apps/extension/src/tools/vom/rendered-surfaces/types.ts index 80289f0..947d26b 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/types.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/types.ts @@ -1,26 +1,18 @@ import type { Rect } from "@browser-skill/vom"; -export type SurfaceExistenceConfidence = "high" | "low"; - export interface RenderedSurface { renderingKind: "canvas"; frameId: string; backendNodeId: number; parentBackendNodeId: number | null; - rect: Rect; - localRect?: Rect; visibleRect: Rect; paintOrder: number; label?: string; - existenceConfidence: SurfaceExistenceConfidence; } export interface RenderedSurfaceGroup { frameId: string; members: RenderedSurface[]; representative: RenderedSurface; - rect: Rect; - visibleRect: Rect; label?: string; - existenceConfidence: SurfaceExistenceConfidence; } diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index a72cac6..ee97bfb 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -29,7 +29,7 @@ function coreRefs(refs: ReturnType["refs"]) { } describe("renderVom single-layer page", () => { - it("renders visual surfaces as screenshot-only refs and low-confidence counts as summaries", () => { + it("renders visual surfaces as screenshot-only refs", () => { const input = scene([ node({ id: 1, role: "RootWebArea", frameId: "main" }), node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }), @@ -40,19 +40,15 @@ describe("renderVom single-layer page", () => { backendNodeId: 99, frameId: "child", renderingKind: "canvas", - rect: { x: 10, y: 20, w: 800, h: 500 }, label: "Data grid", memberCount: 2, }, ]; - input.visualSurfaceSummaries = [{ parentId: 2, frameId: "child", count: 3 }]; - const out = renderVom(input); expect(out.text).toContain( '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]', ); - expect(out.text).toContain("[3 additional visual surfaces are not represented]"); expect(out.refs).toContainEqual( expect.objectContaining({ ref: "e1", diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index 85308d3..c0f1981 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -19,5 +19,4 @@ export type { VomResult, VomScene, VomVisualSurface, - VomVisualSurfaceSummary, } from "./types"; diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 946a440..26eb55f 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -10,7 +10,6 @@ import type { VomResult, VomScene, VomVisualSurface, - VomVisualSurfaceSummary, } from "./types"; const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]); @@ -740,7 +739,6 @@ interface RenderState { surfaceMap: Map; scopeMap: Map; visualSurfaces: Map; - visualSurfaceSummaries: Map; } function groupedByParent( @@ -786,23 +784,6 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render }); state.nextRef += 1; } - - for (const summary of state.visualSurfaceSummaries.get(parentId) ?? []) { - if (depth > state.maxDepth) { - state.truncated = true; - continue; - } - const noun = summary.count === 1 ? "surface is" : "surfaces are"; - const line = `${" ".repeat(depth)}[${summary.count} additional visual ${noun} not represented]`; - const nextTokens = state.tokens + estimateTokens(line); - if (nextTokens > state.maxTokens) { - state.truncated = true; - state.stopped = true; - return; - } - state.lines.push(line); - state.tokens = nextTokens; - } } function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void { @@ -904,7 +885,6 @@ function renderNodes( surfaces: CondSurface[] = [], activeScopeBlocks: ActiveScopeBlock[] = [], visualSurfaces: VomVisualSurface[] = [], - visualSurfaceSummaries: VomVisualSurfaceSummary[] = [], ): RenderState { const children = buildChildren(nodes); const state: RenderState = { @@ -924,7 +904,6 @@ function renderNodes( surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])), scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])), visualSurfaces: groupedByParent(visualSurfaces), - visualSurfaceSummaries: groupedByParent(visualSurfaceSummaries), }; renderTree(children, null, 1, state); @@ -1139,7 +1118,6 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp scene.surfaces, scene.activeScopeBlocks, scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)), - scene.visualSurfaceSummaries?.filter((summary) => visibleSurface(summary.parentId)), ); state.lines.push(renderPageOcclusionLine(hiddenCount)); @@ -1169,7 +1147,6 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult scene.surfaces, scene.activeScopeBlocks, scene.visualSurfaces, - scene.visualSurfaceSummaries, ); return { diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index d5d52b9..6464212 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -63,7 +63,6 @@ export interface VomScene { activeScopeBlocks?: ActiveScopeBlock[]; /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */ visualSurfaces?: VomVisualSurface[]; - visualSurfaceSummaries?: VomVisualSurfaceSummary[]; } export interface VomOptions { @@ -108,18 +107,10 @@ export interface VomVisualSurface { backendNodeId: number; frameId: string; renderingKind: "canvas"; - rect: Rect; - localRect?: Rect; label?: string; memberCount: number; } -export interface VomVisualSurfaceSummary { - parentId: number | null; - frameId: string; - count: number; -} - export interface RenderedRef extends VomRef { role?: string; name?: string;