fix(vom): canvas recognition

apps/extension/src/tools/observation.ts

@@ -1087,7 +1087,13 @@ export async function captureVomObservation(
     ? projectRenderedSurfaces(
         decoratedScene,
         normalizedDocuments,
-        clusterRenderedSurfaces(discoverRenderedSurfaces(normalizedDocuments, captured.viewport)),
+        clusterRenderedSurfaces(
+          discoverRenderedSurfaces(
+            normalizedDocuments,
+            captured.viewport,
+            captured.excludedBackendNodeIds,
+          ),
+        ),
       )
     : decoratedScene;
   const rendered = renderVom(observedScene, {

apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts

@@ -57,18 +57,18 @@ describe("rendered surface discovery", () => {
     );
 
     expect(surfaces).toHaveLength(1);
-    expect(surfaces[0]).toMatchObject({ backendNodeId: 2, existenceConfidence: "high" });
+    expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
   });
 
-  it("keeps tiny canvases only as low-confidence discoveries", () => {
+  it("keeps tiny canvases as individually addressable exact discoveries", () => {
     const surfaces = discoverRenderedSurfaces(
       [document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
       { width: 1280, height: 720 },
     );
-    expect(surfaces[0]?.existenceConfidence).toBe("low");
+    expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
   });
 
-  it("suppresses a canvas with a verified native accessibility mirror", () => {
+  it("keeps a surface when the canvas only has partial native accessibility content", () => {
     const axNodes = [
       {
         nodeId: "canvas",
@@ -86,6 +86,19 @@ describe("rendered surface discovery", () => {
     ] as SemanticAxNode[];
     expect(
       discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
+    ).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
+  });
+
+  it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
+    const owner = domNode(1, { tag: "div", rect: null, localRect: null });
+    const canvas = domNode(2, { parentBackendNodeId: 1 });
+
+    expect(
+      discoverRenderedSurfaces(
+        [document([owner, canvas])],
+        { width: 1280, height: 720 },
+        new Set([1]),
+      ),
     ).toEqual([]);
   });
 });
@@ -134,4 +147,42 @@ describe("rendered surface clustering and projection", () => {
       visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
     });
   });
+
+  it("does not merge a contained canvas with a larger independent canvas", () => {
+    const groups = clusterRenderedSurfaces([
+      ...discoverRenderedSurfaces(
+        [
+          document([
+            domNode(1, { tag: "main", rect: null, localRect: null }),
+            domNode(2, { parentBackendNodeId: 1 }),
+            domNode(3, {
+              parentBackendNodeId: 1,
+              rect: { x: 30, y: 40, w: 200, h: 100 },
+            }),
+          ]),
+        ],
+        { width: 1280, height: 720 },
+      ),
+    ]);
+
+    expect(groups).toHaveLength(2);
+  });
+
+  it("does not merge coincident canvases owned by different containers", () => {
+    const groups = clusterRenderedSurfaces(
+      discoverRenderedSurfaces(
+        [
+          document([
+            domNode(1, { tag: "section", rect: null, localRect: null }),
+            domNode(2, { tag: "section", rect: null, localRect: null }),
+            domNode(3, { parentBackendNodeId: 1 }),
+            domNode(4, { parentBackendNodeId: 2 }),
+          ]),
+        ],
+        { width: 1280, height: 720 },
+      ),
+    );
+
+    expect(groups).toHaveLength(2);
+  });
 });

apps/extension/src/tools/vom/rendered-surfaces/cluster.ts

@@ -1,7 +1,8 @@
 import type { Rect } from "@browser-skill/vom";
 import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
 
-const STACK_OVERLAP_RATIO = 0.9;
+const STACK_IOU_RATIO = 0.9;
+const STACK_SIZE_RATIO = 0.9;
 
 function area(rect: Rect): number {
   return Math.max(0, rect.w) * Math.max(0, rect.h);
@@ -15,21 +16,23 @@ function intersectionArea(a: Rect, b: Rect): number {
 
 function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
   if (a.frameId !== b.frameId) return false;
-  const smallerArea = Math.min(area(a.visibleRect), area(b.visibleRect));
+  if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
+  const aArea = area(a.visibleRect);
+  const bArea = area(b.visibleRect);
+  const intersection = intersectionArea(a.visibleRect, b.visibleRect);
+  const union = aArea + bArea - intersection;
+  const widthRatio =
+    Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
+  const heightRatio =
+    Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
   return (
-    smallerArea > 0 &&
-    intersectionArea(a.visibleRect, b.visibleRect) / smallerArea >= STACK_OVERLAP_RATIO
+    union > 0 &&
+    intersection / union >= STACK_IOU_RATIO &&
+    widthRatio >= STACK_SIZE_RATIO &&
+    heightRatio >= STACK_SIZE_RATIO
   );
 }
 
-function unionRect(rects: Rect[]): Rect {
-  const x = Math.min(...rects.map((rect) => rect.x));
-  const y = Math.min(...rects.map((rect) => rect.y));
-  const right = Math.max(...rects.map((rect) => rect.x + rect.w));
-  const bottom = Math.max(...rects.map((rect) => rect.y + rect.h));
-  return { x, y, w: right - x, h: bottom - y };
-}
-
 function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
   const representative = [...members].sort(
     (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
@@ -38,12 +41,7 @@ function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
     frameId: representative.frameId,
     members,
     representative,
-    rect: unionRect(members.map((surface) => surface.rect)),
-    visibleRect: unionRect(members.map((surface) => surface.visibleRect)),
     label: members.map((surface) => surface.label).find((label) => label !== undefined),
-    existenceConfidence: members.some((surface) => surface.existenceConfidence === "high")
-      ? "high"
-      : "low",
   };
 }
 
@@ -61,7 +59,7 @@ export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSu
     .sort(
       (a, b) =>
         a.frameId.localeCompare(b.frameId) ||
-        a.visibleRect.y - b.visibleRect.y ||
-        a.visibleRect.x - b.visibleRect.x,
+        a.representative.visibleRect.y - b.representative.visibleRect.y ||
+        a.representative.visibleRect.x - b.representative.visibleRect.x,
     );
 }

apps/extension/src/tools/vom/rendered-surfaces/discovery.ts

@@ -4,10 +4,6 @@ import type { CapturedFrameDocument } from "../frame-capture";
 import type { SemanticAxNode } from "../semantic-graph";
 import type { RenderedSurface } from "./types";
 
-const MIN_PROMINENT_EDGE = 32;
-const MIN_PROMINENT_AREA = 4_096;
-const NON_CONTENT_ROLES = new Set(["", "generic", "none", "presentation"]);
-
 function clean(value: string | undefined): string | undefined {
   const normalized = value?.replace(/\s+/g, " ").trim();
   return normalized || undefined;
@@ -22,10 +18,15 @@ function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
   return { x, y, w: right - x, h: bottom - y };
 }
 
-function hiddenByDomPolicy(node: CapturedNode, nodesByBackend: Map<number, CapturedNode>): boolean {
+function excludedByDomPolicy(
+  node: CapturedNode,
+  nodesByBackend: Map<number, CapturedNode>,
+  excludedBackendNodeIds: ReadonlySet<number>,
+): boolean {
   let current: CapturedNode | undefined = node;
   let guard = 0;
   while (current && guard <= nodesByBackend.size) {
+    if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
     const attrs = current.attrs;
     if (
       Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
@@ -43,51 +44,17 @@ function hiddenByDomPolicy(node: CapturedNode, nodesByBackend: Map<number, Captu
   return false;
 }
 
-function hasVerifiedNativeMirror(canvas: CapturedNode, axNodes: SemanticAxNode[]): boolean {
-  const owner = axNodes.find(
-    (node) => node.backendDOMNodeId === canvas.backendNodeId && node.ignored !== true,
-  );
-  if (!owner?.childIds?.length) return false;
-  const byId = new Map(axNodes.map((node) => [node.nodeId, node]));
-  const queue = [...owner.childIds];
-  const seen = new Set<string>();
-  while (queue.length > 0) {
-    const id = queue.shift() as string;
-    if (seen.has(id)) continue;
-    seen.add(id);
-    const node = byId.get(id);
-    if (!node || node.ignored === true) continue;
-    const role = String(node.role?.value ?? "").toLowerCase();
-    if (!NON_CONTENT_ROLES.has(role)) return true;
-    queue.push(...(node.childIds ?? []));
-  }
-  return false;
-}
-
-function isProminent(node: CapturedNode, visibleRect: Rect, label: string | undefined): boolean {
-  const tabindex = Number.parseInt(node.attrs.tabindex ?? "", 10);
-  const interactionSignal =
-    node.cursor === "pointer" || (Number.isFinite(tabindex) && tabindex >= 0);
-  return (
-    label !== undefined ||
-    interactionSignal ||
-    (visibleRect.w >= MIN_PROMINENT_EDGE &&
-      visibleRect.h >= MIN_PROMINENT_EDGE &&
-      visibleRect.w * visibleRect.h >= MIN_PROMINENT_AREA)
-  );
-}
-
 export function discoverRenderedSurfaces(
   documents: CapturedFrameDocument<SemanticAxNode>[],
   viewport: Viewport,
+  excludedBackendNodeIds: ReadonlySet<number> = new Set(),
 ): RenderedSurface[] {
   const surfaces: RenderedSurface[] = [];
   for (const document of documents) {
     const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
     for (const node of document.domNodes) {
       if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
-      if (hiddenByDomPolicy(node, nodesByBackend)) continue;
-      if (hasVerifiedNativeMirror(node, document.axNodes)) continue;
+      if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
       const visibleRect = intersectViewport(node.rect, viewport);
       if (!visibleRect) continue;
       const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
@@ -96,12 +63,9 @@ export function discoverRenderedSurfaces(
         frameId: document.frameId,
         backendNodeId: node.backendNodeId,
         parentBackendNodeId: node.parentBackendNodeId,
-        rect: node.rect,
-        ...(node.localRect ? { localRect: node.localRect } : {}),
         visibleRect,
         paintOrder: node.paintOrder,
         ...(label ? { label } : {}),
-        existenceConfidence: isProminent(node, visibleRect, label) ? "high" : "low",
       });
     }
   }

apps/extension/src/tools/vom/rendered-surfaces/project.ts

@@ -1,4 +1,4 @@
-import type { VomScene, VomVisualSurface, VomVisualSurfaceSummary } from "@browser-skill/vom";
+import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
 import type { CapturedFrameDocument } from "../frame-capture";
 import type { SemanticAxNode } from "../semantic-graph";
 import type { RenderedSurfaceGroup } from "./types";
@@ -7,43 +7,69 @@ function frameBackendKey(frameId: string, backendNodeId: number): string {
   return `${frameId}\u0000${backendNodeId}`;
 }
 
-function nearestSceneParent(
-  scene: VomScene,
-  documents: CapturedFrameDocument<SemanticAxNode>[],
-  group: RenderedSurfaceGroup,
-): number | null {
-  const sceneByBackend = new Map(
-    scene.nodes.flatMap((node) =>
-      node.backendNodeId === undefined
-        ? []
-        : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
-    ),
-  );
-  const document = documents.find((candidate) => candidate.frameId === group.frameId);
-  const domByBackend = new Map(document?.domNodes.map((node) => [node.backendNodeId, node]) ?? []);
+interface ProjectionIndex {
+  sceneByBackend: ReadonlyMap<string, number>;
+  documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
+  domByFrame: ReadonlyMap<
+    string,
+    ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
+  >;
+  rootSceneIdByFrame: ReadonlyMap<string, number>;
+}
+
+function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
+  const document = index.documentByFrame.get(group.frameId);
+  const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
   let parentBackendNodeId = group.representative.parentBackendNodeId;
   let guard = 0;
   while (parentBackendNodeId !== null && guard <= domByBackend.size) {
-    const sceneId = sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
+    const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
     if (sceneId !== undefined) return sceneId;
     parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
     guard += 1;
   }
 
   if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
-    const owner = sceneByBackend.get(
+    const owner = index.sceneByBackend.get(
       frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
     );
     if (owner !== undefined) return owner;
   }
 
-  return (
-    scene.nodes.find(
-      (node) =>
-        node.frameId === group.frameId &&
-        ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
-    )?.id ?? null
-  );
+  return index.rootSceneIdByFrame.get(group.frameId) ?? null;
+}
+
+function buildProjectionIndex(
+  scene: VomScene,
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+): ProjectionIndex {
+  const rootSceneIdByFrame = new Map<string, number>();
+  for (const node of scene.nodes) {
+    if (
+      node.frameId &&
+      !rootSceneIdByFrame.has(node.frameId) &&
+      ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
+    ) {
+      rootSceneIdByFrame.set(node.frameId, node.id);
+    }
+  }
+  return {
+    sceneByBackend: new Map(
+      scene.nodes.flatMap((node) =>
+        node.backendNodeId === undefined
+          ? []
+          : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
+      ),
+    ),
+    documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
+    domByFrame: new Map(
+      documents.map((document) => [
+        document.frameId,
+        new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
+      ]),
+    ),
+    rootSceneIdByFrame,
+  };
 }
 
 export function projectRenderedSurfaces(
@@ -51,37 +77,23 @@ export function projectRenderedSurfaces(
   documents: CapturedFrameDocument<SemanticAxNode>[],
   groups: RenderedSurfaceGroup[],
 ): VomScene {
+  const index = buildProjectionIndex(scene, documents);
   const visualSurfaces: VomVisualSurface[] = [];
-  const summaryCounts = new Map<
-    string,
-    { parentId: number | null; frameId: string; count: number }
-  >();
 
   for (const group of groups) {
-    const parentId = nearestSceneParent(scene, documents, group);
-    if (group.existenceConfidence === "low") {
-      const key = `${group.frameId}\u0000${parentId ?? "root"}`;
-      const current = summaryCounts.get(key) ?? { parentId, frameId: group.frameId, count: 0 };
-      current.count += group.members.length;
-      summaryCounts.set(key, current);
-      continue;
-    }
+    const parentId = nearestSceneParent(index, group);
     visualSurfaces.push({
       parentId,
       backendNodeId: group.representative.backendNodeId,
       frameId: group.frameId,
       renderingKind: "canvas",
-      rect: group.visibleRect,
-      ...(group.representative.localRect ? { localRect: group.representative.localRect } : {}),
       ...(group.label ? { label: group.label } : {}),
       memberCount: group.members.length,
     });
   }
 
-  const visualSurfaceSummaries: VomVisualSurfaceSummary[] = [...summaryCounts.values()];
   return {
     ...scene,
     ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
-    ...(visualSurfaceSummaries.length > 0 ? { visualSurfaceSummaries } : {}),
   };
 }

apps/extension/src/tools/vom/rendered-surfaces/types.ts

@@ -1,26 +1,18 @@
 import type { Rect } from "@browser-skill/vom";
 
-export type SurfaceExistenceConfidence = "high" | "low";
-
 export interface RenderedSurface {
   renderingKind: "canvas";
   frameId: string;
   backendNodeId: number;
   parentBackendNodeId: number | null;
-  rect: Rect;
-  localRect?: Rect;
   visibleRect: Rect;
   paintOrder: number;
   label?: string;
-  existenceConfidence: SurfaceExistenceConfidence;
 }
 
 export interface RenderedSurfaceGroup {
   frameId: string;
   members: RenderedSurface[];
   representative: RenderedSurface;
-  rect: Rect;
-  visibleRect: Rect;
   label?: string;
-  existenceConfidence: SurfaceExistenceConfidence;
 }