feat(vom): two channels to deal with canvas

apps/extension/src/session-manager/__tests__/ref-store.test.ts

@@ -13,6 +13,8 @@ describe("RefStore", () => {
       backendNodeId: 42,
       tabId: 7,
       generation: 0,
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
     expect(s.size()).toBe(1);
     expect(s.isEmpty()).toBe(false);
@@ -48,4 +50,31 @@ describe("RefStore", () => {
       cdpSessionId: "child-session",
     });
   });
+
+  it("preserves explicit capabilities for visual surface refs", () => {
+    const s = new RefStore();
+    s.set("e1", 42, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    expect(s.resolveEntry("e1")).toMatchObject({
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+  });
+
+  it("never grants ordinary interaction to a surface through defaults or caller input", () => {
+    const s = new RefStore();
+    s.set("e1", 42, { tabId: 7, kind: "surface" });
+    s.set("e2", 43, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["interact", "screenshot"],
+    });
+
+    expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
+    expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
+  });
 });

apps/extension/src/session-manager/ref-store.ts

@@ -1,3 +1,5 @@
+import type { Rect } from "@browser-skill/vom";
+
 /**
  * Per-session map from `@e<N>` snapshot refs to a CDP node address.
  *
@@ -12,12 +14,17 @@
  * inside the `SessionContext`.
  */
 export type BackendNodeId = number;
+export type RefCapability = "interact" | "screenshot";
+export type RefTargetKind = "dom" | "surface";
 
 export interface RefEntry {
   backendNodeId: BackendNodeId;
   tabId: number | null;
   frameId?: string;
   cdpSessionId?: string;
+  visibleRect?: Rect;
+  kind: RefTargetKind;
+  capabilities: RefCapability[];
   generation: number;
 }
 
@@ -28,8 +35,19 @@ export type RefInput =
       tabId: number;
       frameId?: string;
       cdpSessionId?: string;
+      visibleRect?: Rect;
+      kind?: RefTargetKind;
+      capabilities?: RefCapability[];
     };
 
+function capabilitiesFor(
+  kind: RefTargetKind,
+  capabilities: RefCapability[] | undefined,
+): RefCapability[] {
+  if (kind === "surface") return ["screenshot"];
+  return capabilities ?? ["interact", "screenshot"];
+}
+
 export class RefStore {
   private readonly map = new Map<string, RefEntry>();
   private generation = 0;
@@ -70,13 +88,20 @@ export class RefStore {
       tabId?: number;
       frameId?: string;
       cdpSessionId?: string;
+      visibleRect?: Rect;
+      kind?: RefTargetKind;
+      capabilities?: RefCapability[];
     } = {},
   ): void {
+    const kind = opts.kind ?? "dom";
     this.map.set(normaliseRef(ref), {
       backendNodeId: id,
       tabId: opts.tabId ?? null,
       ...(opts.frameId ? { frameId: opts.frameId } : {}),
       ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
+      ...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
+      kind,
+      capabilities: capabilitiesFor(kind, opts.capabilities),
       generation: this.generation,
     });
   }
@@ -94,14 +119,20 @@ export class RefStore {
       return {
         backendNodeId: input,
         tabId: null,
+        kind: "dom",
+        capabilities: ["interact", "screenshot"],
         generation: this.generation,
       };
     }
+    const kind = input.kind ?? "dom";
     return {
       backendNodeId: input.backendNodeId,
       tabId: input.tabId,
       ...(input.frameId ? { frameId: input.frameId } : {}),
       ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
+      ...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
+      kind,
+      capabilities: capabilitiesFor(kind, input.capabilities),
       generation: this.generation,
     };
   }

apps/extension/src/tools/__tests__/interaction.test.ts

@@ -123,6 +123,29 @@ describe("handleClick", () => {
     expect(fake.cdp.send).not.toHaveBeenCalled();
   });
 
+  it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+    const fake = makeFakeCdp({});
+
+    const res = await handleClick(
+      sm,
+      { session_id: "aa11", ref: "@e3" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expect(res).toMatchObject({
+      code: "permission_denied",
+      data: { reason: "ref_capability_denied", required_capability: "interact" },
+    });
+    expect(fake.cdp.send).not.toHaveBeenCalled();
+  });
+
   it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");

apps/extension/src/tools/__tests__/observation.test.ts

@@ -300,6 +300,76 @@ describe("handleScreenshot", () => {
     expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
   });
 
+  it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
+      }),
+      "DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
+    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
+      clip?: { width: number; height: number; scale: number };
+    };
+    expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
+    expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
+  });
+
+  it("crops a visual surface screenshot to its observation-time visible region", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 25, y: 30, w: 50, h: 40 },
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
+      }),
+      "DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
+      clip?: { x: number; y: number; width: number; height: number };
+    };
+    expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
+  });
+
   it("returns not_found for unknown ref", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");
@@ -2693,6 +2763,85 @@ describe("handleSnapshot", () => {
     );
   });
 
+  it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Canvas app" },
+      backendDOMNodeId: 100,
+    };
+    const strings = ["body", "canvas", "static", "auto"];
+    const i = (value: string) => strings.indexOf(value);
+    const send = vi.fn(async (_tabId: number, method: string) => {
+      if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
+      }
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0],
+                nodeName: [i("body"), i("canvas")],
+                backendNodeId: [100, 200],
+                attributes: [[], []],
+              },
+              layout: {
+                nodeIndex: [0, 1],
+                styles: [
+                  [i("static"), i("auto"), i("auto")],
+                  [i("static"), i("auto"), i("auto")],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [20, 30, 800, 500],
+                ],
+                paintOrders: [0, 1],
+              },
+            },
+          ],
+        };
+      }
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+    const deps = {
+      cdp: {
+        send: send as unknown as <T = unknown>(
+          tabId: number,
+          method: string,
+          params?: object,
+        ) => Promise<T>,
+        trackSessionTab: vi.fn(),
+      },
+      tabsApi: {
+        get: vi.fn(
+          async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+        ),
+        query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+      },
+      conditionalSurfaceProbe: false,
+    };
+
+    const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
+    if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
+    expect(observed.text).toContain('@e1 surface "canvas visual surface"');
+    expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
+      backendNodeId: 200,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
+    if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
+    expect(snapshot.text).not.toContain("visual surface");
+    expect(snapshot.ref_count).toBe(0);
+  });
+
   it("allows passive snapshots of explicit user-window tabs", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");

apps/extension/src/tools/__tests__/snapshot-ref.test.ts

@@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => {
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
     expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
   });
 
@@ -53,6 +57,8 @@ describe("resolveSnapshotRef", () => {
       tabId: 4,
       frameId: "child-frame",
       cdpSessionId: "child-session",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
 
     const expected = {
@@ -60,6 +66,8 @@ describe("resolveSnapshotRef", () => {
       refKey: "e3",
       frameId: "child-frame",
       cdpSessionId: "child-session",
+      kind: "dom" as const,
+      capabilities: ["interact", "screenshot"] as const,
     };
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => {
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
+    });
+  });
+
+  it("rejects an operation outside the ref's declared capabilities", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
+      code: "permission_denied",
+      message: "ref @e3 does not support interact",
+      data: {
+        reason: "ref_capability_denied",
+        required_capability: "interact",
+        kind: "surface",
+      },
     });
   });
 });

apps/extension/src/tools/interaction.ts

@@ -159,7 +159,7 @@ async function resolveBackendNode(
     };
   }
   if (hasRef) {
-    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
+    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
     if (isRpcError(resolved)) return resolved;
     return {
       backendNodeId: resolved.backendNodeId,

apps/extension/src/tools/observation.ts

@@ -8,6 +8,7 @@ import {
   applyVomInteractionRecovery,
   type CondSurface,
   isVomReferenceNode,
+  type Rect,
   renderVom,
   type VomNode,
   type VomOptions,
@@ -59,6 +60,11 @@ import {
   type CaptureVomObservationResult,
   projectRecordSafeObservation,
 } from "./vom/record-safe-observation";
+import {
+  clusterRenderedSurfaces,
+  discoverRenderedSurfaces,
+  projectRenderedSurfaces,
+} from "./vom/rendered-surfaces";
 import {
   buildSemanticGraph,
   buildSemanticVomScene,
@@ -90,6 +96,9 @@ export const normaliseRef = sharedNormaliseRef;
 // screenshot — `tool.screenshot`
 // ---------------------------------------------------------------------------
 
+const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
+const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
+
 /**
  * Strip the `data:image/...;base64,` prefix from a Chrome
  * `captureVisibleTab` dataURL and return the raw base64 payload.
@@ -182,6 +191,8 @@ async function captureElementScreenshot(
   target: CdpTarget,
   backendNodeId: number,
   frameId?: string,
+  visualSurface = false,
+  observedVisibleRect?: Rect,
   signal?: AbortSignal,
 ): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
   if (signal?.aborted) return cancelled("screenshot");
@@ -189,11 +200,36 @@ async function captureElementScreenshot(
     cdp,
     tabId,
     { target, backendNodeId, ...(frameId ? { frameId } : {}) },
-    { scrollIntoView: true },
+    { scrollIntoView: !visualSurface },
   );
   if (isRpcError(geometry)) return geometry;
   if (signal?.aborted) return cancelled("screenshot");
-  const rect = geometry.topBounds;
+  const liveRect = geometry.topBounds;
+  const rect =
+    visualSurface && observedVisibleRect
+      ? (() => {
+          const x = Math.max(liveRect.x, observedVisibleRect.x);
+          const y = Math.max(liveRect.y, observedVisibleRect.y);
+          const right = Math.min(
+            liveRect.x + liveRect.width,
+            observedVisibleRect.x + observedVisibleRect.w,
+          );
+          const bottom = Math.min(
+            liveRect.y + liveRect.height,
+            observedVisibleRect.y + observedVisibleRect.h,
+          );
+          return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
+        })()
+      : liveRect;
+  if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
+  const scale = visualSurface
+    ? Math.min(
+        1,
+        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
+        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
+        Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
+      )
+    : 1;
 
   try {
     const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
@@ -203,7 +239,7 @@ async function captureElementScreenshot(
         y: rect.y,
         width: rect.width,
         height: rect.height,
-        scale: 1,
+        scale,
       },
     });
     if (signal?.aborted) return cancelled("screenshot");
@@ -310,7 +346,7 @@ export async function handleScreenshot(
     if (!deps.cdp) {
       return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
     }
-    const node = resolveSnapshotRef(ctx, ref, target.tabId);
+    const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
     if (isRpcError(node)) return node;
     if (signal?.aborted) return cancelled("screenshot");
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -330,6 +366,8 @@ export async function handleScreenshot(
           nodeTarget,
           node.backendNodeId,
           node.frameId,
+          node.kind === "surface",
+          node.visibleRect,
           signal,
         ),
       deps.sendToTab,
@@ -1023,6 +1061,7 @@ async function captureForVom(
 
 export interface CaptureVomObservationOptions extends VomOptions {
   conditionalSurfaceProbe?: boolean;
+  renderedSurfaceDiscovery?: boolean;
   hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
   signal?: AbortSignal;
 }
@@ -1064,7 +1103,20 @@ export async function captureVomObservation(
     ),
   );
   const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
-  const rendered = renderVom(decoratedScene, {
+  const observedScene = options.renderedSurfaceDiscovery
+    ? projectRenderedSurfaces(
+        decoratedScene,
+        normalizedDocuments,
+        clusterRenderedSurfaces(
+          discoverRenderedSurfaces(
+            normalizedDocuments,
+            captured.viewport,
+            captured.excludedBackendNodeIds,
+          ),
+        ),
+      )
+    : decoratedScene;
+  const rendered = renderVom(observedScene, {
     maxDepth: options.maxDepth,
     maxTokens: options.maxTokens,
     redactValues: options.redactValues,
@@ -1115,6 +1167,7 @@ async function handleVomObservation(
       maxTokens: params.max_tokens,
       activeRegionPolicy: true,
       conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
+      renderedSurfaceDiscovery: toolName === "observe",
       hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
       signal,
     });
@@ -1132,6 +1185,9 @@ async function handleVomObservation(
             tabId: target.tabId,
             ...(ref.frameId ? { frameId: ref.frameId } : {}),
             ...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
+            ...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
+            ...(ref.kind ? { kind: ref.kind } : {}),
+            ...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
           },
         ] as const;
       }),

apps/extension/src/tools/snapshot-ref.ts

@@ -2,8 +2,9 @@
 // compound CDP node identity via the session RefStore, and build stable
 // `ref_not_found` errors for hard-failure tool paths.
 
+import type { Rect } from "@browser-skill/vom";
 import type { SessionContext } from "@/session-manager/manager";
-import { normaliseRef } from "@/session-manager/ref-store";
+import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
 import type { RpcError } from "@/transport/types";
 import { rpcError } from "./errors";
 
@@ -12,6 +13,9 @@ export interface SnapshotRefLookup {
   refKey: string;
   frameId?: string;
   cdpSessionId?: string;
+  visibleRect?: Rect;
+  kind: RefTargetKind;
+  capabilities: RefCapability[];
 }
 
 function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
@@ -37,6 +41,9 @@ export function lookupSnapshotRef(
     refKey,
     ...(entry.frameId ? { frameId: entry.frameId } : {}),
     ...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
+    ...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
+    kind: entry.kind,
+    capabilities: entry.capabilities,
   };
 }
 
@@ -49,6 +56,7 @@ export function resolveSnapshotRef(
   ctx: SessionContext,
   ref: string,
   tabId: number,
+  requiredCapability?: RefCapability,
 ): SnapshotRefLookup | RpcError {
   const looked = lookupSnapshotRef(ctx, ref, tabId);
   if (looked === null) {
@@ -58,5 +66,13 @@ export function resolveSnapshotRef(
       `ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
     );
   }
+  if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
+    return rpcError(
+      "permission_denied",
+      "ref_capability_denied",
+      `ref ${ref} does not support ${requiredCapability}`,
+      { ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
+    );
+  }
   return looked;
 }

apps/extension/src/tools/vom/__tests__/capture.test.ts

@@ -722,6 +722,44 @@ describe("captureViewModel", () => {
     expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
   });
 
+  it("propagates fully transparent ancestor suppression to painted descendants", async () => {
+    const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
+    const i = (value: string) => S.indexOf(value);
+    const snapshot = {
+      strings: S,
+      documents: [
+        {
+          nodes: {
+            parentIndex: [-1, 0, 1, 2],
+            nodeName: [i("html"), i("body"), i("div"), i("canvas")],
+            backendNodeId: [10, 11, 12, 13],
+            attributes: [[], [], [], []],
+          },
+          layout: {
+            nodeIndex: [1, 2, 3],
+            styles: [
+              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
+              [i("static"), i("auto"), i("auto"), i("visible"), i("0")],
+              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
+            ],
+            bounds: [
+              [0, 0, 1000, 800],
+              [10, 10, 200, 100],
+              [10, 10, 200, 100],
+            ],
+            paintOrders: [0, 1, 2],
+          },
+        },
+      ],
+    };
+
+    const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
+    const canvas = nodes.find((node) => node.backendNodeId === 13);
+
+    expect(canvas?.rendered).toBe(true);
+    expect(canvas?.visuallySuppressed).toBe(true);
+  });
+
   it("subtracts scroll offset to make rects viewport-relative", async () => {
     const cdp = {
       send: vi.fn(async (_t: number, method: string) => {

apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts

@@ -0,0 +1,256 @@
+import type { VomScene } from "@browser-skill/vom";
+import { describe, expect, it } from "vitest";
+import type { CapturedNode } from "../capture";
+import type { CapturedFrameDocument } from "../frame-capture";
+import {
+  clusterRenderedSurfaces,
+  discoverRenderedSurfaces,
+  projectRenderedSurfaces,
+} from "../rendered-surfaces";
+import type { SemanticAxNode } from "../semantic-graph";
+
+function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
+  return {
+    backendNodeId,
+    parentBackendNodeId: null,
+    frameId: "main",
+    ownerFrameBackendNodeId: null,
+    tag: "canvas",
+    attrs: {},
+    rect: { x: 10, y: 20, w: 800, h: 500 },
+    localRect: { x: 10, y: 20, w: 800, h: 500 },
+    paintOrder: 1,
+    position: "static",
+    pointerEvents: "auto",
+    rendered: true,
+    ...overrides,
+  };
+}
+
+function document(
+  domNodes: CapturedNode[],
+  axNodes: SemanticAxNode[] = [],
+): CapturedFrameDocument<SemanticAxNode> {
+  return {
+    frameId: domNodes[0]?.frameId ?? "main",
+    contextScopeId: domNodes[0]?.frameId ?? "main",
+    target: { tabId: 4 },
+    domNodes,
+    axNodes,
+  };
+}
+
+describe("rendered surface discovery", () => {
+  it("finds prominent rendered canvases and excludes hidden descendants", () => {
+    const visible = domNode(2, { parentBackendNodeId: 1 });
+    const hidden = domNode(3, { parentBackendNodeId: 4 });
+    const surfaces = discoverRenderedSurfaces(
+      [
+        document([
+          domNode(1, { tag: "main", rect: null, localRect: null }),
+          visible,
+          domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
+          hidden,
+        ]),
+      ],
+      { width: 1280, height: 720 },
+    );
+
+    expect(surfaces).toHaveLength(1);
+    expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
+  });
+
+  it("keeps tiny canvases as individually addressable exact discoveries", () => {
+    const surfaces = discoverRenderedSurfaces(
+      [document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
+      { width: 1280, height: 720 },
+    );
+    expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
+  });
+
+  it("keeps a surface when the canvas only has partial native accessibility content", () => {
+    const axNodes = [
+      {
+        nodeId: "canvas",
+        backendDOMNodeId: 2,
+        role: { type: "role", value: "presentation" },
+        childIds: ["button"],
+        frameId: "main",
+      },
+      {
+        nodeId: "button",
+        backendDOMNodeId: 3,
+        role: { type: "role", value: "button" },
+        frameId: "main",
+      },
+    ] as SemanticAxNode[];
+    expect(
+      discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
+    ).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
+  });
+
+  it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
+    const canvas = domNode(2, {
+      attrs: {
+        id: "must-not-be-a-label",
+        class: "must-not-be-a-label",
+        "data-testid": "must-not-be-a-label",
+      },
+    });
+
+    expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
+      expect.not.objectContaining({
+        label: expect.anything(),
+      }),
+    ]);
+  });
+
+  it("treats blank explicit Canvas names as unnamed", () => {
+    expect(
+      discoverRenderedSurfaces(
+        [document([domNode(2, { attrs: { "aria-label": "   ", title: "\n\t" } })])],
+        { width: 1280, height: 720 },
+      ),
+    ).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
+  });
+
+  it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
+    const owner = domNode(1, { tag: "div", rect: null, localRect: null });
+    const canvas = domNode(2, { parentBackendNodeId: 1 });
+
+    expect(
+      discoverRenderedSurfaces(
+        [document([owner, canvas])],
+        { width: 1280, height: 720 },
+        new Set([1]),
+      ),
+    ).toEqual([]);
+  });
+
+  it("excludes canvases suppressed by a fully transparent ancestor", () => {
+    const owner = domNode(1, {
+      tag: "div",
+      rect: { x: 0, y: 0, w: 900, h: 600 },
+      visuallySuppressed: true,
+    });
+    const canvas = domNode(2, { parentBackendNodeId: 1 });
+
+    expect(
+      discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
+    ).toEqual([]);
+  });
+
+  it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
+    const owner = domNode(1, {
+      tag: "div",
+      rect: { x: 10, y: 20, w: 100, h: 70 },
+      overflowX: "hidden",
+      overflowY: "hidden",
+    });
+    const partial = domNode(2, {
+      parentBackendNodeId: 1,
+      rect: { x: 55, y: 30, w: 140, h: 60 },
+    });
+    const fullyClipped = domNode(3, {
+      parentBackendNodeId: 1,
+      rect: { x: 140, y: 30, w: 80, h: 50 },
+    });
+
+    expect(
+      discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
+        width: 1280,
+        height: 720,
+      }),
+    ).toEqual([
+      expect.objectContaining({
+        backendNodeId: 2,
+        visibleRect: { x: 55, y: 30, w: 55, h: 60 },
+      }),
+    ]);
+  });
+});
+
+describe("rendered surface clustering and projection", () => {
+  it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
+    const body = domNode(1, { tag: "body", rect: null, localRect: null });
+    const surfaces = discoverRenderedSurfaces(
+      [
+        document([
+          body,
+          domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
+          domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
+        ]),
+      ],
+      { width: 1280, height: 720 },
+    );
+    const groups = clusterRenderedSurfaces(surfaces);
+    const scene: VomScene = {
+      viewport: { width: 1280, height: 720 },
+      nodes: [
+        {
+          id: 10,
+          parentId: null,
+          backendNodeId: 1,
+          frameId: "main",
+          tag: "body",
+          role: "RootWebArea",
+          rect: null,
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    };
+
+    expect(groups).toHaveLength(1);
+    expect(groups[0]?.members).toHaveLength(2);
+    expect(groups[0]?.label).toBeUndefined();
+    expect(
+      projectRenderedSurfaces(
+        scene,
+        [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
+        groups,
+      ),
+    ).toMatchObject({
+      visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
+    });
+  });
+
+  it("does not merge a contained canvas with a larger independent canvas", () => {
+    const groups = clusterRenderedSurfaces([
+      ...discoverRenderedSurfaces(
+        [
+          document([
+            domNode(1, { tag: "main", rect: null, localRect: null }),
+            domNode(2, { parentBackendNodeId: 1 }),
+            domNode(3, {
+              parentBackendNodeId: 1,
+              rect: { x: 30, y: 40, w: 200, h: 100 },
+            }),
+          ]),
+        ],
+        { width: 1280, height: 720 },
+      ),
+    ]);
+
+    expect(groups).toHaveLength(2);
+  });
+
+  it("does not merge coincident canvases owned by different containers", () => {
+    const groups = clusterRenderedSurfaces(
+      discoverRenderedSurfaces(
+        [
+          document([
+            domNode(1, { tag: "section", rect: null, localRect: null }),
+            domNode(2, { tag: "section", rect: null, localRect: null }),
+            domNode(3, { parentBackendNodeId: 1 }),
+            domNode(4, { parentBackendNodeId: 2 }),
+          ]),
+        ],
+        { width: 1280, height: 720 },
+      ),
+    );
+
+    expect(groups).toHaveLength(2);
+  });
+});

apps/extension/src/tools/vom/capture.ts

@@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
 import type { CdpRunner } from "../shared";
 import { clearHover, waitForHover } from "./hover-perception";
 
-const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
+const REQUESTED_STYLES = [
+  "position",
+  "pointer-events",
+  "cursor",
+  "visibility",
+  "opacity",
+  "overflow-x",
+  "overflow-y",
+] as const;
 const STYLE_COL = Object.fromEntries(
   REQUESTED_STYLES.map((name, index) => [name, index]),
 ) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -39,6 +47,10 @@ export interface CapturedNode {
    * `textContent`: the live parser always sets it, hand-built fixtures may not.
    */
   cursor?: string;
+  overflowX?: string;
+  overflowY?: string;
+  /** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
+  visuallySuppressed?: boolean;
   /**
    * Whether the live DOM snapshot provides a painted, non-hidden box for this
    * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
@@ -796,6 +808,8 @@ function parseDocumentNodes(
   const cursorCol = STYLE_COL.cursor;
   const visibilityCol = STYLE_COL.visibility;
   const opacityCol = STYLE_COL.opacity;
+  const overflowXCol = STYLE_COL["overflow-x"];
+  const overflowYCol = STYLE_COL["overflow-y"];
   const inputValues = sparseIndexMap(dn.inputValue);
   const textValues = sparseIndexMap(dn.textValue);
   const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -821,6 +835,7 @@ function parseDocumentNodes(
 
   // Build CapturedNodes.
   const nodes: CapturedNode[] = [];
+  const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
   for (let n = 0; n < count; n++) {
     if (excluded[n]) continue;
     const backendNodeId = dn.backendNodeId[n];
@@ -840,7 +855,11 @@ function parseDocumentNodes(
     let position = "static";
     let pointerEvents = "auto";
     let cursor = "auto";
+    let overflowX = "visible";
+    let overflowY = "visible";
     let rendered = false;
+    let visuallySuppressed =
+      parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
     const li = layoutByNode.get(n);
     if (li !== undefined) {
       const b = dl?.bounds?.[li];
@@ -863,12 +882,19 @@ function parseDocumentNodes(
       cursor = str(strings, styleRow[cursorCol]) || "auto";
       const visibility = str(strings, styleRow[visibilityCol]) || "visible";
       const opacity = str(strings, styleRow[opacityCol]) || "1";
+      overflowX = str(strings, styleRow[overflowXCol]) || "visible";
+      overflowY = str(strings, styleRow[overflowYCol]) || "visible";
+      visuallySuppressed ||=
+        visibility === "hidden" ||
+        visibility === "collapse" ||
+        (Number.parseFloat(opacity) || 0) <= 0;
       rendered =
         localRect !== null &&
         visibility !== "hidden" &&
         visibility !== "collapse" &&
         (Number.parseFloat(opacity) || 0) > 0;
     }
+    visuallySuppressedByNodeIndex[n] = visuallySuppressed;
 
     // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
     // geometry and are never queried by callers.
@@ -903,6 +929,9 @@ function parseDocumentNodes(
       position,
       pointerEvents,
       cursor,
+      overflowX,
+      overflowY,
+      visuallySuppressed,
       rendered,
       textContent,
       ...(formValue !== undefined ? { formValue } : {}),

apps/extension/src/tools/vom/rendered-surfaces/cluster.ts

@@ -0,0 +1,65 @@
+import type { Rect } from "@browser-skill/vom";
+import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
+
+const STACK_IOU_RATIO = 0.9;
+const STACK_SIZE_RATIO = 0.9;
+
+function area(rect: Rect): number {
+  return Math.max(0, rect.w) * Math.max(0, rect.h);
+}
+
+function intersectionArea(a: Rect, b: Rect): number {
+  const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
+  const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
+  return width * height;
+}
+
+function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
+  if (a.frameId !== b.frameId) return false;
+  if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
+  const aArea = area(a.visibleRect);
+  const bArea = area(b.visibleRect);
+  const intersection = intersectionArea(a.visibleRect, b.visibleRect);
+  const union = aArea + bArea - intersection;
+  const widthRatio =
+    Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
+  const heightRatio =
+    Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
+  return (
+    union > 0 &&
+    intersection / union >= STACK_IOU_RATIO &&
+    widthRatio >= STACK_SIZE_RATIO &&
+    heightRatio >= STACK_SIZE_RATIO
+  );
+}
+
+function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
+  const representative = [...members].sort(
+    (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
+  )[0];
+  return {
+    frameId: representative.frameId,
+    members,
+    representative,
+    label: members.map((surface) => surface.label).find((label) => label !== undefined),
+  };
+}
+
+export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
+  const groups: RenderedSurface[][] = [];
+  for (const surface of surfaces) {
+    const matching = groups.find((group) =>
+      group.some((member) => sameVisualStack(member, surface)),
+    );
+    if (matching) matching.push(surface);
+    else groups.push([surface]);
+  }
+  return groups
+    .map(toGroup)
+    .sort(
+      (a, b) =>
+        a.frameId.localeCompare(b.frameId) ||
+        a.representative.visibleRect.y - b.representative.visibleRect.y ||
+        a.representative.visibleRect.x - b.representative.visibleRect.x,
+    );
+}

apps/extension/src/tools/vom/rendered-surfaces/discovery.ts

@@ -0,0 +1,113 @@
+import type { Rect, Viewport } from "@browser-skill/vom";
+import type { CapturedNode } from "../capture";
+import type { CapturedFrameDocument } from "../frame-capture";
+import type { SemanticAxNode } from "../semantic-graph";
+import type { RenderedSurface } from "./types";
+
+function clean(value: string | undefined): string | undefined {
+  const normalized = value?.replace(/\s+/g, " ").trim();
+  return normalized || undefined;
+}
+
+function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
+  return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
+}
+
+function intersectRect(a: Rect, b: Rect): Rect | null {
+  const x = Math.max(a.x, b.x);
+  const y = Math.max(a.y, b.y);
+  const right = Math.min(a.x + a.w, b.x + b.w);
+  const bottom = Math.min(a.y + a.h, b.y + b.h);
+  if (right <= x || bottom <= y) return null;
+  return { x, y, w: right - x, h: bottom - y };
+}
+
+const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
+
+function clipToOverflowAncestors(
+  node: CapturedNode,
+  rect: Rect,
+  nodesByBackend: ReadonlyMap<number, CapturedNode>,
+): Rect | null {
+  let clipped: Rect | null = rect;
+  let parentBackendNodeId = node.parentBackendNodeId;
+  let guard = 0;
+  while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
+    const parent = nodesByBackend.get(parentBackendNodeId);
+    if (!parent) break;
+    if (parent.rect) {
+      const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
+      const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
+      if (clipX || clipY) {
+        const clipRect = {
+          x: clipX ? parent.rect.x : clipped.x,
+          y: clipY ? parent.rect.y : clipped.y,
+          w: clipX ? parent.rect.w : clipped.w,
+          h: clipY ? parent.rect.h : clipped.h,
+        };
+        clipped = intersectRect(clipped, clipRect);
+      }
+    }
+    parentBackendNodeId = parent.parentBackendNodeId;
+    guard += 1;
+  }
+  return clipped;
+}
+
+function excludedByDomPolicy(
+  node: CapturedNode,
+  nodesByBackend: Map<number, CapturedNode>,
+  excludedBackendNodeIds: ReadonlySet<number>,
+): boolean {
+  let current: CapturedNode | undefined = node;
+  let guard = 0;
+  while (current && guard <= nodesByBackend.size) {
+    if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
+    if (current.visuallySuppressed === true) return true;
+    const attrs = current.attrs;
+    if (
+      Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
+      Object.prototype.hasOwnProperty.call(attrs, "inert") ||
+      (attrs["aria-hidden"] ?? "").toLowerCase() === "true"
+    ) {
+      return true;
+    }
+    current =
+      current.parentBackendNodeId === null
+        ? undefined
+        : nodesByBackend.get(current.parentBackendNodeId);
+    guard += 1;
+  }
+  return false;
+}
+
+export function discoverRenderedSurfaces(
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+  viewport: Viewport,
+  excludedBackendNodeIds: ReadonlySet<number> = new Set(),
+): RenderedSurface[] {
+  const surfaces: RenderedSurface[] = [];
+  for (const document of documents) {
+    const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
+    for (const node of document.domNodes) {
+      if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
+      if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
+      const viewportRect = intersectViewport(node.rect, viewport);
+      const visibleRect = viewportRect
+        ? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
+        : null;
+      if (!visibleRect) continue;
+      const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
+      surfaces.push({
+        renderingKind: "canvas",
+        frameId: document.frameId,
+        backendNodeId: node.backendNodeId,
+        parentBackendNodeId: node.parentBackendNodeId,
+        visibleRect,
+        paintOrder: node.paintOrder,
+        ...(label ? { label } : {}),
+      });
+    }
+  }
+  return surfaces;
+}

apps/extension/src/tools/vom/rendered-surfaces/index.ts

@@ -0,0 +1,4 @@
+export { clusterRenderedSurfaces } from "./cluster";
+export { discoverRenderedSurfaces } from "./discovery";
+export { projectRenderedSurfaces } from "./project";
+export type { RenderedSurface, RenderedSurfaceGroup } from "./types";

apps/extension/src/tools/vom/rendered-surfaces/project.ts

@@ -0,0 +1,100 @@
+import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
+import type { CapturedFrameDocument } from "../frame-capture";
+import type { SemanticAxNode } from "../semantic-graph";
+import type { RenderedSurfaceGroup } from "./types";
+
+function frameBackendKey(frameId: string, backendNodeId: number): string {
+  return `${frameId}\u0000${backendNodeId}`;
+}
+
+interface ProjectionIndex {
+  sceneByBackend: ReadonlyMap<string, number>;
+  documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
+  domByFrame: ReadonlyMap<
+    string,
+    ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
+  >;
+  rootSceneIdByFrame: ReadonlyMap<string, number>;
+}
+
+function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
+  const document = index.documentByFrame.get(group.frameId);
+  const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
+  let parentBackendNodeId = group.representative.parentBackendNodeId;
+  let guard = 0;
+  while (parentBackendNodeId !== null && guard <= domByBackend.size) {
+    const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
+    if (sceneId !== undefined) return sceneId;
+    parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
+    guard += 1;
+  }
+
+  if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
+    const owner = index.sceneByBackend.get(
+      frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
+    );
+    if (owner !== undefined) return owner;
+  }
+
+  return index.rootSceneIdByFrame.get(group.frameId) ?? null;
+}
+
+function buildProjectionIndex(
+  scene: VomScene,
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+): ProjectionIndex {
+  const rootSceneIdByFrame = new Map<string, number>();
+  for (const node of scene.nodes) {
+    if (
+      node.frameId &&
+      !rootSceneIdByFrame.has(node.frameId) &&
+      ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
+    ) {
+      rootSceneIdByFrame.set(node.frameId, node.id);
+    }
+  }
+  return {
+    sceneByBackend: new Map(
+      scene.nodes.flatMap((node) =>
+        node.backendNodeId === undefined
+          ? []
+          : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
+      ),
+    ),
+    documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
+    domByFrame: new Map(
+      documents.map((document) => [
+        document.frameId,
+        new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
+      ]),
+    ),
+    rootSceneIdByFrame,
+  };
+}
+
+export function projectRenderedSurfaces(
+  scene: VomScene,
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+  groups: RenderedSurfaceGroup[],
+): VomScene {
+  const index = buildProjectionIndex(scene, documents);
+  const visualSurfaces: VomVisualSurface[] = [];
+
+  for (const group of groups) {
+    const parentId = nearestSceneParent(index, group);
+    visualSurfaces.push({
+      parentId,
+      backendNodeId: group.representative.backendNodeId,
+      frameId: group.frameId,
+      renderingKind: "canvas",
+      visibleRect: group.representative.visibleRect,
+      ...(group.label ? { label: group.label } : {}),
+      memberCount: group.members.length,
+    });
+  }
+
+  return {
+    ...scene,
+    ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
+  };
+}

apps/extension/src/tools/vom/rendered-surfaces/types.ts

@@ -0,0 +1,18 @@
+import type { Rect } from "@browser-skill/vom";
+
+export interface RenderedSurface {
+  renderingKind: "canvas";
+  frameId: string;
+  backendNodeId: number;
+  parentBackendNodeId: number | null;
+  visibleRect: Rect;
+  paintOrder: number;
+  label?: string;
+}
+
+export interface RenderedSurfaceGroup {
+  frameId: string;
+  members: RenderedSurface[];
+  representative: RenderedSurface;
+  label?: string;
+}

apps/extension/src/transport/types.ts

@@ -23,6 +23,7 @@ export type ErrorCode =
 export type RpcErrorReason =
   | "agent_window_scope"
   | "element_not_visible"
+  | "ref_capability_denied"
   | "ref_not_found"
   | "selector_not_found"
   | "target_not_fillable"

apps/extension/test-fixtures/rendered-surfaces/README.md

@@ -0,0 +1,105 @@
+# Rendered Surface browser fixtures
+
+These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
+
+## Serve
+
+From the repository root:
+
+```bash
+python3 -m http.server 4173 --bind 127.0.0.1 \
+  --directory apps/extension/test-fixtures/rendered-surfaces
+```
+
+Use an Agent Window large enough to keep the primary matrix visible:
+
+```bash
+bsk session start --width 1440 --height 1100
+bsk navigate http://127.0.0.1:4173/index.html --session <id>
+bsk observe --session <id>
+```
+
+Always stop the session after the run.
+
+## Pages
+
+### `index.html`
+
+Expected individual Surface labels:
+
+- `fixture-basic-large`
+- `fixture-tiny-8px`
+- `fixture-pointer-events-none`
+- `fixture-partial-ax`
+- `fixture-contained-outer`
+- `fixture-contained-inner`
+- `fixture-different-parent-a`
+- `fixture-different-parent-b`
+
+Expected one grouped Surface:
+
+- `fixture-layered-group`, with `layers=2`
+
+Expected absent:
+
+- `fixture-display-none`
+- `fixture-visibility-hidden`
+- `fixture-opacity-zero`
+- `fixture-parent-opacity-zero`
+- `fixture-hidden-attribute`
+- `fixture-inert-parent`
+- `fixture-aria-hidden-parent`
+- `fixture-zero-size`
+- `fixture-offscreen-right`
+- `fixture-fully-overflow-clipped`
+
+`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
+
+### `unlabeled.html`
+
+This page contains eight Canvas elements but must render seven generic Surface markers because
+the two coincident layers form one group.
+
+- Every marker uses the stable fallback label `canvas visual surface`.
+- Exactly one marker has `layers=2`.
+- The two contained canvases remain separate markers.
+- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
+- Blank `aria-label` and `title` values behave as no label.
+- Fallback AX content does not suppress the corresponding Surface.
+
+### `frames.html`
+
+Expected Surface labels:
+
+- `fixture-same-origin-frame`
+- `fixture-cross-origin-frame`
+- `fixture-nested-frame`
+
+The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
+
+### `viewport.html`
+
+Run with a `900x700` Agent Window.
+
+- `fixture-partial-viewport` is partly visible and must be represented.
+- `fixture-below-fold` is absent initially.
+- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
+
+### `dynamic.html`
+
+Use the normal DOM buttons and re-observe after each action:
+
+- Initial state: `fixture-dynamic` absent.
+- After **Insert canvas**: present.
+- After **Resize to zero**: absent.
+- After **Restore size**: present with a fresh ref.
+- After **Remove canvas**: absent.
+
+## Invariants
+
+- `snapshot` must not emit any of these Surface markers.
+- `observe` must emit only currently visible Canvas surfaces.
+- Existing DOM/AX controls remain readable alongside Surface markers.
+- Missing labels never prevent discovery; naming and discovery are independent.
+- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
+- No fixture URL, id, label, or layout constant may appear in production discovery code.

apps/extension/test-fixtures/rendered-surfaces/dynamic.html

@@ -0,0 +1,46 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface dynamic matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 16px; }
+      .controls { display: flex; gap: 8px; margin-bottom: 16px; }
+      button { padding: 8px 12px; }
+      canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
+    </style>
+  </head>
+  <body>
+    <h1>Dynamic Canvas lifecycle</h1>
+    <div class="controls">
+      <button id="insert" type="button">Insert canvas</button>
+      <button id="zero" type="button">Resize to zero</button>
+      <button id="restore" type="button">Restore size</button>
+      <button id="remove" type="button">Remove canvas</button>
+    </div>
+    <div id="mount"></div>
+    <script>
+      const mount = document.querySelector("#mount");
+      const getCanvas = () => mount.querySelector("canvas");
+      document.querySelector("#insert").addEventListener("click", () => {
+        if (getCanvas()) return;
+        const canvas = document.createElement("canvas");
+        canvas.width = 260;
+        canvas.height = 120;
+        canvas.setAttribute("aria-label", "fixture-dynamic");
+        mount.append(canvas);
+      });
+      document.querySelector("#zero").addEventListener("click", () => {
+        const canvas = getCanvas();
+        if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
+      });
+      document.querySelector("#restore").addEventListener("click", () => {
+        const canvas = getCanvas();
+        if (canvas) canvas.removeAttribute("style");
+      });
+      document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
+    </script>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/frame.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 12px; background: #fdfefe; }
+      canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
+      iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
+    </style>
+  </head>
+  <body>
+    <canvas id="frame-canvas" width="220" height="90"></canvas>
+    <script>
+      const params = new URLSearchParams(location.search);
+      const canvas = document.querySelector("#frame-canvas");
+      if (params.get("canvas") === "0") canvas.remove();
+      else canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
+      if (params.get("nested") === "1") {
+        const iframe = document.createElement("iframe");
+        iframe.title = "nested fixture frame";
+        iframe.src = "frame.html?label=fixture-nested-frame";
+        document.body.append(iframe);
+      }
+    </script>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/frames.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface frame matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 16px; }
+      .frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
+      iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
+    </style>
+  </head>
+  <body>
+    <h1>Frame matrix</h1>
+    <div class="frames">
+      <iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame"></iframe>
+      <iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame"></iframe>
+      <iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
+    </div>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/index.html

@@ -0,0 +1,118 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface discovery matrix</title>
+    <style>
+      * { box-sizing: border-box; }
+      html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
+      body { padding: 16px; background: #f4f6f8; color: #17202a; }
+      h1 { margin: 0 0 12px; font-size: 20px; }
+      .matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
+      .case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
+      .case h2 { margin: 0 0 8px; font-size: 13px; }
+      canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
+      .stack { position: relative; width: 180px; height: 90px; }
+      .stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
+      .stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
+      .contained { position: relative; width: 190px; height: 100px; }
+      .contained > canvas:first-child { width: 190px; height: 100px; }
+      .contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
+      .same-place { position: relative; width: 180px; height: 90px; }
+      .same-place > div { position: absolute; inset: 0; }
+      .same-place canvas { width: 180px; height: 90px; }
+      .same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
+      .clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
+      .partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
+      .fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
+      .tiny { width: 8px; height: 8px; border-width: 1px; }
+    </style>
+  </head>
+  <body>
+    <h1>Rendered Surface discovery matrix</h1>
+    <main class="matrix">
+      <section class="case">
+        <h2>Visible named canvas</h2>
+        <canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Tiny canvas remains addressable</h2>
+        <canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Visible but pointer-events none</h2>
+        <canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Partial AX/fallback content</h2>
+        <canvas aria-label="fixture-partial-ax" width="180" height="80">
+          <button type="button">Fallback button must not suppress the surface</button>
+        </canvas>
+      </section>
+
+      <section class="case">
+        <h2>Two real rendering layers</h2>
+        <div class="stack">
+          <canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
+          <canvas width="180" height="90"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Contained canvases are independent</h2>
+        <div class="contained">
+          <canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
+          <canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Same geometry, different parents</h2>
+        <div class="same-place">
+          <div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
+          <div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Partially overflow-clipped</h2>
+        <div class="clip-box">
+          <canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Fully overflow-clipped</h2>
+        <div class="clip-box">
+          <canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>CSS-hidden variants</h2>
+        <canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
+        <canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
+        <canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Hidden ancestor variants</h2>
+        <div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
+        <div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
+        <div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
+        <div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
+      </section>
+
+      <section class="case">
+        <h2>Zero-size and offscreen</h2>
+        <canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
+        <canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
+      </section>
+    </main>
+  </body>
+</html>
+

apps/extension/test-fixtures/rendered-surfaces/unlabeled.html

@@ -0,0 +1,68 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Unlabeled Rendered Surface matrix</title>
+    <style>
+      * { box-sizing: border-box; }
+      html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
+      body { padding: 16px; background: #f4f6f8; color: #17202a; }
+      h1 { margin: 0 0 12px; font-size: 20px; }
+      .matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
+      .case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
+      .case h2 { margin: 0 0 8px; font-size: 13px; }
+      .case p { margin: 0 0 8px; }
+      canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
+      .stack { position: relative; width: 180px; height: 90px; }
+      .stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
+      .stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
+      .contained { position: relative; width: 190px; height: 100px; }
+      .contained > canvas:first-child { width: 190px; height: 100px; }
+      .contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
+    </style>
+  </head>
+  <body>
+    <h1>Unlabeled Rendered Surface matrix</h1>
+    <main class="matrix">
+      <section class="case">
+        <h2>Only engineering attributes</h2>
+        <canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Blank explicit names</h2>
+        <canvas aria-label="   " title="   " width="180" height="80"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Nearby prose is not a Canvas name</h2>
+        <p>must-not-be-inferred-as-a-surface-label</p>
+        <canvas width="180" height="80"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Fallback AX content does not name or suppress the Canvas</h2>
+        <canvas width="180" height="80">
+          <button type="button">Fallback control remains separate</button>
+        </canvas>
+      </section>
+
+      <section class="case">
+        <h2>Unnamed rendering layers still cluster</h2>
+        <div class="stack">
+          <canvas width="180" height="90"></canvas>
+          <canvas width="180" height="90"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Unnamed contained canvases stay independent</h2>
+        <div class="contained">
+          <canvas width="190" height="100"></canvas>
+          <canvas width="70" height="40"></canvas>
+        </div>
+      </section>
+    </main>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/viewport.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface viewport matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { min-height: 1800px; padding: 16px; }
+      canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
+      #partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
+      #below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
+    </style>
+  </head>
+  <body>
+    <h1>Viewport matrix</h1>
+    <p>The orange canvas on the left is intentionally clipped by the viewport.</p>
+    <canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
+    <canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
+  </body>
+</html>
+

crates/bsk-cli/skill/SKILL.md

@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
 
 When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
 
+When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
+
 ## Observation priority
 
 Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:

crates/bsk-cli/src/cli/render_error.rs

@@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode;
 pub mod reason {
     pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
     pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
+    pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
     pub const REF_NOT_FOUND: &str = "ref_not_found";
     pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
     pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
@@ -228,6 +229,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
             ),
             exit_code: base.exit_code,
         },
+        (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
+            summary: "snapshot ref does not support this operation",
+            hint: Some(
+                "visual surface refs are screenshot-only; inspect `bsk screenshot --ref <ref> --session <id>` with image-understanding capability",
+            ),
+            exit_code: base.exit_code,
+        },
         (ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
             summary: "target element is not a <select>",
             hint: Some(
@@ -377,6 +385,18 @@ mod tests {
         assert!(!info.summary.contains("sandbox"));
     }
 
+    #[test]
+    fn ref_capability_denied_explains_visual_surface_refs() {
+        let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
+        let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
+        assert_eq!(info.summary, "snapshot ref does not support this operation");
+        assert!(
+            info.hint.unwrap().contains("screenshot-only"),
+            "expected visual-surface-specific hint"
+        );
+        assert_eq!(info.exit_code, 1);
+    }
+
     #[test]
     fn session_busy_overrides_timeout_copy() {
         let data = serde_json::json!({ "reason": reason::SESSION_BUSY });

packages/dsh-plugin-browserskill/src/tools.ts

@@ -489,7 +489,9 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       ? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
         "carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
       : "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
-        "probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
+        "probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " +
+        "understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " +
+        "alongside them. Read-only: never submits input.";
     register(
       defineTool({
         name,
@@ -752,12 +754,17 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       name: "inspect.screenshot",
       description:
         "Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
+        "This is the supported operation for visual surface refs emitted by inspect.observe. Only use " +
+        "it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
+        "or image-reading capability, tell the user to switch to a model with image understanding; do " +
+        "not guess content or coordinates. " +
         "Returns the image itself when the deployment supports image input, otherwise a file path.",
       parameters: {
         session: SESSION_PARAM,
         ref: {
           type: "string",
-          description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
+          description:
+            "Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
         },
       },
       output: {

packages/vom/src/__tests__/render.test.ts

@@ -29,6 +29,66 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
 }
 
 describe("renderVom single-layer page", () => {
+  it("renders visual surfaces as screenshot-only refs", () => {
+    const input = scene([
+      node({ id: 1, role: "RootWebArea", frameId: "main" }),
+      node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
+    ]);
+    input.visualSurfaces = [
+      {
+        parentId: 2,
+        backendNodeId: 99,
+        frameId: "child",
+        renderingKind: "canvas",
+        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+        label: "Data grid",
+        memberCount: 2,
+      },
+    ];
+    const out = renderVom(input);
+
+    expect(out.text).toContain(
+      '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
+    );
+    expect(out.refs).toContainEqual(
+      expect.objectContaining({
+        ref: "e1",
+        backendNodeId: 99,
+        kind: "surface",
+        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+        capabilities: ["screenshot"],
+      }),
+    );
+  });
+
+  it("renders an unnamed visual surface with a stable generic label", () => {
+    const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
+    input.visualSurfaces = [
+      {
+        parentId: 1,
+        backendNodeId: 99,
+        frameId: "main",
+        renderingKind: "canvas",
+        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+        memberCount: 1,
+      },
+    ];
+
+    const out = renderVom(input);
+
+    expect(out.text).toContain(
+      '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
+    );
+    expect(out.refs).toContainEqual(
+      expect.objectContaining({
+        ref: "e1",
+        name: "canvas visual surface",
+        kind: "surface",
+        capabilities: ["screenshot"],
+      }),
+    );
+  });
+
   it("does not derive handle context across frame scopes", () => {
     const out = renderVom(
       scene([

packages/vom/src/index.ts

@@ -15,6 +15,8 @@ export type {
   VomNode,
   VomOptions,
   VomRef,
+  VomRefCapability,
   VomResult,
   VomScene,
+  VomVisualSurface,
 } from "./types";

packages/vom/src/render.ts

@@ -9,6 +9,7 @@ import type {
   VomOptions,
   VomResult,
   VomScene,
+  VomVisualSurface,
 } from "./types";
 
 const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -793,6 +794,53 @@ interface RenderState {
   duplicateRefNames: Set<string>;
   surfaceMap: Map<number, CondSurface>;
   scopeMap: Map<number, ActiveScopeBlock>;
+  visualSurfaces: Map<number | null, VomVisualSurface[]>;
+}
+
+function groupedByParent<T extends { parentId: number | null }>(
+  items: T[],
+): Map<number | null, T[]> {
+  const grouped = new Map<number | null, T[]>();
+  for (const item of items) {
+    const siblings = grouped.get(item.parentId) ?? [];
+    siblings.push(item);
+    grouped.set(item.parentId, siblings);
+  }
+  return grouped;
+}
+
+function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
+  if (state.stopped) return;
+  for (const surface of state.visualSurfaces.get(parentId) ?? []) {
+    if (depth > state.maxDepth) {
+      state.truncated = true;
+      continue;
+    }
+    const ref = `e${state.nextRef}`;
+    const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
+    const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
+    const line = `${"  ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
+    const nextTokens = state.tokens + estimateTokens(line);
+    if (nextTokens > state.maxTokens) {
+      state.truncated = true;
+      state.stopped = true;
+      return;
+    }
+    state.lines.push(line);
+    state.tokens = nextTokens;
+    state.refs.push({
+      ref,
+      backendNodeId: surface.backendNodeId,
+      frameId: surface.frameId,
+      visibleRect: surface.visibleRect,
+      role: "surface",
+      name: label,
+      kind: "surface",
+      capabilities: ["screenshot"],
+      line: state.lines.length - 1,
+    });
+    state.nextRef += 1;
+  }
 }
 
 function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
@@ -824,24 +872,34 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
   }
 }
 
+type RenderTask =
+  | { kind: "node"; node: VomNode; depth: number }
+  | { kind: "visual"; parentId: number | null; depth: number };
+
 function pushRenderChildren(
-  stack: Array<{ node: VomNode; depth: number }>,
+  stack: RenderTask[],
   children: readonly VomNode[],
   depth: number,
 ): void {
   for (let index = children.length - 1; index >= 0; index -= 1) {
-    stack.push({ node: children[index], depth });
+    stack.push({ kind: "node", node: children[index], depth });
   }
 }
 
 function renderTree(children: Map<number | null, VomNode[]>, state: RenderState): void {
-  const stack: Array<{ node: VomNode; depth: number }> = [];
+  const stack: RenderTask[] = [{ kind: "visual", parentId: null, depth: 1 }];
   pushRenderChildren(stack, children.get(null) ?? [], 1);
 
   while (stack.length > 0 && !state.stopped) {
-    const { node, depth } = stack.pop() as { node: VomNode; depth: number };
+    const task = stack.pop() as RenderTask;
+    if (task.kind === "visual") {
+      emitVisualEntries(task.parentId, task.depth, state);
+      continue;
+    }
+    const { node, depth } = task;
 
     if (!shouldRender(node)) {
+      stack.push({ kind: "visual", parentId: node.id, depth });
       pushRenderChildren(stack, children.get(node.id) ?? [], depth);
       continue;
     }
@@ -873,6 +931,8 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
         ...(node.role ? { role: node.role } : {}),
         ...(name ? { name } : {}),
         ...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
+        kind: "dom",
+        capabilities: ["interact", "screenshot"],
         line: state.lines.length - 1,
       });
       state.nextRef += 1;
@@ -882,8 +942,10 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
     if (scope) emitActiveScopeBlock(scope, depth + 1, state);
 
     if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
+      emitVisualEntries(node.id, depth + 1, state);
       continue;
     }
+    stack.push({ kind: "visual", parentId: node.id, depth: depth + 1 });
     pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
   }
 }
@@ -894,6 +956,7 @@ function renderNodes(
   initialLines: string[],
   surfaces: CondSurface[] = [],
   activeScopeBlocks: ActiveScopeBlock[] = [],
+  visualSurfaces: VomVisualSurface[] = [],
 ): RenderState {
   const children = buildChildren(nodes);
   const state: RenderState = {
@@ -913,6 +976,7 @@ function renderNodes(
     duplicateRefNames: duplicateReferenceNames(nodes),
     surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
     scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
+    visualSurfaces: groupedByParent(visualSurfaces),
   };
 
   renderTree(children, state);
@@ -1124,7 +1188,16 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
     "@layers 2 focus=L1",
     `L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
   ];
-  const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
+  const visibleSurface = (parentId: number | null): boolean =>
+    parentId === null || included.has(parentId);
+  const state = renderNodes(
+    visibleNodes,
+    options,
+    header,
+    scene.surfaces,
+    scene.activeScopeBlocks,
+    scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
+  );
   state.lines.push(renderPageOcclusionLine(hiddenCount));
 
   return {
@@ -1152,6 +1225,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
     ],
     scene.surfaces,
     scene.activeScopeBlocks,
+    scene.visualSurfaces,
   );
 
   return {

packages/vom/src/types.ts

@@ -61,6 +61,8 @@ export interface VomScene {
   rootFrameId?: string;
   surfaces?: CondSurface[];
   activeScopeBlocks?: ActiveScopeBlock[];
+  /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
+  visualSurfaces?: VomVisualSurface[];
 }
 
 export interface VomOptions {
@@ -94,6 +96,22 @@ export interface VomRef {
   ref: string;
   backendNodeId: number;
   frameId?: string;
+  /** Observation-time top-viewport crop for screenshot-only surfaces. */
+  visibleRect?: Rect;
+  kind?: "dom" | "surface";
+  capabilities?: VomRefCapability[];
+}
+
+export type VomRefCapability = "interact" | "screenshot";
+
+export interface VomVisualSurface {
+  parentId: number | null;
+  backendNodeId: number;
+  frameId: string;
+  renderingKind: "canvas";
+  visibleRect: Rect;
+  label?: string;
+  memberCount: number;
 }
 
 export interface RenderedRef extends VomRef {

skill/SKILL.md

@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
 
 When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
 
+When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
+
 ## Observation priority
 
 Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: