feat(vom): vom can distinguish important canvas area

packages/dsh-plugin-browserskill/src/tools.ts

@@ -489,7 +489,8 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       ? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
         "carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
       : "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
-        "probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
+        "probes) with @eN refs for browser_interact actions. Visual surface refs are screenshot-only; " +
+        "DOM refs support interaction. Read-only: never submits input.";
     register(
       defineTool({
         name,
@@ -754,12 +755,14 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       name: "inspect.screenshot",
       description:
         "Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
-        "Returns the image itself when the deployment supports image input, otherwise a file path.",
+        "This is the supported operation for visual surface refs emitted by browser_observe. Returns " +
+        "the image itself when the deployment supports image input, otherwise a file path.",
       parameters: {
         session: SESSION_PARAM,
         ref: {
           type: "string",
-          description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
+          description:
+            "Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
         },
       },
       output: {

packages/vom/src/__tests__/render.test.ts

@@ -29,6 +29,40 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
 }
 
 describe("renderVom single-layer page", () => {
+  it("renders visual surfaces as screenshot-only refs and low-confidence counts as summaries", () => {
+    const input = scene([
+      node({ id: 1, role: "RootWebArea", frameId: "main" }),
+      node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
+    ]);
+    input.visualSurfaces = [
+      {
+        parentId: 2,
+        backendNodeId: 99,
+        frameId: "child",
+        renderingKind: "canvas",
+        rect: { x: 10, y: 20, w: 800, h: 500 },
+        label: "Data grid",
+        memberCount: 2,
+      },
+    ];
+    input.visualSurfaceSummaries = [{ parentId: 2, frameId: "child", count: 3 }];
+
+    const out = renderVom(input);
+
+    expect(out.text).toContain(
+      '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]',
+    );
+    expect(out.text).toContain("[3 additional visual surfaces are not represented]");
+    expect(out.refs).toContainEqual(
+      expect.objectContaining({
+        ref: "e1",
+        backendNodeId: 99,
+        kind: "surface",
+        capabilities: ["screenshot"],
+      }),
+    );
+  });
+
   it("does not derive handle context across frame scopes", () => {
     const out = renderVom(
       scene([

packages/vom/src/index.ts

@@ -15,6 +15,9 @@ export type {
   VomNode,
   VomOptions,
   VomRef,
+  VomRefCapability,
   VomResult,
   VomScene,
+  VomVisualSurface,
+  VomVisualSurfaceSummary,
 } from "./types";

packages/vom/src/render.ts

@@ -9,6 +9,8 @@ import type {
   VomOptions,
   VomResult,
   VomScene,
+  VomVisualSurface,
+  VomVisualSurfaceSummary,
 } from "./types";
 
 const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -793,6 +795,70 @@ interface RenderState {
   duplicateRefNames: Set<string>;
   surfaceMap: Map<number, CondSurface>;
   scopeMap: Map<number, ActiveScopeBlock>;
+  visualSurfaces: Map<number | null, VomVisualSurface[]>;
+  visualSurfaceSummaries: Map<number | null, VomVisualSurfaceSummary[]>;
+}
+
+function groupedByParent<T extends { parentId: number | null }>(
+  items: T[],
+): Map<number | null, T[]> {
+  const grouped = new Map<number | null, T[]>();
+  for (const item of items) {
+    const siblings = grouped.get(item.parentId) ?? [];
+    siblings.push(item);
+    grouped.set(item.parentId, siblings);
+  }
+  return grouped;
+}
+
+function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
+  if (state.stopped) return;
+  for (const surface of state.visualSurfaces.get(parentId) ?? []) {
+    if (depth > state.maxDepth) {
+      state.truncated = true;
+      continue;
+    }
+    const ref = `e${state.nextRef}`;
+    const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
+    const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
+    const line = `${"  ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; use: bsk screenshot --ref @${ref}]`;
+    const nextTokens = state.tokens + estimateTokens(line);
+    if (nextTokens > state.maxTokens) {
+      state.truncated = true;
+      state.stopped = true;
+      return;
+    }
+    state.lines.push(line);
+    state.tokens = nextTokens;
+    state.refs.push({
+      ref,
+      backendNodeId: surface.backendNodeId,
+      frameId: surface.frameId,
+      role: "surface",
+      name: label,
+      kind: "surface",
+      capabilities: ["screenshot"],
+      line: state.lines.length - 1,
+    });
+    state.nextRef += 1;
+  }
+
+  for (const summary of state.visualSurfaceSummaries.get(parentId) ?? []) {
+    if (depth > state.maxDepth) {
+      state.truncated = true;
+      continue;
+    }
+    const noun = summary.count === 1 ? "surface is" : "surfaces are";
+    const line = `${"  ".repeat(depth)}[${summary.count} additional visual ${noun} not represented]`;
+    const nextTokens = state.tokens + estimateTokens(line);
+    if (nextTokens > state.maxTokens) {
+      state.truncated = true;
+      state.stopped = true;
+      return;
+    }
+    state.lines.push(line);
+    state.tokens = nextTokens;
+  }
 }
 
 function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
@@ -873,6 +939,8 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
         ...(node.role ? { role: node.role } : {}),
         ...(name ? { name } : {}),
         ...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
+        kind: "dom",
+        capabilities: ["interact", "screenshot"],
         line: state.lines.length - 1,
       });
       state.nextRef += 1;
@@ -882,10 +950,12 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
     if (scope) emitActiveScopeBlock(scope, depth + 1, state);
 
     if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
+      emitVisualEntries(node.id, depth + 1, state);
       continue;
     }
     pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
   }
+  emitVisualEntries(parentId, depth, state);
 }
 
 function renderNodes(
@@ -894,6 +964,8 @@ function renderNodes(
   initialLines: string[],
   surfaces: CondSurface[] = [],
   activeScopeBlocks: ActiveScopeBlock[] = [],
+  visualSurfaces: VomVisualSurface[] = [],
+  visualSurfaceSummaries: VomVisualSurfaceSummary[] = [],
 ): RenderState {
   const children = buildChildren(nodes);
   const state: RenderState = {
@@ -913,6 +985,8 @@ function renderNodes(
     duplicateRefNames: duplicateReferenceNames(nodes),
     surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
     scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
+    visualSurfaces: groupedByParent(visualSurfaces),
+    visualSurfaceSummaries: groupedByParent(visualSurfaceSummaries),
   };
 
   renderTree(children, state);
@@ -1124,7 +1198,17 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
     "@layers 2 focus=L1",
     `L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
   ];
-  const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
+  const visibleSurface = (parentId: number | null): boolean =>
+    parentId === null || included.has(parentId);
+  const state = renderNodes(
+    visibleNodes,
+    options,
+    header,
+    scene.surfaces,
+    scene.activeScopeBlocks,
+    scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
+    scene.visualSurfaceSummaries?.filter((summary) => visibleSurface(summary.parentId)),
+  );
   state.lines.push(renderPageOcclusionLine(hiddenCount));
 
   return {
@@ -1152,6 +1236,8 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
     ],
     scene.surfaces,
     scene.activeScopeBlocks,
+    scene.visualSurfaces,
+    scene.visualSurfaceSummaries,
   );
 
   return {

packages/vom/src/types.ts

@@ -61,6 +61,9 @@ export interface VomScene {
   rootFrameId?: string;
   surfaces?: CondSurface[];
   activeScopeBlocks?: ActiveScopeBlock[];
+  /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
+  visualSurfaces?: VomVisualSurface[];
+  visualSurfaceSummaries?: VomVisualSurfaceSummary[];
 }
 
 export interface VomOptions {
@@ -94,6 +97,27 @@ export interface VomRef {
   ref: string;
   backendNodeId: number;
   frameId?: string;
+  kind?: "dom" | "surface";
+  capabilities?: VomRefCapability[];
+}
+
+export type VomRefCapability = "interact" | "screenshot";
+
+export interface VomVisualSurface {
+  parentId: number | null;
+  backendNodeId: number;
+  frameId: string;
+  renderingKind: "canvas";
+  rect: Rect;
+  localRect?: Rect;
+  label?: string;
+  memberCount: number;
+}
+
+export interface VomVisualSurfaceSummary {
+  parentId: number | null;
+  frameId: string;
+  count: number;
 }
 
 export interface RenderedRef extends VomRef {