feat(vom): vom can distinguish important canvas area

apps/extension/src/tools/observation.ts

@@ -62,6 +62,11 @@ import {
   type CaptureVomObservationResult,
   projectRecordSafeObservation,
 } from "./vom/record-safe-observation";
+import {
+  clusterRenderedSurfaces,
+  discoverRenderedSurfaces,
+  projectRenderedSurfaces,
+} from "./vom/rendered-surfaces";
 import {
   buildSemanticGraph,
   buildSemanticVomScene,
@@ -93,6 +98,9 @@ export const normaliseRef = sharedNormaliseRef;
 // screenshot — `tool.screenshot`
 // ---------------------------------------------------------------------------
 
+const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
+const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
+
 /**
  * Strip the `data:image/...;base64,` prefix from a Chrome
  * `captureVisibleTab` dataURL and return the raw base64 payload.
@@ -185,6 +193,7 @@ async function captureElementScreenshot(
   target: CdpTarget,
   backendNodeId: number,
   frameId?: string,
+  visualSurface = false,
   signal?: AbortSignal,
 ): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
   if (signal?.aborted) return cancelled("screenshot");
@@ -192,11 +201,19 @@ async function captureElementScreenshot(
     cdp,
     tabId,
     { target, backendNodeId, ...(frameId ? { frameId } : {}) },
-    { scrollIntoView: true },
+    { scrollIntoView: !visualSurface },
   );
   if (isRpcError(geometry)) return geometry;
   if (signal?.aborted) return cancelled("screenshot");
   const rect = geometry.topBounds;
+  const scale = visualSurface
+    ? Math.min(
+        1,
+        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
+        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
+        Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
+      )
+    : 1;
 
   try {
     const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
@@ -206,7 +223,7 @@ async function captureElementScreenshot(
         y: rect.y,
         width: rect.width,
         height: rect.height,
-        scale: 1,
+        scale,
       },
     });
     if (signal?.aborted) return cancelled("screenshot");
@@ -313,7 +330,7 @@ export async function handleScreenshot(
     if (!deps.cdp) {
       return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
     }
-    const node = resolveSnapshotRef(ctx, ref, target.tabId);
+    const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
     if (isRpcError(node)) return node;
     if (signal?.aborted) return cancelled("screenshot");
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -333,6 +350,7 @@ export async function handleScreenshot(
           nodeTarget,
           node.backendNodeId,
           node.frameId,
+          node.kind === "surface",
           signal,
         ),
       deps.sendToTab,
@@ -1083,6 +1101,7 @@ async function runHoverProbes(
 
 export interface CaptureVomObservationOptions extends VomOptions {
   conditionalSurfaceProbe?: boolean;
+  renderedSurfaceDiscovery?: boolean;
   hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
   signal?: AbortSignal;
 }
@@ -1131,7 +1150,14 @@ export async function captureVomObservation(
     captured,
     hoverProbes.surfaceProbes,
   );
-  const rendered = renderVom(decoratedScene, {
+  const observedScene = options.renderedSurfaceDiscovery
+    ? projectRenderedSurfaces(
+        decoratedScene,
+        normalizedDocuments,
+        clusterRenderedSurfaces(discoverRenderedSurfaces(normalizedDocuments, captured.viewport)),
+      )
+    : decoratedScene;
+  const rendered = renderVom(observedScene, {
     maxDepth: options.maxDepth,
     maxTokens: options.maxTokens,
     redactValues: options.redactValues,
@@ -1186,6 +1212,7 @@ async function handleVomObservation(
       maxTokens: params.max_tokens,
       activeRegionPolicy: true,
       conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
+      renderedSurfaceDiscovery: toolName === "observe",
       hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
       signal,
     });
@@ -1203,6 +1230,8 @@ async function handleVomObservation(
             tabId: target.tabId,
             ...(ref.frameId ? { frameId: ref.frameId } : {}),
             ...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
+            ...(ref.kind ? { kind: ref.kind } : {}),
+            ...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
           },
         ] as const;
       }),