diff --git a/apps/extension/src/session-manager/__tests__/ref-store.test.ts b/apps/extension/src/session-manager/__tests__/ref-store.test.ts index 1a8d003..9cb540b 100644 --- a/apps/extension/src/session-manager/__tests__/ref-store.test.ts +++ b/apps/extension/src/session-manager/__tests__/ref-store.test.ts @@ -13,6 +13,8 @@ describe("RefStore", () => { backendNodeId: 42, tabId: 7, generation: 0, + kind: "dom", + capabilities: ["interact", "screenshot"], }); expect(s.size()).toBe(1); expect(s.isEmpty()).toBe(false); @@ -48,4 +50,18 @@ describe("RefStore", () => { cdpSessionId: "child-session", }); }); + + it("preserves explicit capabilities for visual surface refs", () => { + const s = new RefStore(); + s.set("e1", 42, { + tabId: 7, + kind: "surface", + capabilities: ["screenshot"], + }); + + expect(s.resolveEntry("e1")).toMatchObject({ + kind: "surface", + capabilities: ["screenshot"], + }); + }); }); diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index 568392f..5f68c8b 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -12,12 +12,16 @@ * inside the `SessionContext`. */ export type BackendNodeId = number; +export type RefCapability = "interact" | "screenshot"; +export type RefTargetKind = "dom" | "surface"; export interface RefEntry { backendNodeId: BackendNodeId; tabId: number | null; frameId?: string; cdpSessionId?: string; + kind: RefTargetKind; + capabilities: RefCapability[]; generation: number; } @@ -28,6 +32,8 @@ export type RefInput = tabId: number; frameId?: string; cdpSessionId?: string; + kind?: RefTargetKind; + capabilities?: RefCapability[]; }; export class RefStore { @@ -70,6 +76,8 @@ export class RefStore { tabId?: number; frameId?: string; cdpSessionId?: string; + kind?: RefTargetKind; + capabilities?: RefCapability[]; } = {}, ): void { this.map.set(normaliseRef(ref), { @@ -77,6 +85,8 @@ export class RefStore { tabId: opts.tabId ?? null, ...(opts.frameId ? { frameId: opts.frameId } : {}), ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}), + kind: opts.kind ?? "dom", + capabilities: opts.capabilities ?? ["interact", "screenshot"], generation: this.generation, }); } @@ -94,6 +104,8 @@ export class RefStore { return { backendNodeId: input, tabId: null, + kind: "dom", + capabilities: ["interact", "screenshot"], generation: this.generation, }; } @@ -102,6 +114,8 @@ export class RefStore { tabId: input.tabId, ...(input.frameId ? { frameId: input.frameId } : {}), ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}), + kind: input.kind ?? "dom", + capabilities: input.capabilities ?? ["interact", "screenshot"], generation: this.generation, }; } diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 7cb1d4d..985192b 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -123,6 +123,29 @@ describe("handleClick", () => { expect(fake.cdp.send).not.toHaveBeenCalled(); }); + it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + kind: "surface", + capabilities: ["screenshot"], + }); + const fake = makeFakeCdp({}); + + const res = await handleClick( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(res).toMatchObject({ + code: "permission_denied", + data: { reason: "ref_capability_denied", required_capability: "interact" }, + }); + expect(fake.cdp.send).not.toHaveBeenCalled(); + }); + it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 655e30c..f972aad 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -300,6 +300,42 @@ describe("handleScreenshot", () => { expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); }); + it("captures a bounded visible crop for a visual surface without scrolling it", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e5", 999, { + tabId: 7, + kind: "surface", + capabilities: ["screenshot"], + }); + const { cdp, sent } = makeFakeCdp({ + "Page.getLayoutMetrics": () => ({ + cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 }, + }), + "DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }), + "Page.captureScreenshot": () => ({ data: TINY_PNG }), + }); + + const res = await handleScreenshot( + sm, + { session_id: "aa11", ref: "@e5", tab_id: 7 }, + makeScreenshotDeps({ + cdp, + get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab), + query: vi.fn(), + captureVisibleTab: vi.fn(), + }), + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false); + const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as { + clip?: { width: number; height: number; scale: number }; + }; + expect(clip.clip).toMatchObject({ width: 4096, height: 4096 }); + expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096))); + }); + it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); @@ -2833,6 +2869,85 @@ describe("handleSnapshot", () => { ); }); + it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + const root: CdpAxNode = { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "Canvas app" }, + backendDOMNodeId: 100, + }; + const strings = ["body", "canvas", "static", "auto"]; + const i = (value: string) => strings.indexOf(value); + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: [root] }; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + } + if (method === "DOMSnapshot.captureSnapshot") { + return { + strings, + documents: [ + { + nodes: { + parentIndex: [-1, 0], + nodeName: [i("body"), i("canvas")], + backendNodeId: [100, 200], + attributes: [[], []], + }, + layout: { + nodeIndex: [0, 1], + styles: [ + [i("static"), i("auto"), i("auto")], + [i("static"), i("auto"), i("auto")], + ], + bounds: [ + [0, 0, 1000, 800], + [20, 30, 800, 500], + ], + paintOrders: [0, 1], + }, + }, + ], + }; + } + throw new Error(`unexpected CDP method ${method}`); + }); + const deps = { + cdp: { + send: send as unknown as ( + tabId: number, + method: string, + params?: object, + ) => Promise, + trackSessionTab: vi.fn(), + }, + tabsApi: { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + conditionalSurfaceProbe: false, + }; + + const observed = await handleObserve(sm, { session_id: "aa11" }, deps); + if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`); + expect(observed.text).toContain('@e1 surface "canvas visual surface"'); + expect(ctx.refStore.resolveEntry("e1")).toMatchObject({ + backendNodeId: 200, + kind: "surface", + capabilities: ["screenshot"], + }); + + const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps); + if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`); + expect(snapshot.text).not.toContain("visual surface"); + expect(snapshot.ref_count).toBe(0); + }); + it("allows passive snapshots of explicit user-window tabs", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); diff --git a/apps/extension/src/tools/__tests__/snapshot-ref.test.ts b/apps/extension/src/tools/__tests__/snapshot-ref.test.ts index 8c9b0ae..60c0d95 100644 --- a/apps/extension/src/tools/__tests__/snapshot-ref.test.ts +++ b/apps/extension/src/tools/__tests__/snapshot-ref.test.ts @@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => { expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({ backendNodeId: 1234, refKey: "e3", + kind: "dom", + capabilities: ["interact", "screenshot"], }); expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({ backendNodeId: 1234, refKey: "e3", + kind: "dom", + capabilities: ["interact", "screenshot"], }); }); @@ -53,6 +57,8 @@ describe("resolveSnapshotRef", () => { tabId: 4, frameId: "child-frame", cdpSessionId: "child-session", + kind: "dom", + capabilities: ["interact", "screenshot"], }); const expected = { @@ -60,6 +66,8 @@ describe("resolveSnapshotRef", () => { refKey: "e3", frameId: "child-frame", cdpSessionId: "child-session", + kind: "dom" as const, + capabilities: ["interact", "screenshot"] as const, }; expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected); expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected); @@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => { expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({ backendNodeId: 1234, refKey: "e3", + kind: "dom", + capabilities: ["interact", "screenshot"], + }); + }); + + it("rejects an operation outside the ref's declared capabilities", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + kind: "surface", + capabilities: ["screenshot"], + }); + + expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({ + code: "permission_denied", + message: "ref @e3 does not support interact", + data: { + reason: "ref_capability_denied", + required_capability: "interact", + kind: "surface", + }, }); }); }); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 9e36c2d..abce158 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -169,7 +169,7 @@ export async function resolveBackendNode( }; } if (hasRef) { - const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId); + const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact"); if (isRpcError(resolved)) return resolved; return { backendNodeId: resolved.backendNodeId, diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 9c944ab..46ec735 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -62,6 +62,11 @@ import { type CaptureVomObservationResult, projectRecordSafeObservation, } from "./vom/record-safe-observation"; +import { + clusterRenderedSurfaces, + discoverRenderedSurfaces, + projectRenderedSurfaces, +} from "./vom/rendered-surfaces"; import { buildSemanticGraph, buildSemanticVomScene, @@ -93,6 +98,9 @@ export const normaliseRef = sharedNormaliseRef; // screenshot — `tool.screenshot` // --------------------------------------------------------------------------- +const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048; +const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000; + /** * Strip the `data:image/...;base64,` prefix from a Chrome * `captureVisibleTab` dataURL and return the raw base64 payload. @@ -185,6 +193,7 @@ async function captureElementScreenshot( target: CdpTarget, backendNodeId: number, frameId?: string, + visualSurface = false, signal?: AbortSignal, ): Promise<{ image_base64: string; width: number; height: number } | RpcError> { if (signal?.aborted) return cancelled("screenshot"); @@ -192,11 +201,19 @@ async function captureElementScreenshot( cdp, tabId, { target, backendNodeId, ...(frameId ? { frameId } : {}) }, - { scrollIntoView: true }, + { scrollIntoView: !visualSurface }, ); if (isRpcError(geometry)) return geometry; if (signal?.aborted) return cancelled("screenshot"); const rect = geometry.topBounds; + const scale = visualSurface + ? Math.min( + 1, + VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width, + VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height, + Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)), + ) + : 1; try { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { @@ -206,7 +223,7 @@ async function captureElementScreenshot( y: rect.y, width: rect.width, height: rect.height, - scale: 1, + scale, }, }); if (signal?.aborted) return cancelled("screenshot"); @@ -313,7 +330,7 @@ export async function handleScreenshot( if (!deps.cdp) { return { code: "cdp_failed", message: "screenshot ref capture requires CDP" }; } - const node = resolveSnapshotRef(ctx, ref, target.tabId); + const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot"); if (isRpcError(node)) return node; if (signal?.aborted) return cancelled("screenshot"); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); @@ -333,6 +350,7 @@ export async function handleScreenshot( nodeTarget, node.backendNodeId, node.frameId, + node.kind === "surface", signal, ), deps.sendToTab, @@ -1083,6 +1101,7 @@ async function runHoverProbes( export interface CaptureVomObservationOptions extends VomOptions { conditionalSurfaceProbe?: boolean; + renderedSurfaceDiscovery?: boolean; hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; signal?: AbortSignal; } @@ -1131,7 +1150,14 @@ export async function captureVomObservation( captured, hoverProbes.surfaceProbes, ); - const rendered = renderVom(decoratedScene, { + const observedScene = options.renderedSurfaceDiscovery + ? projectRenderedSurfaces( + decoratedScene, + normalizedDocuments, + clusterRenderedSurfaces(discoverRenderedSurfaces(normalizedDocuments, captured.viewport)), + ) + : decoratedScene; + const rendered = renderVom(observedScene, { maxDepth: options.maxDepth, maxTokens: options.maxTokens, redactValues: options.redactValues, @@ -1186,6 +1212,7 @@ async function handleVomObservation( maxTokens: params.max_tokens, activeRegionPolicy: true, conditionalSurfaceProbe: effectiveConditionalSurfaceProbe, + renderedSurfaceDiscovery: toolName === "observe", hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay, signal, }); @@ -1203,6 +1230,8 @@ async function handleVomObservation( tabId: target.tabId, ...(ref.frameId ? { frameId: ref.frameId } : {}), ...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}), + ...(ref.kind ? { kind: ref.kind } : {}), + ...(ref.capabilities ? { capabilities: ref.capabilities } : {}), }, ] as const; }), diff --git a/apps/extension/src/tools/snapshot-ref.ts b/apps/extension/src/tools/snapshot-ref.ts index 90f46bc..081b966 100644 --- a/apps/extension/src/tools/snapshot-ref.ts +++ b/apps/extension/src/tools/snapshot-ref.ts @@ -3,7 +3,7 @@ // `ref_not_found` errors for hard-failure tool paths. import type { SessionContext } from "@/session-manager/manager"; -import { normaliseRef } from "@/session-manager/ref-store"; +import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store"; import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; @@ -12,6 +12,8 @@ export interface SnapshotRefLookup { refKey: string; frameId?: string; cdpSessionId?: string; + kind: RefTargetKind; + capabilities: RefCapability[]; } function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) { @@ -37,6 +39,8 @@ export function lookupSnapshotRef( refKey, ...(entry.frameId ? { frameId: entry.frameId } : {}), ...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}), + kind: entry.kind, + capabilities: entry.capabilities, }; } @@ -49,6 +53,7 @@ export function resolveSnapshotRef( ctx: SessionContext, ref: string, tabId: number, + requiredCapability?: RefCapability, ): SnapshotRefLookup | RpcError { const looked = lookupSnapshotRef(ctx, ref, tabId); if (looked === null) { @@ -58,5 +63,13 @@ export function resolveSnapshotRef( `ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`, ); } + if (requiredCapability && !looked.capabilities.includes(requiredCapability)) { + return rpcError( + "permission_denied", + "ref_capability_denied", + `ref ${ref} does not support ${requiredCapability}`, + { ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind }, + ); + } return looked; } diff --git a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts new file mode 100644 index 0000000..40fddd1 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts @@ -0,0 +1,137 @@ +import type { VomScene } from "@browser-skill/vom"; +import { describe, expect, it } from "vitest"; +import type { CapturedNode } from "../capture"; +import type { CapturedFrameDocument } from "../frame-capture"; +import { + clusterRenderedSurfaces, + discoverRenderedSurfaces, + projectRenderedSurfaces, +} from "../rendered-surfaces"; +import type { SemanticAxNode } from "../semantic-graph"; + +function domNode(backendNodeId: number, overrides: Partial = {}): CapturedNode { + return { + backendNodeId, + parentBackendNodeId: null, + frameId: "main", + ownerFrameBackendNodeId: null, + tag: "canvas", + attrs: {}, + rect: { x: 10, y: 20, w: 800, h: 500 }, + localRect: { x: 10, y: 20, w: 800, h: 500 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + rendered: true, + ...overrides, + }; +} + +function document( + domNodes: CapturedNode[], + axNodes: SemanticAxNode[] = [], +): CapturedFrameDocument { + return { + frameId: domNodes[0]?.frameId ?? "main", + contextScopeId: domNodes[0]?.frameId ?? "main", + target: { tabId: 4 }, + domNodes, + axNodes, + }; +} + +describe("rendered surface discovery", () => { + it("finds prominent rendered canvases and excludes hidden descendants", () => { + const visible = domNode(2, { parentBackendNodeId: 1 }); + const hidden = domNode(3, { parentBackendNodeId: 4 }); + const surfaces = discoverRenderedSurfaces( + [ + document([ + domNode(1, { tag: "main", rect: null, localRect: null }), + visible, + domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }), + hidden, + ]), + ], + { width: 1280, height: 720 }, + ); + + expect(surfaces).toHaveLength(1); + expect(surfaces[0]).toMatchObject({ backendNodeId: 2, existenceConfidence: "high" }); + }); + + it("keeps tiny canvases only as low-confidence discoveries", () => { + const surfaces = discoverRenderedSurfaces( + [document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])], + { width: 1280, height: 720 }, + ); + expect(surfaces[0]?.existenceConfidence).toBe("low"); + }); + + it("suppresses a canvas with a verified native accessibility mirror", () => { + const axNodes = [ + { + nodeId: "canvas", + backendDOMNodeId: 2, + role: { type: "role", value: "presentation" }, + childIds: ["button"], + frameId: "main", + }, + { + nodeId: "button", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + frameId: "main", + }, + ] as SemanticAxNode[]; + expect( + discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }), + ).toEqual([]); + }); +}); + +describe("rendered surface clustering and projection", () => { + it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => { + const body = domNode(1, { tag: "body", rect: null, localRect: null }); + const surfaces = discoverRenderedSurfaces( + [ + document([ + body, + domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }), + domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }), + ]), + ], + { width: 1280, height: 720 }, + ); + const groups = clusterRenderedSurfaces(surfaces); + const scene: VomScene = { + viewport: { width: 1280, height: 720 }, + nodes: [ + { + id: 10, + parentId: null, + backendNodeId: 1, + frameId: "main", + tag: "body", + role: "RootWebArea", + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + ], + }; + + expect(groups).toHaveLength(1); + expect(groups[0]?.members).toHaveLength(2); + expect( + projectRenderedSurfaces( + scene, + [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])], + groups, + ), + ).toMatchObject({ + visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })], + }); + }); +}); diff --git a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts new file mode 100644 index 0000000..06f54e1 --- /dev/null +++ b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts @@ -0,0 +1,67 @@ +import type { Rect } from "@browser-skill/vom"; +import type { RenderedSurface, RenderedSurfaceGroup } from "./types"; + +const STACK_OVERLAP_RATIO = 0.9; + +function area(rect: Rect): number { + return Math.max(0, rect.w) * Math.max(0, rect.h); +} + +function intersectionArea(a: Rect, b: Rect): number { + const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x)); + const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y)); + return width * height; +} + +function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean { + if (a.frameId !== b.frameId) return false; + const smallerArea = Math.min(area(a.visibleRect), area(b.visibleRect)); + return ( + smallerArea > 0 && + intersectionArea(a.visibleRect, b.visibleRect) / smallerArea >= STACK_OVERLAP_RATIO + ); +} + +function unionRect(rects: Rect[]): Rect { + const x = Math.min(...rects.map((rect) => rect.x)); + const y = Math.min(...rects.map((rect) => rect.y)); + const right = Math.max(...rects.map((rect) => rect.x + rect.w)); + const bottom = Math.max(...rects.map((rect) => rect.y + rect.h)); + return { x, y, w: right - x, h: bottom - y }; +} + +function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup { + const representative = [...members].sort( + (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder, + )[0]; + return { + frameId: representative.frameId, + members, + representative, + rect: unionRect(members.map((surface) => surface.rect)), + visibleRect: unionRect(members.map((surface) => surface.visibleRect)), + label: members.map((surface) => surface.label).find((label) => label !== undefined), + existenceConfidence: members.some((surface) => surface.existenceConfidence === "high") + ? "high" + : "low", + }; +} + +export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] { + const groups: RenderedSurface[][] = []; + for (const surface of surfaces) { + const matching = groups.find((group) => + group.some((member) => sameVisualStack(member, surface)), + ); + if (matching) matching.push(surface); + else groups.push([surface]); + } + return groups + .map(toGroup) + .sort( + (a, b) => + a.frameId.localeCompare(b.frameId) || + a.visibleRect.y - b.visibleRect.y || + a.visibleRect.x - b.visibleRect.x, + ); +} diff --git a/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts b/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts new file mode 100644 index 0000000..a1be11d --- /dev/null +++ b/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts @@ -0,0 +1,109 @@ +import type { Rect, Viewport } from "@browser-skill/vom"; +import type { CapturedNode } from "../capture"; +import type { CapturedFrameDocument } from "../frame-capture"; +import type { SemanticAxNode } from "../semantic-graph"; +import type { RenderedSurface } from "./types"; + +const MIN_PROMINENT_EDGE = 32; +const MIN_PROMINENT_AREA = 4_096; +const NON_CONTENT_ROLES = new Set(["", "generic", "none", "presentation"]); + +function clean(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + return normalized || undefined; +} + +function intersectViewport(rect: Rect, viewport: Viewport): Rect | null { + const x = Math.max(0, rect.x); + const y = Math.max(0, rect.y); + const right = Math.min(viewport.width, rect.x + rect.w); + const bottom = Math.min(viewport.height, rect.y + rect.h); + if (right <= x || bottom <= y) return null; + return { x, y, w: right - x, h: bottom - y }; +} + +function hiddenByDomPolicy(node: CapturedNode, nodesByBackend: Map): boolean { + let current: CapturedNode | undefined = node; + let guard = 0; + while (current && guard <= nodesByBackend.size) { + const attrs = current.attrs; + if ( + Object.prototype.hasOwnProperty.call(attrs, "hidden") || + Object.prototype.hasOwnProperty.call(attrs, "inert") || + (attrs["aria-hidden"] ?? "").toLowerCase() === "true" + ) { + return true; + } + current = + current.parentBackendNodeId === null + ? undefined + : nodesByBackend.get(current.parentBackendNodeId); + guard += 1; + } + return false; +} + +function hasVerifiedNativeMirror(canvas: CapturedNode, axNodes: SemanticAxNode[]): boolean { + const owner = axNodes.find( + (node) => node.backendDOMNodeId === canvas.backendNodeId && node.ignored !== true, + ); + if (!owner?.childIds?.length) return false; + const byId = new Map(axNodes.map((node) => [node.nodeId, node])); + const queue = [...owner.childIds]; + const seen = new Set(); + while (queue.length > 0) { + const id = queue.shift() as string; + if (seen.has(id)) continue; + seen.add(id); + const node = byId.get(id); + if (!node || node.ignored === true) continue; + const role = String(node.role?.value ?? "").toLowerCase(); + if (!NON_CONTENT_ROLES.has(role)) return true; + queue.push(...(node.childIds ?? [])); + } + return false; +} + +function isProminent(node: CapturedNode, visibleRect: Rect, label: string | undefined): boolean { + const tabindex = Number.parseInt(node.attrs.tabindex ?? "", 10); + const interactionSignal = + node.cursor === "pointer" || (Number.isFinite(tabindex) && tabindex >= 0); + return ( + label !== undefined || + interactionSignal || + (visibleRect.w >= MIN_PROMINENT_EDGE && + visibleRect.h >= MIN_PROMINENT_EDGE && + visibleRect.w * visibleRect.h >= MIN_PROMINENT_AREA) + ); +} + +export function discoverRenderedSurfaces( + documents: CapturedFrameDocument[], + viewport: Viewport, +): RenderedSurface[] { + const surfaces: RenderedSurface[] = []; + for (const document of documents) { + const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node])); + for (const node of document.domNodes) { + if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue; + if (hiddenByDomPolicy(node, nodesByBackend)) continue; + if (hasVerifiedNativeMirror(node, document.axNodes)) continue; + const visibleRect = intersectViewport(node.rect, viewport); + if (!visibleRect) continue; + const label = clean(node.attrs["aria-label"] ?? node.attrs.title); + surfaces.push({ + renderingKind: "canvas", + frameId: document.frameId, + backendNodeId: node.backendNodeId, + parentBackendNodeId: node.parentBackendNodeId, + rect: node.rect, + ...(node.localRect ? { localRect: node.localRect } : {}), + visibleRect, + paintOrder: node.paintOrder, + ...(label ? { label } : {}), + existenceConfidence: isProminent(node, visibleRect, label) ? "high" : "low", + }); + } + } + return surfaces; +} diff --git a/apps/extension/src/tools/vom/rendered-surfaces/index.ts b/apps/extension/src/tools/vom/rendered-surfaces/index.ts new file mode 100644 index 0000000..6fd1ec9 --- /dev/null +++ b/apps/extension/src/tools/vom/rendered-surfaces/index.ts @@ -0,0 +1,4 @@ +export { clusterRenderedSurfaces } from "./cluster"; +export { discoverRenderedSurfaces } from "./discovery"; +export { projectRenderedSurfaces } from "./project"; +export type { RenderedSurface, RenderedSurfaceGroup } from "./types"; diff --git a/apps/extension/src/tools/vom/rendered-surfaces/project.ts b/apps/extension/src/tools/vom/rendered-surfaces/project.ts new file mode 100644 index 0000000..64f3ad3 --- /dev/null +++ b/apps/extension/src/tools/vom/rendered-surfaces/project.ts @@ -0,0 +1,87 @@ +import type { VomScene, VomVisualSurface, VomVisualSurfaceSummary } from "@browser-skill/vom"; +import type { CapturedFrameDocument } from "../frame-capture"; +import type { SemanticAxNode } from "../semantic-graph"; +import type { RenderedSurfaceGroup } from "./types"; + +function frameBackendKey(frameId: string, backendNodeId: number): string { + return `${frameId}\u0000${backendNodeId}`; +} + +function nearestSceneParent( + scene: VomScene, + documents: CapturedFrameDocument[], + group: RenderedSurfaceGroup, +): number | null { + const sceneByBackend = new Map( + scene.nodes.flatMap((node) => + node.backendNodeId === undefined + ? [] + : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const], + ), + ); + const document = documents.find((candidate) => candidate.frameId === group.frameId); + const domByBackend = new Map(document?.domNodes.map((node) => [node.backendNodeId, node]) ?? []); + let parentBackendNodeId = group.representative.parentBackendNodeId; + let guard = 0; + while (parentBackendNodeId !== null && guard <= domByBackend.size) { + const sceneId = sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId)); + if (sceneId !== undefined) return sceneId; + parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null; + guard += 1; + } + + if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) { + const owner = sceneByBackend.get( + frameBackendKey(document.parentFrameId, document.ownerBackendNodeId), + ); + if (owner !== undefined) return owner; + } + + return ( + scene.nodes.find( + (node) => + node.frameId === group.frameId && + ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""), + )?.id ?? null + ); +} + +export function projectRenderedSurfaces( + scene: VomScene, + documents: CapturedFrameDocument[], + groups: RenderedSurfaceGroup[], +): VomScene { + const visualSurfaces: VomVisualSurface[] = []; + const summaryCounts = new Map< + string, + { parentId: number | null; frameId: string; count: number } + >(); + + for (const group of groups) { + const parentId = nearestSceneParent(scene, documents, group); + if (group.existenceConfidence === "low") { + const key = `${group.frameId}\u0000${parentId ?? "root"}`; + const current = summaryCounts.get(key) ?? { parentId, frameId: group.frameId, count: 0 }; + current.count += group.members.length; + summaryCounts.set(key, current); + continue; + } + visualSurfaces.push({ + parentId, + backendNodeId: group.representative.backendNodeId, + frameId: group.frameId, + renderingKind: "canvas", + rect: group.visibleRect, + ...(group.representative.localRect ? { localRect: group.representative.localRect } : {}), + ...(group.label ? { label: group.label } : {}), + memberCount: group.members.length, + }); + } + + const visualSurfaceSummaries: VomVisualSurfaceSummary[] = [...summaryCounts.values()]; + return { + ...scene, + ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}), + ...(visualSurfaceSummaries.length > 0 ? { visualSurfaceSummaries } : {}), + }; +} diff --git a/apps/extension/src/tools/vom/rendered-surfaces/types.ts b/apps/extension/src/tools/vom/rendered-surfaces/types.ts new file mode 100644 index 0000000..80289f0 --- /dev/null +++ b/apps/extension/src/tools/vom/rendered-surfaces/types.ts @@ -0,0 +1,26 @@ +import type { Rect } from "@browser-skill/vom"; + +export type SurfaceExistenceConfidence = "high" | "low"; + +export interface RenderedSurface { + renderingKind: "canvas"; + frameId: string; + backendNodeId: number; + parentBackendNodeId: number | null; + rect: Rect; + localRect?: Rect; + visibleRect: Rect; + paintOrder: number; + label?: string; + existenceConfidence: SurfaceExistenceConfidence; +} + +export interface RenderedSurfaceGroup { + frameId: string; + members: RenderedSurface[]; + representative: RenderedSurface; + rect: Rect; + visibleRect: Rect; + label?: string; + existenceConfidence: SurfaceExistenceConfidence; +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 2187cd1..713dba1 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -23,6 +23,7 @@ export type ErrorCode = export type RpcErrorReason = | "agent_window_scope" | "element_not_visible" + | "ref_capability_denied" | "ref_not_found" | "selector_not_found" | "target_not_fillable" diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index f27ab2f..5e72708 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -73,6 +73,11 @@ CSS-only hover menu looks like from here. It hovers a bounded set of likely trig few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` is cheaper and more precise. +When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered +canvas surface, not an interactable DOM control. Use +`bsk screenshot --ref @eN --session ` to inspect its visible crop. Do not pass a visual surface +ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. + Escalate page reading only as needed: 1. `bsk observe` for normal semantic understanding, text, controls, and refs. diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index 2f2d6fa..04984f1 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode; pub mod reason { pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope"; pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible"; + pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied"; pub const REF_NOT_FOUND: &str = "ref_not_found"; pub const SELECTOR_NOT_FOUND: &str = "selector_not_found"; pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable"; @@ -299,6 +300,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ), exit_code: base.exit_code, }, + (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo { + summary: "snapshot ref does not support this operation", + hint: Some( + "visual surface refs are screenshot-only; run `bsk screenshot --ref --session `", + ), + exit_code: base.exit_code, + }, (ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo { summary: "target element is not a