diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index 661e997..af7a872 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -1,3 +1,5 @@ +import type { Rect } from "@browser-skill/vom"; + /** * Per-session map from `@e` snapshot refs to a CDP node address. * @@ -20,6 +22,7 @@ export interface RefEntry { tabId: number | null; frameId?: string; cdpSessionId?: string; + visibleRect?: Rect; kind: RefTargetKind; capabilities: RefCapability[]; generation: number; @@ -32,6 +35,7 @@ export type RefInput = tabId: number; frameId?: string; cdpSessionId?: string; + visibleRect?: Rect; kind?: RefTargetKind; capabilities?: RefCapability[]; }; @@ -84,6 +88,7 @@ export class RefStore { tabId?: number; frameId?: string; cdpSessionId?: string; + visibleRect?: Rect; kind?: RefTargetKind; capabilities?: RefCapability[]; } = {}, @@ -94,6 +99,7 @@ export class RefStore { tabId: opts.tabId ?? null, ...(opts.frameId ? { frameId: opts.frameId } : {}), ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}), + ...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}), kind, capabilities: capabilitiesFor(kind, opts.capabilities), generation: this.generation, @@ -124,6 +130,7 @@ export class RefStore { tabId: input.tabId, ...(input.frameId ? { frameId: input.frameId } : {}), ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}), + ...(input.visibleRect ? { visibleRect: input.visibleRect } : {}), kind, capabilities: capabilitiesFor(kind, input.capabilities), generation: this.generation, diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 6e93d2f..cc3b070 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -336,6 +336,40 @@ describe("handleScreenshot", () => { expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096))); }); + it("crops a visual surface screenshot to its observation-time visible region", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e5", 999, { + tabId: 7, + kind: "surface", + visibleRect: { x: 25, y: 30, w: 50, h: 40 }, + }); + const { cdp, sent } = makeFakeCdp({ + "Page.getLayoutMetrics": () => ({ + cssLayoutViewport: { clientWidth: 800, clientHeight: 600 }, + }), + "DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }), + "Page.captureScreenshot": () => ({ data: TINY_PNG }), + }); + + const res = await handleScreenshot( + sm, + { session_id: "aa11", ref: "@e5", tab_id: 7 }, + makeScreenshotDeps({ + cdp, + get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab), + query: vi.fn(), + captureVisibleTab: vi.fn(), + }), + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as { + clip?: { x: number; y: number; width: number; height: number }; + }; + expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 }); + }); + it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 3420541..f529048 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,6 +8,7 @@ import { applyVomInteractionRecovery, type CondSurface, isVomReferenceNode, + type Rect, renderVom, type VomNode, type VomOptions, @@ -191,6 +192,7 @@ async function captureElementScreenshot( backendNodeId: number, frameId?: string, visualSurface = false, + observedVisibleRect?: Rect, signal?: AbortSignal, ): Promise<{ image_base64: string; width: number; height: number } | RpcError> { if (signal?.aborted) return cancelled("screenshot"); @@ -202,7 +204,24 @@ async function captureElementScreenshot( ); if (isRpcError(geometry)) return geometry; if (signal?.aborted) return cancelled("screenshot"); - const rect = geometry.topBounds; + const liveRect = geometry.topBounds; + const rect = + visualSurface && observedVisibleRect + ? (() => { + const x = Math.max(liveRect.x, observedVisibleRect.x); + const y = Math.max(liveRect.y, observedVisibleRect.y); + const right = Math.min( + liveRect.x + liveRect.width, + observedVisibleRect.x + observedVisibleRect.w, + ); + const bottom = Math.min( + liveRect.y + liveRect.height, + observedVisibleRect.y + observedVisibleRect.h, + ); + return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null; + })() + : liveRect; + if (!rect) return { code: "permission_denied", message: "surface is no longer visible" }; const scale = visualSurface ? Math.min( 1, @@ -348,6 +367,7 @@ export async function handleScreenshot( node.backendNodeId, node.frameId, node.kind === "surface", + node.visibleRect, signal, ), deps.sendToTab, @@ -1165,6 +1185,7 @@ async function handleVomObservation( tabId: target.tabId, ...(ref.frameId ? { frameId: ref.frameId } : {}), ...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}), + ...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}), ...(ref.kind ? { kind: ref.kind } : {}), ...(ref.capabilities ? { capabilities: ref.capabilities } : {}), }, diff --git a/apps/extension/src/tools/snapshot-ref.ts b/apps/extension/src/tools/snapshot-ref.ts index 081b966..19e3a25 100644 --- a/apps/extension/src/tools/snapshot-ref.ts +++ b/apps/extension/src/tools/snapshot-ref.ts @@ -2,6 +2,7 @@ // compound CDP node identity via the session RefStore, and build stable // `ref_not_found` errors for hard-failure tool paths. +import type { Rect } from "@browser-skill/vom"; import type { SessionContext } from "@/session-manager/manager"; import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store"; import type { RpcError } from "@/transport/types"; @@ -12,6 +13,7 @@ export interface SnapshotRefLookup { refKey: string; frameId?: string; cdpSessionId?: string; + visibleRect?: Rect; kind: RefTargetKind; capabilities: RefCapability[]; } @@ -39,6 +41,7 @@ export function lookupSnapshotRef( refKey, ...(entry.frameId ? { frameId: entry.frameId } : {}), ...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}), + ...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}), kind: entry.kind, capabilities: entry.capabilities, }; diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 9463383..afddb57 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -722,6 +722,44 @@ describe("captureViewModel", () => { expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false); }); + it("propagates fully transparent ancestor suppression to painted descendants", async () => { + const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"]; + const i = (value: string) => S.indexOf(value); + const snapshot = { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 2], + nodeName: [i("html"), i("body"), i("div"), i("canvas")], + backendNodeId: [10, 11, 12, 13], + attributes: [[], [], [], []], + }, + layout: { + nodeIndex: [1, 2, 3], + styles: [ + [i("static"), i("auto"), i("auto"), i("visible"), i("1")], + [i("static"), i("auto"), i("auto"), i("visible"), i("0")], + [i("static"), i("auto"), i("auto"), i("visible"), i("1")], + ], + bounds: [ + [0, 0, 1000, 800], + [10, 10, 200, 100], + [10, 10, 200, 100], + ], + paintOrders: [0, 1, 2], + }, + }, + ], + }; + + const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const canvas = nodes.find((node) => node.backendNodeId === 13); + + expect(canvas?.rendered).toBe(true); + expect(canvas?.visuallySuppressed).toBe(true); + }); + it("subtracts scroll offset to make rects viewport-relative", async () => { const cdp = { send: vi.fn(async (_t: number, method: string) => { diff --git a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts index 05861d0..03edf59 100644 --- a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts +++ b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts @@ -89,6 +89,31 @@ describe("rendered surface discovery", () => { ).toEqual([expect.objectContaining({ backendNodeId: 2 })]); }); + it("discovers unnamed canvases without guessing labels from engineering attributes", () => { + const canvas = domNode(2, { + attrs: { + id: "must-not-be-a-label", + class: "must-not-be-a-label", + "data-testid": "must-not-be-a-label", + }, + }); + + expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([ + expect.not.objectContaining({ + label: expect.anything(), + }), + ]); + }); + + it("treats blank explicit Canvas names as unnamed", () => { + expect( + discoverRenderedSurfaces( + [document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])], + { width: 1280, height: 720 }, + ), + ).toEqual([expect.not.objectContaining({ label: expect.anything() })]); + }); + it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => { const owner = domNode(1, { tag: "div", rect: null, localRect: null }); const canvas = domNode(2, { parentBackendNodeId: 1 }); @@ -101,6 +126,48 @@ describe("rendered surface discovery", () => { ), ).toEqual([]); }); + + it("excludes canvases suppressed by a fully transparent ancestor", () => { + const owner = domNode(1, { + tag: "div", + rect: { x: 0, y: 0, w: 900, h: 600 }, + visuallySuppressed: true, + }); + const canvas = domNode(2, { parentBackendNodeId: 1 }); + + expect( + discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }), + ).toEqual([]); + }); + + it("clips canvases to overflow ancestors and drops fully clipped canvases", () => { + const owner = domNode(1, { + tag: "div", + rect: { x: 10, y: 20, w: 100, h: 70 }, + overflowX: "hidden", + overflowY: "hidden", + }); + const partial = domNode(2, { + parentBackendNodeId: 1, + rect: { x: 55, y: 30, w: 140, h: 60 }, + }); + const fullyClipped = domNode(3, { + parentBackendNodeId: 1, + rect: { x: 140, y: 30, w: 80, h: 50 }, + }); + + expect( + discoverRenderedSurfaces([document([owner, partial, fullyClipped])], { + width: 1280, + height: 720, + }), + ).toEqual([ + expect.objectContaining({ + backendNodeId: 2, + visibleRect: { x: 55, y: 30, w: 55, h: 60 }, + }), + ]); + }); }); describe("rendered surface clustering and projection", () => { @@ -137,6 +204,7 @@ describe("rendered surface clustering and projection", () => { expect(groups).toHaveLength(1); expect(groups[0]?.members).toHaveLength(2); + expect(groups[0]?.label).toBeUndefined(); expect( projectRenderedSurfaces( scene, diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 1db0e29..522c48c 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport } import type { CdpRunner } from "../shared"; import { clearHover, waitForHover } from "./hover-perception"; -const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const; +const REQUESTED_STYLES = [ + "position", + "pointer-events", + "cursor", + "visibility", + "opacity", + "overflow-x", + "overflow-y", +] as const; const STYLE_COL = Object.fromEntries( REQUESTED_STYLES.map((name, index) => [name, index]), ) as Record<(typeof REQUESTED_STYLES)[number], number>; @@ -39,6 +47,10 @@ export interface CapturedNode { * `textContent`: the live parser always sets it, hand-built fixtures may not. */ cursor?: string; + overflowX?: string; + overflowY?: string; + /** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */ + visuallySuppressed?: boolean; /** * Whether the live DOM snapshot provides a painted, non-hidden box for this * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed @@ -796,6 +808,8 @@ function parseDocumentNodes( const cursorCol = STYLE_COL.cursor; const visibilityCol = STYLE_COL.visibility; const opacityCol = STYLE_COL.opacity; + const overflowXCol = STYLE_COL["overflow-x"]; + const overflowYCol = STYLE_COL["overflow-y"]; const inputValues = sparseIndexMap(dn.inputValue); const textValues = sparseIndexMap(dn.textValue); const checkedInputs = new Set(dn.inputChecked?.index ?? []); @@ -821,6 +835,7 @@ function parseDocumentNodes( // Build CapturedNodes. const nodes: CapturedNode[] = []; + const visuallySuppressedByNodeIndex = new Array(count).fill(false); for (let n = 0; n < count; n++) { if (excluded[n]) continue; const backendNodeId = dn.backendNodeId[n]; @@ -840,7 +855,11 @@ function parseDocumentNodes( let position = "static"; let pointerEvents = "auto"; let cursor = "auto"; + let overflowX = "visible"; + let overflowY = "visible"; let rendered = false; + let visuallySuppressed = + parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false; const li = layoutByNode.get(n); if (li !== undefined) { const b = dl?.bounds?.[li]; @@ -863,12 +882,19 @@ function parseDocumentNodes( cursor = str(strings, styleRow[cursorCol]) || "auto"; const visibility = str(strings, styleRow[visibilityCol]) || "visible"; const opacity = str(strings, styleRow[opacityCol]) || "1"; + overflowX = str(strings, styleRow[overflowXCol]) || "visible"; + overflowY = str(strings, styleRow[overflowYCol]) || "visible"; + visuallySuppressed ||= + visibility === "hidden" || + visibility === "collapse" || + (Number.parseFloat(opacity) || 0) <= 0; rendered = localRect !== null && visibility !== "hidden" && visibility !== "collapse" && (Number.parseFloat(opacity) || 0) > 0; } + visuallySuppressedByNodeIndex[n] = visuallySuppressed; // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no // geometry and are never queried by callers. @@ -903,6 +929,9 @@ function parseDocumentNodes( position, pointerEvents, cursor, + overflowX, + overflowY, + visuallySuppressed, rendered, textContent, ...(formValue !== undefined ? { formValue } : {}), diff --git a/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts b/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts index f4cb8e8..1af9bc9 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/discovery.ts @@ -10,14 +10,50 @@ function clean(value: string | undefined): string | undefined { } function intersectViewport(rect: Rect, viewport: Viewport): Rect | null { - const x = Math.max(0, rect.x); - const y = Math.max(0, rect.y); - const right = Math.min(viewport.width, rect.x + rect.w); - const bottom = Math.min(viewport.height, rect.y + rect.h); + return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height }); +} + +function intersectRect(a: Rect, b: Rect): Rect | null { + const x = Math.max(a.x, b.x); + const y = Math.max(a.y, b.y); + const right = Math.min(a.x + a.w, b.x + b.w); + const bottom = Math.min(a.y + a.h, b.y + b.h); if (right <= x || bottom <= y) return null; return { x, y, w: right - x, h: bottom - y }; } +const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]); + +function clipToOverflowAncestors( + node: CapturedNode, + rect: Rect, + nodesByBackend: ReadonlyMap, +): Rect | null { + let clipped: Rect | null = rect; + let parentBackendNodeId = node.parentBackendNodeId; + let guard = 0; + while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) { + const parent = nodesByBackend.get(parentBackendNodeId); + if (!parent) break; + if (parent.rect) { + const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible"); + const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible"); + if (clipX || clipY) { + const clipRect = { + x: clipX ? parent.rect.x : clipped.x, + y: clipY ? parent.rect.y : clipped.y, + w: clipX ? parent.rect.w : clipped.w, + h: clipY ? parent.rect.h : clipped.h, + }; + clipped = intersectRect(clipped, clipRect); + } + } + parentBackendNodeId = parent.parentBackendNodeId; + guard += 1; + } + return clipped; +} + function excludedByDomPolicy( node: CapturedNode, nodesByBackend: Map, @@ -27,6 +63,7 @@ function excludedByDomPolicy( let guard = 0; while (current && guard <= nodesByBackend.size) { if (excludedBackendNodeIds.has(current.backendNodeId)) return true; + if (current.visuallySuppressed === true) return true; const attrs = current.attrs; if ( Object.prototype.hasOwnProperty.call(attrs, "hidden") || @@ -55,7 +92,10 @@ export function discoverRenderedSurfaces( for (const node of document.domNodes) { if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue; if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue; - const visibleRect = intersectViewport(node.rect, viewport); + const viewportRect = intersectViewport(node.rect, viewport); + const visibleRect = viewportRect + ? clipToOverflowAncestors(node, viewportRect, nodesByBackend) + : null; if (!visibleRect) continue; const label = clean(node.attrs["aria-label"] ?? node.attrs.title); surfaces.push({ diff --git a/apps/extension/src/tools/vom/rendered-surfaces/project.ts b/apps/extension/src/tools/vom/rendered-surfaces/project.ts index 97e0b7c..c219457 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/project.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/project.ts @@ -87,6 +87,7 @@ export function projectRenderedSurfaces( backendNodeId: group.representative.backendNodeId, frameId: group.frameId, renderingKind: "canvas", + visibleRect: group.representative.visibleRect, ...(group.label ? { label: group.label } : {}), memberCount: group.members.length, }); diff --git a/apps/extension/test-fixtures/rendered-surfaces/README.md b/apps/extension/test-fixtures/rendered-surfaces/README.md new file mode 100644 index 0000000..d0f10a0 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/README.md @@ -0,0 +1,105 @@ +# Rendered Surface browser fixtures + +These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent. + +## Serve + +From the repository root: + +```bash +python3 -m http.server 4173 --bind 127.0.0.1 \ + --directory apps/extension/test-fixtures/rendered-surfaces +``` + +Use an Agent Window large enough to keep the primary matrix visible: + +```bash +bsk session start --width 1440 --height 1100 +bsk navigate http://127.0.0.1:4173/index.html --session +bsk observe --session +``` + +Always stop the session after the run. + +## Pages + +### `index.html` + +Expected individual Surface labels: + +- `fixture-basic-large` +- `fixture-tiny-8px` +- `fixture-pointer-events-none` +- `fixture-partial-ax` +- `fixture-contained-outer` +- `fixture-contained-inner` +- `fixture-different-parent-a` +- `fixture-different-parent-b` + +Expected one grouped Surface: + +- `fixture-layered-group`, with `layers=2` + +Expected absent: + +- `fixture-display-none` +- `fixture-visibility-hidden` +- `fixture-opacity-zero` +- `fixture-parent-opacity-zero` +- `fixture-hidden-attribute` +- `fixture-inert-parent` +- `fixture-aria-hidden-parent` +- `fixture-zero-size` +- `fixture-offscreen-right` +- `fixture-fully-overflow-clipped` + +`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view. + +### `unlabeled.html` + +This page contains eight Canvas elements but must render seven generic Surface markers because +the two coincident layers form one group. + +- Every marker uses the stable fallback label `canvas visual surface`. +- Exactly one marker has `layers=2`. +- The two contained canvases remain separate markers. +- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label. +- Blank `aria-label` and `title` values behave as no label. +- Fallback AX content does not suppress the corresponding Surface. + +### `frames.html` + +Expected Surface labels: + +- `fixture-same-origin-frame` +- `fixture-cross-origin-frame` +- `fixture-nested-frame` + +The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service. + +### `viewport.html` + +Run with a `900x700` Agent Window. + +- `fixture-partial-viewport` is partly visible and must be represented. +- `fixture-below-fold` is absent initially. +- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref. + +### `dynamic.html` + +Use the normal DOM buttons and re-observe after each action: + +- Initial state: `fixture-dynamic` absent. +- After **Insert canvas**: present. +- After **Resize to zero**: absent. +- After **Restore size**: present with a fresh ref. +- After **Remove canvas**: absent. + +## Invariants + +- `snapshot` must not emit any of these Surface markers. +- `observe` must emit only currently visible Canvas surfaces. +- Existing DOM/AX controls remain readable alongside Surface markers. +- Missing labels never prevent discovery; naming and discovery are independent. +- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select. +- No fixture URL, id, label, or layout constant may appear in production discovery code. diff --git a/apps/extension/test-fixtures/rendered-surfaces/dynamic.html b/apps/extension/test-fixtures/rendered-surfaces/dynamic.html new file mode 100644 index 0000000..56459bf --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/dynamic.html @@ -0,0 +1,46 @@ + + + + + + Rendered Surface dynamic matrix + + + +

Dynamic Canvas lifecycle

+
+ + + + +
+
+ + + diff --git a/apps/extension/test-fixtures/rendered-surfaces/frame.html b/apps/extension/test-fixtures/rendered-surfaces/frame.html new file mode 100644 index 0000000..cc42cd4 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/frame.html @@ -0,0 +1,28 @@ + + + + + + + + + + + + diff --git a/apps/extension/test-fixtures/rendered-surfaces/frames.html b/apps/extension/test-fixtures/rendered-surfaces/frames.html new file mode 100644 index 0000000..6355030 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/frames.html @@ -0,0 +1,22 @@ + + + + + + Rendered Surface frame matrix + + + +

Frame matrix

+
+ + + +
+ + diff --git a/apps/extension/test-fixtures/rendered-surfaces/index.html b/apps/extension/test-fixtures/rendered-surfaces/index.html new file mode 100644 index 0000000..ca31cae --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/index.html @@ -0,0 +1,118 @@ + + + + + + Rendered Surface discovery matrix + + + +

Rendered Surface discovery matrix

+
+
+

Visible named canvas

+ +
+ +
+

Tiny canvas remains addressable

+ +
+ +
+

Visible but pointer-events none

+ +
+ +
+

Partial AX/fallback content

+ + + +
+ +
+

Two real rendering layers

+
+ + +
+
+ +
+

Contained canvases are independent

+
+ + +
+
+ +
+

Same geometry, different parents

+
+
+
+
+
+ +
+

Partially overflow-clipped

+
+ +
+
+ +
+

Fully overflow-clipped

+
+ +
+
+ +
+

CSS-hidden variants

+ + + +
+ +
+

Hidden ancestor variants

+
+ +
+ +
+ +
+

Zero-size and offscreen

+ + +
+
+ + + diff --git a/apps/extension/test-fixtures/rendered-surfaces/unlabeled.html b/apps/extension/test-fixtures/rendered-surfaces/unlabeled.html new file mode 100644 index 0000000..ecfe74a --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/unlabeled.html @@ -0,0 +1,68 @@ + + + + + + Unlabeled Rendered Surface matrix + + + +

Unlabeled Rendered Surface matrix

+
+
+

Only engineering attributes

+ +
+ +
+

Blank explicit names

+ +
+ +
+

Nearby prose is not a Canvas name

+

must-not-be-inferred-as-a-surface-label

+ +
+ +
+

Fallback AX content does not name or suppress the Canvas

+ + + +
+ +
+

Unnamed rendering layers still cluster

+
+ + +
+
+ +
+

Unnamed contained canvases stay independent

+
+ + +
+
+
+ + diff --git a/apps/extension/test-fixtures/rendered-surfaces/viewport.html b/apps/extension/test-fixtures/rendered-surfaces/viewport.html new file mode 100644 index 0000000..03e61d1 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/viewport.html @@ -0,0 +1,22 @@ + + + + + + Rendered Surface viewport matrix + + + +

Viewport matrix

+

The orange canvas on the left is intentionally clipped by the viewport.

+ + + + + diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index ee97bfb..08f7350 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -40,6 +40,7 @@ describe("renderVom single-layer page", () => { backendNodeId: 99, frameId: "child", renderingKind: "canvas", + visibleRect: { x: 10, y: 20, w: 800, h: 500 }, label: "Data grid", memberCount: 2, }, @@ -54,6 +55,35 @@ describe("renderVom single-layer page", () => { ref: "e1", backendNodeId: 99, kind: "surface", + visibleRect: { x: 10, y: 20, w: 800, h: 500 }, + capabilities: ["screenshot"], + }), + ); + }); + + it("renders an unnamed visual surface with a stable generic label", () => { + const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]); + input.visualSurfaces = [ + { + parentId: 1, + backendNodeId: 99, + frameId: "main", + renderingKind: "canvas", + visibleRect: { x: 10, y: 20, w: 800, h: 500 }, + memberCount: 1, + }, + ]; + + const out = renderVom(input); + + expect(out.text).toContain( + '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]', + ); + expect(out.refs).toContainEqual( + expect.objectContaining({ + ref: "e1", + name: "canvas visual surface", + kind: "surface", capabilities: ["screenshot"], }), ); diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 26eb55f..53076d0 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -776,6 +776,7 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render ref, backendNodeId: surface.backendNodeId, frameId: surface.frameId, + visibleRect: surface.visibleRect, role: "surface", name: label, kind: "surface", diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 6464212..b76f33d 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -96,6 +96,8 @@ export interface VomRef { ref: string; backendNodeId: number; frameId?: string; + /** Observation-time top-viewport crop for screenshot-only surfaces. */ + visibleRect?: Rect; kind?: "dom" | "surface"; capabilities?: VomRefCapability[]; } @@ -107,6 +109,7 @@ export interface VomVisualSurface { backendNodeId: number; frameId: string; renderingKind: "canvas"; + visibleRect: Rect; label?: string; memberCount: number; }