figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): fixture test

Browse Source
This commit is contained in:
Ljy-0827
2026-09-08 10:54:33 +08:00
parent 320c610c76
commit 672b17cea8
19 changed files with 691 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/session-manager/ref-store.ts
+7
View File
@@ -1,3 +1,5 @@
import type { Rect } from "@browser-skill/vom";
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
@@ -20,6 +22,7 @@ export interface RefEntry {
tabId: number | null;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
generation: number;
@@ -32,6 +35,7 @@ export type RefInput =
tabId: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
};
@@ -84,6 +88,7 @@ export class RefStore {
tabId?: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
} = {},
@@ -94,6 +99,7 @@ export class RefStore {
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, opts.capabilities),
generation: this.generation,
@@ -124,6 +130,7 @@ export class RefStore {
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, input.capabilities),
generation: this.generation,
apps/extension/src/tools/__tests__/observation.test.ts
+34
View File
@@ -336,6 +336,40 @@ describe("handleScreenshot", () => {
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
});
it("crops a visual surface screenshot to its observation-time visible region", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { x: number; y: number; width: number; height: number };
};
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
});
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
apps/extension/src/tools/observation.ts
+22 -1
View File
@@ -8,6 +8,7 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type Rect,
renderVom,
type VomNode,
type VomOptions,
@@ -194,6 +195,7 @@ async function captureElementScreenshot(
backendNodeId: number,
frameId?: string,
visualSurface = false,
observedVisibleRect?: Rect,
signal?: AbortSignal,
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
if (signal?.aborted) return cancelled("screenshot");
@@ -205,7 +207,24 @@ async function captureElementScreenshot(
);
if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds;
const liveRect = geometry.topBounds;
const rect =
visualSurface && observedVisibleRect
? (() => {
const x = Math.max(liveRect.x, observedVisibleRect.x);
const y = Math.max(liveRect.y, observedVisibleRect.y);
const right = Math.min(
liveRect.x + liveRect.width,
observedVisibleRect.x + observedVisibleRect.w,
);
const bottom = Math.min(
liveRect.y + liveRect.height,
observedVisibleRect.y + observedVisibleRect.h,
);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
})()
: liveRect;
if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
const scale = visualSurface
? Math.min(
1,
@@ -351,6 +370,7 @@ export async function handleScreenshot(
node.backendNodeId,
node.frameId,
node.kind === "surface",
node.visibleRect,
signal,
),
deps.sendToTab,
@@ -1236,6 +1256,7 @@ async function handleVomObservation(
tabId: target.tabId,
...(ref.frameId ? { frameId: ref.frameId } : {}),
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
...(ref.kind ? { kind: ref.kind } : {}),
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
},
apps/extension/src/tools/snapshot-ref.ts
+3
View File
@@ -2,6 +2,7 @@
// compound CDP node identity via the session RefStore, and build stable
// `ref_not_found` errors for hard-failure tool paths.
import type { Rect } from "@browser-skill/vom";
import type { SessionContext } from "@/session-manager/manager";
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
import type { RpcError } from "@/transport/types";
@@ -12,6 +13,7 @@ export interface SnapshotRefLookup {
refKey: string;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
}
@@ -39,6 +41,7 @@ export function lookupSnapshotRef(
refKey,
...(entry.frameId ? { frameId: entry.frameId } : {}),
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
kind: entry.kind,
capabilities: entry.capabilities,
};
apps/extension/src/tools/vom/__tests__/capture.test.ts
+38
View File
@@ -735,6 +735,44 @@ describe("captureViewModel", () => {
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
});
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
const i = (value: string) => S.indexOf(value);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
backendNodeId: [10, 11, 12, 13],
attributes: [[], [], [], []],
},
layout: {
nodeIndex: [1, 2, 3],
styles: [
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
],
bounds: [
[0, 0, 1000, 800],
[10, 10, 200, 100],
[10, 10, 200, 100],
],
paintOrders: [0, 1, 2],
},
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const canvas = nodes.find((node) => node.backendNodeId === 13);
expect(canvas?.rendered).toBe(true);
expect(canvas?.visuallySuppressed).toBe(true);
});
it("subtracts scroll offset to make rects viewport-relative", async () => {
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts
+68
View File
@@ -89,6 +89,31 @@ describe("rendered surface discovery", () => {
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
const canvas = domNode(2, {
attrs: {
id: "must-not-be-a-label",
class: "must-not-be-a-label",
"data-testid": "must-not-be-a-label",
},
});
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
expect.not.objectContaining({
label: expect.anything(),
}),
]);
});
it("treats blank explicit Canvas names as unnamed", () => {
expect(
discoverRenderedSurfaces(
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
{ width: 1280, height: 720 },
),
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
});
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
const canvas = domNode(2, { parentBackendNodeId: 1 });
@@ -101,6 +126,48 @@ describe("rendered surface discovery", () => {
),
).toEqual([]);
});
it("excludes canvases suppressed by a fully transparent ancestor", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 0, y: 0, w: 900, h: 600 },
visuallySuppressed: true,
});
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
).toEqual([]);
});
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 10, y: 20, w: 100, h: 70 },
overflowX: "hidden",
overflowY: "hidden",
});
const partial = domNode(2, {
parentBackendNodeId: 1,
rect: { x: 55, y: 30, w: 140, h: 60 },
});
const fullyClipped = domNode(3, {
parentBackendNodeId: 1,
rect: { x: 140, y: 30, w: 80, h: 50 },
});
expect(
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
width: 1280,
height: 720,
}),
).toEqual([
expect.objectContaining({
backendNodeId: 2,
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
}),
]);
});
});
describe("rendered surface clustering and projection", () => {
@@ -137,6 +204,7 @@ describe("rendered surface clustering and projection", () => {
expect(groups).toHaveLength(1);
expect(groups[0]?.members).toHaveLength(2);
expect(groups[0]?.label).toBeUndefined();
expect(
projectRenderedSurfaces(
scene,
apps/extension/src/tools/vom/capture.ts
+30 -1
View File
@@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
import type { CdpRunner } from "../shared";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"overflow-x",
"overflow-y",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -39,6 +47,10 @@ export interface CapturedNode {
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
overflowX?: string;
overflowY?: string;
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
visuallySuppressed?: boolean;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
@@ -849,6 +861,8 @@ function parseDocumentNodes(
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const overflowXCol = STYLE_COL["overflow-x"];
const overflowYCol = STYLE_COL["overflow-y"];
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -874,6 +888,7 @@ function parseDocumentNodes(
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
@@ -893,7 +908,11 @@ function parseDocumentNodes(
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let overflowX = "visible";
let overflowY = "visible";
let rendered = false;
let visuallySuppressed =
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
@@ -916,12 +935,19 @@ function parseDocumentNodes(
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
visuallySuppressed ||=
visibility === "hidden" ||
visibility === "collapse" ||
(Number.parseFloat(opacity) || 0) <= 0;
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
@@ -956,6 +982,9 @@ function parseDocumentNodes(
position,
pointerEvents,
cursor,
overflowX,
overflowY,
visuallySuppressed,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
apps/extension/src/tools/vom/rendered-surfaces/discovery.ts
+45 -5
View File
@@ -10,14 +10,50 @@ function clean(value: string | undefined): string | undefined {
}
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
const x = Math.max(0, rect.x);
const y = Math.max(0, rect.y);
const right = Math.min(viewport.width, rect.x + rect.w);
const bottom = Math.min(viewport.height, rect.y + rect.h);
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
}
function intersectRect(a: Rect, b: Rect): Rect | null {
const x = Math.max(a.x, b.x);
const y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.w, b.x + b.w);
const bottom = Math.min(a.y + a.h, b.y + b.h);
if (right <= x || bottom <= y) return null;
return { x, y, w: right - x, h: bottom - y };
}
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
function clipToOverflowAncestors(
node: CapturedNode,
rect: Rect,
nodesByBackend: ReadonlyMap<number, CapturedNode>,
): Rect | null {
let clipped: Rect | null = rect;
let parentBackendNodeId = node.parentBackendNodeId;
let guard = 0;
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
const parent = nodesByBackend.get(parentBackendNodeId);
if (!parent) break;
if (parent.rect) {
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
if (clipX || clipY) {
const clipRect = {
x: clipX ? parent.rect.x : clipped.x,
y: clipY ? parent.rect.y : clipped.y,
w: clipX ? parent.rect.w : clipped.w,
h: clipY ? parent.rect.h : clipped.h,
};
clipped = intersectRect(clipped, clipRect);
}
}
parentBackendNodeId = parent.parentBackendNodeId;
guard += 1;
}
return clipped;
}
function excludedByDomPolicy(
node: CapturedNode,
nodesByBackend: Map<number, CapturedNode>,
@@ -27,6 +63,7 @@ function excludedByDomPolicy(
let guard = 0;
while (current && guard <= nodesByBackend.size) {
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
if (current.visuallySuppressed === true) return true;
const attrs = current.attrs;
if (
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
@@ -55,7 +92,10 @@ export function discoverRenderedSurfaces(
for (const node of document.domNodes) {
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
const visibleRect = intersectViewport(node.rect, viewport);
const viewportRect = intersectViewport(node.rect, viewport);
const visibleRect = viewportRect
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
: null;
if (!visibleRect) continue;
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
surfaces.push({
apps/extension/src/tools/vom/rendered-surfaces/project.ts
+1
View File
@@ -87,6 +87,7 @@ export function projectRenderedSurfaces(
backendNodeId: group.representative.backendNodeId,
frameId: group.frameId,
renderingKind: "canvas",
visibleRect: group.representative.visibleRect,
...(group.label ? { label: group.label } : {}),
memberCount: group.members.length,
});
apps/extension/test-fixtures/rendered-surfaces/README.md
+105
View File
@@ -0,0 +1,105 @@
# Rendered Surface browser fixtures
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
## Serve
From the repository root:
```bash
python3 -m http.server 4173 --bind 127.0.0.1 \
--directory apps/extension/test-fixtures/rendered-surfaces
```
Use an Agent Window large enough to keep the primary matrix visible:
```bash
bsk session start --width 1440 --height 1100
bsk navigate http://127.0.0.1:4173/index.html --session <id>
bsk observe --session <id>
```
Always stop the session after the run.
## Pages
### `index.html`
Expected individual Surface labels:
- `fixture-basic-large`
- `fixture-tiny-8px`
- `fixture-pointer-events-none`
- `fixture-partial-ax`
- `fixture-contained-outer`
- `fixture-contained-inner`
- `fixture-different-parent-a`
- `fixture-different-parent-b`
Expected one grouped Surface:
- `fixture-layered-group`, with `layers=2`
Expected absent:
- `fixture-display-none`
- `fixture-visibility-hidden`
- `fixture-opacity-zero`
- `fixture-parent-opacity-zero`
- `fixture-hidden-attribute`
- `fixture-inert-parent`
- `fixture-aria-hidden-parent`
- `fixture-zero-size`
- `fixture-offscreen-right`
- `fixture-fully-overflow-clipped`
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
### `unlabeled.html`
This page contains eight Canvas elements but must render seven generic Surface markers because
the two coincident layers form one group.
- Every marker uses the stable fallback label `canvas visual surface`.
- Exactly one marker has `layers=2`.
- The two contained canvases remain separate markers.
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
- Blank `aria-label` and `title` values behave as no label.
- Fallback AX content does not suppress the corresponding Surface.
### `frames.html`
Expected Surface labels:
- `fixture-same-origin-frame`
- `fixture-cross-origin-frame`
- `fixture-nested-frame`
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
### `viewport.html`
Run with a `900x700` Agent Window.
- `fixture-partial-viewport` is partly visible and must be represented.
- `fixture-below-fold` is absent initially.
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
### `dynamic.html`
Use the normal DOM buttons and re-observe after each action:
- Initial state: `fixture-dynamic` absent.
- After **Insert canvas**: present.
- After **Resize to zero**: absent.
- After **Restore size**: present with a fresh ref.
- After **Remove canvas**: absent.
## Invariants
- `snapshot` must not emit any of these Surface markers.
- `observe` must emit only currently visible Canvas surfaces.
- Existing DOM/AX controls remain readable alongside Surface markers.
- Missing labels never prevent discovery; naming and discovery are independent.
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
- No fixture URL, id, label, or layout constant may appear in production discovery code.
apps/extension/test-fixtures/rendered-surfaces/dynamic.html
+46
View File
@@ -0,0 +1,46 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface dynamic matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
button { padding: 8px 12px; }
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
</style>
</head>
<body>
<h1>Dynamic Canvas lifecycle</h1>
<div class="controls">
<button id="insert" type="button">Insert canvas</button>
<button id="zero" type="button">Resize to zero</button>
<button id="restore" type="button">Restore size</button>
<button id="remove" type="button">Remove canvas</button>
</div>
<div id="mount"></div>
<script>
const mount = document.querySelector("#mount");
const getCanvas = () => mount.querySelector("canvas");
document.querySelector("#insert").addEventListener("click", () => {
if (getCanvas()) return;
const canvas = document.createElement("canvas");
canvas.width = 260;
canvas.height = 120;
canvas.setAttribute("aria-label", "fixture-dynamic");
mount.append(canvas);
});
document.querySelector("#zero").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
});
document.querySelector("#restore").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.removeAttribute("style");
});
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
</script>
</body>
</html>
apps/extension/test-fixtures/rendered-surfaces/frame.html
+28
View File
@@ -0,0 +1,28 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 12px; background: #fdfefe; }
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
</style>
</head>
<body>
<canvas id="frame-canvas" width="220" height="90"></canvas>
<script>
const params = new URLSearchParams(location.search);
const canvas = document.querySelector("#frame-canvas");
if (params.get("canvas") === "0") canvas.remove();
else canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
if (params.get("nested") === "1") {
const iframe = document.createElement("iframe");
iframe.title = "nested fixture frame";
iframe.src = "frame.html?label=fixture-nested-frame";
document.body.append(iframe);
}
</script>
</body>
</html>
apps/extension/test-fixtures/rendered-surfaces/frames.html
+22
View File
@@ -0,0 +1,22 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface frame matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
</style>
</head>
<body>
<h1>Frame matrix</h1>
<div class="frames">
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame"></iframe>
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame"></iframe>
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
</div>
</body>
</html>
apps/extension/test-fixtures/rendered-surfaces/index.html
+118
View File
@@ -0,0 +1,118 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface discovery matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
.same-place { position: relative; width: 180px; height: 90px; }
.same-place > div { position: absolute; inset: 0; }
.same-place canvas { width: 180px; height: 90px; }
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
.tiny { width: 8px; height: 8px; border-width: 1px; }
</style>
</head>
<body>
<h1>Rendered Surface discovery matrix</h1>
<main class="matrix">
<section class="case">
<h2>Visible named canvas</h2>
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Tiny canvas remains addressable</h2>
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
</section>
<section class="case">
<h2>Visible but pointer-events none</h2>
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
</section>
<section class="case">
<h2>Partial AX/fallback content</h2>
<canvas aria-label="fixture-partial-ax" width="180" height="80">
<button type="button">Fallback button must not suppress the surface</button>
</canvas>
</section>
<section class="case">
<h2>Two real rendering layers</h2>
<div class="stack">
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Contained canvases are independent</h2>
<div class="contained">
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
</div>
</section>
<section class="case">
<h2>Same geometry, different parents</h2>
<div class="same-place">
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
</div>
</section>
<section class="case">
<h2>Partially overflow-clipped</h2>
<div class="clip-box">
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
</div>
</section>
<section class="case">
<h2>Fully overflow-clipped</h2>
<div class="clip-box">
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
</div>
</section>
<section class="case">
<h2>CSS-hidden variants</h2>
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
</section>
<section class="case">
<h2>Hidden ancestor variants</h2>
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
</section>
<section class="case">
<h2>Zero-size and offscreen</h2>
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
</section>
</main>
</body>
</html>
apps/extension/test-fixtures/rendered-surfaces/unlabeled.html
+68
View File
@@ -0,0 +1,68 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Unlabeled Rendered Surface matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
.case p { margin: 0 0 8px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
</style>
</head>
<body>
<h1>Unlabeled Rendered Surface matrix</h1>
<main class="matrix">
<section class="case">
<h2>Only engineering attributes</h2>
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Blank explicit names</h2>
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Nearby prose is not a Canvas name</h2>
<p>must-not-be-inferred-as-a-surface-label</p>
<canvas width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Fallback AX content does not name or suppress the Canvas</h2>
<canvas width="180" height="80">
<button type="button">Fallback control remains separate</button>
</canvas>
</section>
<section class="case">
<h2>Unnamed rendering layers still cluster</h2>
<div class="stack">
<canvas width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Unnamed contained canvases stay independent</h2>
<div class="contained">
<canvas width="190" height="100"></canvas>
<canvas width="70" height="40"></canvas>
</div>
</section>
</main>
</body>
</html>
apps/extension/test-fixtures/rendered-surfaces/viewport.html
+22
View File
@@ -0,0 +1,22 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface viewport matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { min-height: 1800px; padding: 16px; }
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
</style>
</head>
<body>
<h1>Viewport matrix</h1>
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
</body>
</html>
packages/vom/src/__tests__/render.test.ts
+30
View File
@@ -40,6 +40,7 @@ describe("renderVom single-layer page", () => {
backendNodeId: 99,
frameId: "child",
renderingKind: "canvas",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
label: "Data grid",
memberCount: 2,
},
@@ -54,6 +55,35 @@ describe("renderVom single-layer page", () => {
ref: "e1",
backendNodeId: 99,
kind: "surface",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
capabilities: ["screenshot"],
}),
);
});
it("renders an unnamed visual surface with a stable generic label", () => {
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
input.visualSurfaces = [
{
parentId: 1,
backendNodeId: 99,
frameId: "main",
renderingKind: "canvas",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
memberCount: 1,
},
];
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
ref: "e1",
name: "canvas visual surface",
kind: "surface",
capabilities: ["screenshot"],
}),
);
packages/vom/src/render.ts
+1
View File
@@ -832,6 +832,7 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render
ref,
backendNodeId: surface.backendNodeId,
frameId: surface.frameId,
visibleRect: surface.visibleRect,
role: "surface",
name: label,
kind: "surface",
packages/vom/src/types.ts
+3
View File
@@ -96,6 +96,8 @@ export interface VomRef {
ref: string;
backendNodeId: number;
frameId?: string;
/** Observation-time top-viewport crop for screenshot-only surfaces. */
visibleRect?: Rect;
kind?: "dom" | "surface";
capabilities?: VomRefCapability[];
}
@@ -107,6 +109,7 @@ export interface VomVisualSurface {
backendNodeId: number;
frameId: string;
renderingKind: "canvas";
visibleRect: Rect;
label?: string;
memberCount: number;
}