From 52a8e65b346c1c8e69c18dbfc03d0429c2d74abb Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 9 Sep 2026 18:27:02 +0800 Subject: [PATCH] fix(vom): fix border radius incorrectly reject canvas --- .../vom/__tests__/capture-coordinator.test.ts | 85 +++++++++++++++++-- .../src/tools/vom/__tests__/facts.test.ts | 27 ++---- .../vom/__tests__/visual-discovery.test.ts | 51 ++++++++++- .../tools/vom/__tests__/visual-region.test.ts | 58 +++++++++++++ .../src/tools/vom/capture-coordinator.ts | 22 ++++- apps/extension/src/tools/vom/facts.ts | 19 +++-- apps/extension/src/tools/vom/normalize.ts | 16 +++- apps/extension/src/tools/vom/snapshot.ts | 25 ++++-- .../src/tools/vom/visual-discovery.ts | 28 ++++-- apps/extension/src/tools/vom/visual-region.ts | 2 + 10 files changed, 275 insertions(+), 58 deletions(-) diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index 0a4f9bd..0dde3b9 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -5,8 +5,8 @@ import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import { captureVomObservation } from "../../observation"; import type { CdpRunner } from "../../shared"; import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; -import { buildSemanticGraph } from "../semantic-graph/build"; -import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot"; +import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph"; +import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot"; import { discoverVisualCandidates } from "../visual-discovery"; function fixture( @@ -85,7 +85,8 @@ function fixture( }; if (method === "DOMSnapshot.captureSnapshot") { snapshots++; - expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES); + const requested = (params as { computedStyles: readonly string[] }).computedStyles; + expect([REQUESTED_STYLES, VISUAL_STYLES]).toContainEqual(requested); result = { strings: [ "#document", @@ -153,10 +154,14 @@ function fixture( [0, 0, 1000, 800], [0, 0, 100, 40], ], - styles: [ - [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11], - [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11], - ], + styles: Array.from({ length: 2 }, () => + requested.map( + (name) => + [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11][ + VISUAL_STYLES.indexOf(name as (typeof VISUAL_STYLES)[number]) + ], + ), + ), }, }; }), @@ -1374,12 +1379,76 @@ describe("AX frame scheduling", () => { }); describe("visual facts integration", () => { + it("defaults to base facts across targets and shares semantic behavior with visual capture", async () => { + const base = fixture({ canvas: true }); + const visual = fixture({ canvas: true }); + const baseFacts = await captureObservationFacts(base.cdp, 4); + const visualFacts = await captureObservationFacts(visual.cdp, 4, undefined, undefined, { + includeVisualFacts: true, + }); + expect(baseFacts.visualFactsCollected).toBe(false); + expect(visualFacts.visualFactsCollected).toBe(true); + expect(baseFacts.documents).toHaveLength(4); + expect(visualFacts.documents).toHaveLength(4); + for (const doc of baseFacts.documents) { + expect(doc.geometry).toBeUndefined(); + expect(doc.index.ancestryComplete).toBeUndefined(); + for (const node of doc.index.nodes.values()) { + expect(node.parentMissing).toBeUndefined(); + expect(node.layout?.clientRect).toBeUndefined(); + if (node.layout) expect(Object.keys(node.layout.styles)).toEqual([...REQUESTED_STYLES]); + } + } + // This fixture has no child frame projection measurements. Enabling visual + // collection must preserve that missing evidence, not invent geometry. + expect( + visualFacts.documents.find((doc) => doc.frame.frameId === "main")?.geometry, + ).toBeDefined(); + expect(visualFacts.documents.filter((doc) => doc.geometry)).toHaveLength(1); + for (const doc of visualFacts.documents) { + expect(doc.index.ancestryComplete).toBeDefined(); + for (const node of doc.index.nodes.values()) + if (node.layout) { + expect(Object.keys(node.layout.styles)).toEqual([...VISUAL_STYLES]); + expect(Object.keys(node.layout.styles).some((key) => key.includes("radius"))).toBe(false); + } + } + for (const [capture, expected] of [ + [base, REQUESTED_STYLES], + [visual, VISUAL_STYLES], + ] as const) { + const calls = capture.logs.filter((c) => c.method === "DOMSnapshot.captureSnapshot"); + expect(calls).toHaveLength(2); + for (const call of calls) expect(call.params.computedStyles).toEqual(expected); + } + expect(base.logs.map((c) => [c.target, c.method])).toEqual( + visual.logs.map((c) => [c.target, c.method]), + ); + const scene = (input: typeof baseFacts) => + buildSemanticVomScene({ + documents: semanticCapture(input).documents, + viewport: input.viewport, + rootFrameId: input.rootFrameId, + excludedBackendNodeIds: semanticCapture(input).captured.excludedBackendNodeIds, + }); + expect(scene(baseFacts)).toEqual(scene(visualFacts)); + const before = base.logs.length; + expect(await discoverVisualCandidates(baseFacts)).toMatchObject({ + complete: false, + candidates: [], + issues: [{ reason: "visual-facts-not-collected" }], + }); + expect(base.logs).toHaveLength(before); + }); + it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => { const { cdp, logs } = fixture({ canvas: true, frames: [{ frameId: "main", target: { tabId: 4 } }], }); - const facts = await captureObservationFacts(cdp, 4); + const facts = await captureObservationFacts(cdp, 4, undefined, undefined, { + includeVisualFacts: true, + }); const before = logs.length; const result = await discoverVisualCandidates(facts); expect(logs).toHaveLength(before); diff --git a/apps/extension/src/tools/vom/__tests__/facts.test.ts b/apps/extension/src/tools/vom/__tests__/facts.test.ts index d99da01..c9b3367 100644 --- a/apps/extension/src/tools/vom/__tests__/facts.test.ts +++ b/apps/extension/src/tools/vom/__tests__/facts.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import { createCaptureCheckpoint } from "../capture-abort"; import { buildDocumentIndex, type DecodedNode } from "../facts"; -import { decodeDocument, REQUESTED_STYLES } from "../snapshot"; +import { decodeDocument, REQUESTED_STYLES, VISUAL_SNAPSHOT } from "../snapshot"; function node(id: number, parent: number | null): DecodedNode { return { @@ -80,19 +80,6 @@ describe("document facts", () => { boundsSpace: "snapshot-document-layout", bounds: [10, 20, 120, 40], styles: { - rotate: "auto", - scale: "auto", - perspective: "auto", - clip: "auto", - contain: "auto", - "overflow-clip-margin": "auto", - display: "auto", - "overflow-x": "auto", - "overflow-y": "auto", - transform: "auto", - zoom: "auto", - "clip-path": "auto", - "mask-image": "auto", position: "static", "pointer-events": "auto", cursor: "auto", @@ -153,9 +140,9 @@ describe("visual ancestry facts", () => { { ...node(8, null), nodeType: 1 }, { ...node(9, null), nodeType: 9, parentMissing: true }, ]; - const index = await buildDocumentIndex(input.reverse()); - for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete.get(id)).toBe(true); - for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete.get(id)).toBe(false); + const index = await buildDocumentIndex(input.reverse(), undefined, true); + for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete!.get(id)).toBe(true); + for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete!.get(id)).toBe(false); }); it("retains client offsets in their source units and does not disguise missing parent indices", async () => { @@ -169,11 +156,13 @@ describe("visual ancestry facts", () => { }, }, ["div"], + undefined, + VISUAL_SNAPSHOT, ); expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]); expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]); expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true); - const index = await buildDocumentIndex(decoded.nodes); - expect([...index.ancestryComplete.values()]).toEqual([false, false]); + const index = await buildDocumentIndex(decoded.nodes, undefined, true); + expect([...index.ancestryComplete!.values()]).toEqual([false, false]); }); }); diff --git a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts index 8c3c97f..60c7d55 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts @@ -5,7 +5,7 @@ import type { GeometryProjection } from "../../geometry"; import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts"; import type { FrameOwnedAxNode } from "../frame-document"; import { normalizeDocument } from "../normalize"; -import { REQUESTED_STYLES } from "../snapshot"; +import { VISUAL_SNAPSHOT, VISUAL_STYLES } from "../snapshot"; import { discoverVisualCandidates } from "../visual-discovery"; const defaults: Record = { @@ -86,7 +86,7 @@ async function document( bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]), clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]), styles: layouts.map((n) => - REQUESTED_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])), + VISUAL_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])), ), }, }, @@ -108,6 +108,8 @@ async function document( }, pageScale: options.pageScale ?? 1, }, + undefined, + VISUAL_SNAPSHOT, ); return { frame, @@ -128,6 +130,7 @@ function facts( issues: ObservationFacts["issues"] = [], ): ObservationFacts { return { + visualFactsCollected: true, rootFrameId: "main", documents, viewport: { width: 1000, height: 800 }, @@ -138,6 +141,46 @@ function facts( } describe("Canvas discovery", () => { + it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => { + const empty = facts([]); + expect(await discoverVisualCandidates(empty)).toMatchObject({ + complete: true, + candidates: [], + issues: [], + }); + expect(await discoverVisualCandidates({ ...empty, visualFactsCollected: false })).toMatchObject( + { complete: false, candidates: [], issues: [{ reason: "visual-facts-not-collected" }] }, + ); + const controller = new AbortController(); + controller.abort(); + await expect( + discoverVisualCandidates({ ...empty, visualFactsCollected: false }, controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }); + }); + + it("discovers Canvas and frame contents without corner radius facts", async () => { + const own = await document([{ id: 2, parent: 1, styles: {} }]); + expect(await discoverVisualCandidates(facts([own]))).toMatchObject({ + candidates: [{ backendNodeId: 2 }], + complete: true, + issues: [], + }); + const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]); + const child = await document([{ id: 3, parent: 1 }], { + frame: { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 2, + target: { tabId: 1 }, + }, + }); + expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({ + candidates: [{ backendNodeId: 3 }], + complete: true, + issues: [], + }); + }); + it.each([ 0.8, 1, 1.25, 2, ])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => { @@ -383,7 +426,7 @@ describe("Canvas discovery", () => { ]); for (const parent of [99, 3]) { doc.index.nodes.get(2)!.parentBackendNodeId = parent; - const index = await buildDocumentIndex([...doc.index.nodes.values()]); + const index = await buildDocumentIndex([...doc.index.nodes.values()], undefined, true); const result = await discoverVisualCandidates(facts([{ ...doc, index }])); expect(result.candidates).toHaveLength(0); expect(result.issues[0].reason).toBe("ancestry-incomplete"); @@ -445,7 +488,7 @@ describe("Canvas discovery", () => { }); input.push(node); } - const index = await buildDocumentIndex(input); + const index = await buildDocumentIndex(input, undefined, true); reads = 0; const result = await discoverVisualCandidates(facts([{ ...doc, index }])); expect(result.candidates).toHaveLength(count / 2 + 2); diff --git a/apps/extension/src/tools/vom/__tests__/visual-region.test.ts b/apps/extension/src/tools/vom/__tests__/visual-region.test.ts index 3003283..dc9ccce 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-region.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-region.test.ts @@ -39,6 +39,64 @@ const ancestor: VisualAncestor = { const box = { x: 0, y: 0, width: 120, height: 40 }; describe("shared visual region rules", () => { + it.each([ + "visible", + "hidden", + "auto", + "scroll", + "clip", + ])("keeps %s screenshot bounds independent of rounded styling", (overflow) => { + for (const clipContents of [true, false]) { + const node = { + ...ancestor, + styles: { ...styles, "overflow-x": overflow, "overflow-y": overflow }, + }; + const baseline = extendVisualContext(EMPTY_VISUAL_CONTEXT, node, clipContents); + for (const radius of ["0px", "8px", "50%", "4px 8px"]) { + const rounded = extendVisualContext( + EMPTY_VISUAL_CONTEXT, + { + ...node, + styles: { + ...node.styles, + "border-top-left-radius": radius, + "border-top-right-radius": radius, + "border-bottom-left-radius": radius, + "border-bottom-right-radius": radius, + }, + }, + clipContents, + ); + expect(rounded).toEqual(baseline); + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: box, + visibility: "visible", + context: rounded, + }), + ).toMatchObject({ status: "available" }); + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: box, + visibility: "hidden", + context: rounded, + }), + ).toEqual({ status: "empty" }); + if (clipContents && overflow !== "visible") + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: { x: 80, y: 0, width: 20, height: 20 }, + visibility: "visible", + context: rounded, + }), + ).toEqual({ status: "empty" }); + } + } + }); + it("accepts independently supplied normalized facts and preserves clipping policy", () => { const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...ancestor, diff --git a/apps/extension/src/tools/vom/capture-coordinator.ts b/apps/extension/src/tools/vom/capture-coordinator.ts index a6dba2d..0a90a03 100644 --- a/apps/extension/src/tools/vom/capture-coordinator.ts +++ b/apps/extension/src/tools/vom/capture-coordinator.ts @@ -22,7 +22,12 @@ import { type FrameOwnedAxNode, } from "./frame-document"; import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize"; -import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot"; +import { + BASIC_SNAPSHOT, + describeSnapshotFrames, + type SnapshotReply, + VISUAL_SNAPSHOT, +} from "./snapshot"; interface TargetBatch { target: CdpTarget; @@ -69,7 +74,9 @@ export async function captureObservationFacts( tabId: number, signal?: AbortSignal, pageUrl?: string, + options: { includeVisualFacts?: boolean } = {}, ): Promise> { + const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT; const startedAt = Date.now(); throwCaptureAborted(signal); let graph: CdpFrameGraph | undefined; @@ -125,7 +132,7 @@ export async function captureObservationFacts( throwCaptureAborted(signal); batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId); const snapshot = await scoped.send(tabId, "DOMSnapshot.captureSnapshot", { - computedStyles: REQUESTED_STYLES, + computedStyles: profile.computedStyles, includePaintOrder: true, includeDOMRects: true, }); @@ -146,6 +153,7 @@ export async function captureObservationFacts( issues, signal, observed.rootFrameId, + profile, ); const formsAvailable = await enrichFormControlStates( scoped, @@ -384,7 +392,15 @@ export async function captureObservationFacts( throwCaptureAborted(signal); if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length)) throw firstFailure; - return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() }; + return { + visualFactsCollected: profile.includeVisualFacts, + rootFrameId, + viewport, + documents: facts, + issues, + startedAt, + finishedAt: Date.now(), + }; } /** Existing semantic consumers get a narrow view; no raw snapshot diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts index 9c1f5c2..a512ed1 100644 --- a/apps/extension/src/tools/vom/facts.ts +++ b/apps/extension/src/tools/vom/facts.ts @@ -30,7 +30,7 @@ export interface NodeFacts extends CapturedNode { export interface DocumentIndex { readonly nodes: ReadonlyMap; - readonly ancestryComplete: ReadonlyMap; + readonly ancestryComplete?: ReadonlyMap; readonly excludedBackendNodeIds: ReadonlySet; } @@ -43,11 +43,12 @@ export interface DecodedDocument { export async function buildDocumentIndex( input: readonly T[], signal?: AbortSignal, + includeVisualFacts = false, ): Promise> { const checkpoint = createCaptureCheckpoint(signal); const nodes = new Map(); const overlayByNode = new Map(); - const ancestryComplete = new Map(); + const ancestryComplete = includeVisualFacts ? new Map() : undefined; const excludedBackendNodeIds = new Set(); for (let i = 0; i < input.length; i++) { if (i % 256 === 0) { @@ -84,7 +85,7 @@ export async function buildDocumentIndex( visiting.add(current.backendNodeId); path.push(current); if (current.parentBackendNodeId === null) { - complete = !current.parentMissing && current.nodeType === 9; + if (ancestryComplete) complete = !current.parentMissing && current.nodeType === 9; current = undefined; break; } @@ -92,7 +93,7 @@ export async function buildDocumentIndex( } if (current && overlayByNode.has(current.backendNodeId)) { overlay = overlayByNode.get(current.backendNodeId)!; - complete = ancestryComplete.get(current.backendNodeId) === true; + if (ancestryComplete) complete = ancestryComplete.get(current.backendNodeId) === true; } for (let i = path.length - 1; i >= 0; i--) { if (work++ % 256 === 0) { @@ -102,12 +103,14 @@ export async function buildDocumentIndex( const item = path[i]; overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)); overlayByNode.set(item.backendNodeId, overlay); - complete = complete && !item.parentMissing; - ancestryComplete.set(item.backendNodeId, complete); + if (ancestryComplete) { + complete = complete && !item.parentMissing; + ancestryComplete.set(item.backendNodeId, complete); + } if (overlay) excludedBackendNodeIds.add(item.backendNodeId); } } - return { nodes, ancestryComplete, excludedBackendNodeIds }; + return { nodes, ...(ancestryComplete ? { ancestryComplete } : {}), excludedBackendNodeIds }; } export interface DocumentIdentity { @@ -158,6 +161,8 @@ export interface DocumentFacts { } export interface ObservationFacts { + /** Whether visual collection was enabled; partial capture failures remain in issues. */ + readonly visualFactsCollected: boolean; readonly rootFrameId: string; readonly viewport: Viewport; readonly documents: readonly DocumentFacts[]; diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 43c0165..4375cd5 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -26,8 +26,10 @@ import { type NodeFacts, } from "./facts"; import { + BASIC_SNAPSHOT, decodeDocument, type SnapshotDocument, + type SnapshotProfile, type SnapshotReply, snapshotFrameId, } from "./snapshot"; @@ -55,8 +57,9 @@ export async function normalizeDocument( strings: string[], context: FrameContext, signal?: AbortSignal, + profile: SnapshotProfile = BASIC_SNAPSHOT, ): Promise { - const decoded = await decodeDocument(doc, strings, signal); + const decoded = await decodeDocument(doc, strings, signal, profile); const checkpoint = createCaptureCheckpoint(signal); const nodes: NodeFacts[] = []; for (let i = 0; i < decoded.nodes.length; i++) { @@ -101,13 +104,14 @@ export async function normalizeDocument( (Number.parseFloat(opacity) || 0) > 0, }); } - const index = await buildDocumentIndex(nodes, signal); + const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts); return { nodes: nodes.filter( (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), ), index, - ...(context.coordinates && + ...(profile.includeVisualFacts && + context.coordinates && context.projection?.status === "available" && context.targetProjection !== null ? { @@ -144,6 +148,7 @@ export async function normalizeSnapshot( issues: CaptureIssue[], signal?: AbortSignal, rootFrameId?: string, + profile: SnapshotProfile = BASIC_SNAPSHOT, ): Promise { const strings = snapshot.strings ?? []; const raw = snapshot.documents ?? []; @@ -335,12 +340,15 @@ export async function normalizeSnapshot( frameId: frame.frameId, ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null, projection: state, - pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale, + ...(profile.includeVisualFacts + ? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale } + : {}), target, ...(target.sessionId ? { targetProjection } : {}), coordinates, }, signal, + profile, ); result.push({ ...normalized, frame }); } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 88da903..3409541 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -8,6 +8,9 @@ export const REQUESTED_STYLES = [ "cursor", "visibility", "opacity", +] as const; +export const VISUAL_STYLES = [ + ...REQUESTED_STYLES, "display", "overflow-x", "overflow-y", @@ -22,9 +25,14 @@ export const REQUESTED_STYLES = [ "contain", "overflow-clip-margin", ] as const; -const STYLE_COL = Object.fromEntries( - REQUESTED_STYLES.map((name, index) => [name, index]), -) as Record<(typeof REQUESTED_STYLES)[number], number>; + +/** The same profile owns request columns, decoding and visual-only derived facts. */ +export const BASIC_SNAPSHOT = { + includeVisualFacts: false, + computedStyles: REQUESTED_STYLES, +} as const; +export const VISUAL_SNAPSHOT = { includeVisualFacts: true, computedStyles: VISUAL_STYLES } as const; +export type SnapshotProfile = typeof BASIC_SNAPSHOT | typeof VISUAL_SNAPSHOT; /** Sparse array format Chrome uses for infrequently-set per-node fields. */ interface SparseArray { index: number[]; @@ -117,6 +125,7 @@ export async function decodeDocument( doc: SnapshotDocument, strings: string[], signal?: AbortSignal, + profile: SnapshotProfile = BASIC_SNAPSHOT, ): Promise { const checkpoint = createCaptureCheckpoint(signal); const dn = doc.nodes; @@ -184,14 +193,17 @@ export async function decodeDocument( const li = layoutByNode.get(n); const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []); const styles: Record = {}; - for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]); + for (let i = 0; i < profile.computedStyles.length; i++) + styles[profile.computedStyles[i]] = str(strings, styleRow[i]); const layout = li === undefined ? undefined : { boundsSpace: "snapshot-document-layout" as const, bounds: dl?.bounds?.[li], - ...(dl?.clientRects?.[li] ? { clientRect: dl.clientRects[li] } : {}), + ...(profile.includeVisualFacts && dl?.clientRects?.[li] + ? { clientRect: dl.clientRects[li] } + : {}), styles, }; @@ -215,7 +227,8 @@ export async function decodeDocument( backendNodeId, nodeType: dn.nodeType?.[n], parentBackendNodeId, - ...(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null) + ...(profile.includeVisualFacts && + (dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null)) ? { parentMissing: true } : {}), diff --git a/apps/extension/src/tools/vom/visual-discovery.ts b/apps/extension/src/tools/vom/visual-discovery.ts index 70ebe42..43c9b35 100644 --- a/apps/extension/src/tools/vom/visual-discovery.ts +++ b/apps/extension/src/tools/vom/visual-discovery.ts @@ -28,12 +28,19 @@ export interface VisualCandidate { readonly region: Extract; } -export interface VisualDiscoveryIssue { - readonly target: CdpTarget; - readonly frameId: string; - readonly backendNodeId?: number; - readonly reason: VisualIssueReason; -} +export type VisualDiscoveryIssue = + | { + readonly reason: "visual-facts-not-collected"; + readonly target?: never; + readonly frameId?: never; + readonly backendNodeId?: never; + } + | { + readonly target: CdpTarget; + readonly frameId: string; + readonly backendNodeId?: number; + readonly reason: VisualIssueReason; + }; export interface VisualDiscoveryResult { readonly candidates: readonly VisualCandidate[]; @@ -95,6 +102,13 @@ export async function discoverVisualCandidates( signal?: AbortSignal, ): Promise { const checkpoint = createCaptureCheckpoint(signal); + if (!facts.visualFactsCollected) + return { + candidates: [], + complete: false, + captureIssues: facts.issues, + issues: [{ reason: "visual-facts-not-collected" }], + }; let work = 0; const candidates: VisualCandidate[] = []; const issues: VisualDiscoveryIssue[] = []; @@ -112,7 +126,7 @@ export async function discoverVisualCandidates( if (pending) await pending; } if ( - !document.index.ancestryComplete.get(node.backendNodeId) || + !document.index.ancestryComplete?.get(node.backendNodeId) || path.length >= document.index.nodes.size ) { const incomplete = { diff --git a/apps/extension/src/tools/vom/visual-region.ts b/apps/extension/src/tools/vom/visual-region.ts index bf0318d..6f8f3a8 100644 --- a/apps/extension/src/tools/vom/visual-region.ts +++ b/apps/extension/src/tools/vom/visual-region.ts @@ -260,6 +260,8 @@ export function extendVisualContext( }; } +/** A page screenshot rectangle, not an exact mask of visible Canvas pixels. + * Rounded corners are rendered by the browser and do not invalidate this range. */ export type VisualRegionResult = | { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource } | { status: "empty" }