figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(vom): fix border radius incorrectly reject canvas

Browse Source
This commit is contained in:
Ljy-0827
2026-09-10 12:21:19 +08:00
committed by GitHub
parent eb6cc0419b
commit 52a8e65b34
10 changed files with 275 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts
+77 -8
View File
@@ -5,8 +5,8 @@ import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { captureVomObservation } from "../../observation";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot";
import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph";
import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot";
import { discoverVisualCandidates } from "../visual-discovery";
function fixture(
@@ -85,7 +85,8 @@ function fixture(
};
if (method === "DOMSnapshot.captureSnapshot") {
snapshots++;
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
const requested = (params as { computedStyles: readonly string[] }).computedStyles;
expect([REQUESTED_STYLES, VISUAL_STYLES]).toContainEqual(requested);
result = {
strings: [
"#document",
@@ -153,10 +154,14 @@ function fixture(
[0, 0, 1000, 800],
[0, 0, 100, 40],
],
styles: [
[7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11],
[7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11],
],
styles: Array.from({ length: 2 }, () =>
requested.map(
(name) =>
[7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11][
VISUAL_STYLES.indexOf(name as (typeof VISUAL_STYLES)[number])
],
),
),
},
};
}),
@@ -1374,12 +1379,76 @@ describe("AX frame scheduling", () => {
});
describe("visual facts integration", () => {
it("defaults to base facts across targets and shares semantic behavior with visual capture", async () => {
const base = fixture({ canvas: true });
const visual = fixture({ canvas: true });
const baseFacts = await captureObservationFacts(base.cdp, 4);
const visualFacts = await captureObservationFacts(visual.cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
expect(baseFacts.visualFactsCollected).toBe(false);
expect(visualFacts.visualFactsCollected).toBe(true);
expect(baseFacts.documents).toHaveLength(4);
expect(visualFacts.documents).toHaveLength(4);
for (const doc of baseFacts.documents) {
expect(doc.geometry).toBeUndefined();
expect(doc.index.ancestryComplete).toBeUndefined();
for (const node of doc.index.nodes.values()) {
expect(node.parentMissing).toBeUndefined();
expect(node.layout?.clientRect).toBeUndefined();
if (node.layout) expect(Object.keys(node.layout.styles)).toEqual([...REQUESTED_STYLES]);
}
}
// This fixture has no child frame projection measurements. Enabling visual
// collection must preserve that missing evidence, not invent geometry.
expect(
visualFacts.documents.find((doc) => doc.frame.frameId === "main")?.geometry,
).toBeDefined();
expect(visualFacts.documents.filter((doc) => doc.geometry)).toHaveLength(1);
for (const doc of visualFacts.documents) {
expect(doc.index.ancestryComplete).toBeDefined();
for (const node of doc.index.nodes.values())
if (node.layout) {
expect(Object.keys(node.layout.styles)).toEqual([...VISUAL_STYLES]);
expect(Object.keys(node.layout.styles).some((key) => key.includes("radius"))).toBe(false);
}
}
for (const [capture, expected] of [
[base, REQUESTED_STYLES],
[visual, VISUAL_STYLES],
] as const) {
const calls = capture.logs.filter((c) => c.method === "DOMSnapshot.captureSnapshot");
expect(calls).toHaveLength(2);
for (const call of calls) expect(call.params.computedStyles).toEqual(expected);
}
expect(base.logs.map((c) => [c.target, c.method])).toEqual(
visual.logs.map((c) => [c.target, c.method]),
);
const scene = (input: typeof baseFacts) =>
buildSemanticVomScene({
documents: semanticCapture(input).documents,
viewport: input.viewport,
rootFrameId: input.rootFrameId,
excludedBackendNodeIds: semanticCapture(input).captured.excludedBackendNodeIds,
});
expect(scene(baseFacts)).toEqual(scene(visualFacts));
const before = base.logs.length;
expect(await discoverVisualCandidates(baseFacts)).toMatchObject({
complete: false,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(base.logs).toHaveLength(before);
});
it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => {
const { cdp, logs } = fixture({
canvas: true,
frames: [{ frameId: "main", target: { tabId: 4 } }],
});
const facts = await captureObservationFacts(cdp, 4);
const facts = await captureObservationFacts(cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
const before = logs.length;
const result = await discoverVisualCandidates(facts);
expect(logs).toHaveLength(before);
apps/extension/src/tools/vom/__tests__/facts.test.ts
+8 -19
View File
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
import { decodeDocument, REQUESTED_STYLES, VISUAL_SNAPSHOT } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
@@ -80,19 +80,6 @@ describe("document facts", () => {
boundsSpace: "snapshot-document-layout",
bounds: [10, 20, 120, 40],
styles: {
rotate: "auto",
scale: "auto",
perspective: "auto",
clip: "auto",
contain: "auto",
"overflow-clip-margin": "auto",
display: "auto",
"overflow-x": "auto",
"overflow-y": "auto",
transform: "auto",
zoom: "auto",
"clip-path": "auto",
"mask-image": "auto",
position: "static",
"pointer-events": "auto",
cursor: "auto",
@@ -153,9 +140,9 @@ describe("visual ancestry facts", () => {
{ ...node(8, null), nodeType: 1 },
{ ...node(9, null), nodeType: 9, parentMissing: true },
];
const index = await buildDocumentIndex(input.reverse());
for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete.get(id)).toBe(true);
for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete.get(id)).toBe(false);
const index = await buildDocumentIndex(input.reverse(), undefined, true);
for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete!.get(id)).toBe(true);
for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete!.get(id)).toBe(false);
});
it("retains client offsets in their source units and does not disguise missing parent indices", async () => {
@@ -169,11 +156,13 @@ describe("visual ancestry facts", () => {
},
},
["div"],
undefined,
VISUAL_SNAPSHOT,
);
expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]);
expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]);
expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true);
const index = await buildDocumentIndex(decoded.nodes);
expect([...index.ancestryComplete.values()]).toEqual([false, false]);
const index = await buildDocumentIndex(decoded.nodes, undefined, true);
expect([...index.ancestryComplete!.values()]).toEqual([false, false]);
});
});
apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts
+47 -4
View File
@@ -5,7 +5,7 @@ import type { GeometryProjection } from "../../geometry";
import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts";
import type { FrameOwnedAxNode } from "../frame-document";
import { normalizeDocument } from "../normalize";
import { REQUESTED_STYLES } from "../snapshot";
import { VISUAL_SNAPSHOT, VISUAL_STYLES } from "../snapshot";
import { discoverVisualCandidates } from "../visual-discovery";
const defaults: Record<string, string> = {
@@ -86,7 +86,7 @@ async function document(
bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]),
clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]),
styles: layouts.map((n) =>
REQUESTED_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])),
VISUAL_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])),
),
},
},
@@ -108,6 +108,8 @@ async function document(
},
pageScale: options.pageScale ?? 1,
},
undefined,
VISUAL_SNAPSHOT,
);
return {
frame,
@@ -128,6 +130,7 @@ function facts(
issues: ObservationFacts<FrameOwnedAxNode>["issues"] = [],
): ObservationFacts<FrameOwnedAxNode> {
return {
visualFactsCollected: true,
rootFrameId: "main",
documents,
viewport: { width: 1000, height: 800 },
@@ -138,6 +141,46 @@ function facts(
}
describe("Canvas discovery", () => {
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
const empty = facts([]);
expect(await discoverVisualCandidates(empty)).toMatchObject({
complete: true,
candidates: [],
issues: [],
});
expect(await discoverVisualCandidates({ ...empty, visualFactsCollected: false })).toMatchObject(
{ complete: false, candidates: [], issues: [{ reason: "visual-facts-not-collected" }] },
);
const controller = new AbortController();
controller.abort();
await expect(
discoverVisualCandidates({ ...empty, visualFactsCollected: false }, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
});
it("discovers Canvas and frame contents without corner radius facts", async () => {
const own = await document([{ id: 2, parent: 1, styles: {} }]);
expect(await discoverVisualCandidates(facts([own]))).toMatchObject({
candidates: [{ backendNodeId: 2 }],
complete: true,
issues: [],
});
const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]);
const child = await document([{ id: 3, parent: 1 }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 2,
target: { tabId: 1 },
},
});
expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({
candidates: [{ backendNodeId: 3 }],
complete: true,
issues: [],
});
});
it.each([
0.8, 1, 1.25, 2,
])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => {
@@ -383,7 +426,7 @@ describe("Canvas discovery", () => {
]);
for (const parent of [99, 3]) {
doc.index.nodes.get(2)!.parentBackendNodeId = parent;
const index = await buildDocumentIndex([...doc.index.nodes.values()]);
const index = await buildDocumentIndex([...doc.index.nodes.values()], undefined, true);
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("ancestry-incomplete");
@@ -445,7 +488,7 @@ describe("Canvas discovery", () => {
});
input.push(node);
}
const index = await buildDocumentIndex(input);
const index = await buildDocumentIndex(input, undefined, true);
reads = 0;
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(count / 2 + 2);
apps/extension/src/tools/vom/__tests__/visual-region.test.ts
+58
View File
@@ -39,6 +39,64 @@ const ancestor: VisualAncestor = {
const box = { x: 0, y: 0, width: 120, height: 40 };
describe("shared visual region rules", () => {
it.each([
"visible",
"hidden",
"auto",
"scroll",
"clip",
])("keeps %s screenshot bounds independent of rounded styling", (overflow) => {
for (const clipContents of [true, false]) {
const node = {
...ancestor,
styles: { ...styles, "overflow-x": overflow, "overflow-y": overflow },
};
const baseline = extendVisualContext(EMPTY_VISUAL_CONTEXT, node, clipContents);
for (const radius of ["0px", "8px", "50%", "4px 8px"]) {
const rounded = extendVisualContext(
EMPTY_VISUAL_CONTEXT,
{
...node,
styles: {
...node.styles,
"border-top-left-radius": radius,
"border-top-right-radius": radius,
"border-bottom-left-radius": radius,
"border-bottom-right-radius": radius,
},
},
clipContents,
);
expect(rounded).toEqual(baseline);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: rounded,
}),
).toMatchObject({ status: "available" });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "hidden",
context: rounded,
}),
).toEqual({ status: "empty" });
if (clipContents && overflow !== "visible")
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: { x: 80, y: 0, width: 20, height: 20 },
visibility: "visible",
context: rounded,
}),
).toEqual({ status: "empty" });
}
}
});
it("accepts independently supplied normalized facts and preserves clipping policy", () => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
apps/extension/src/tools/vom/capture-coordinator.ts
+19 -3
View File
@@ -22,7 +22,12 @@ import {
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
import {
BASIC_SNAPSHOT,
describeSnapshotFrames,
type SnapshotReply,
VISUAL_SNAPSHOT,
} from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
@@ -69,7 +74,9 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
options: { includeVisualFacts?: boolean } = {},
): Promise<ObservationFacts<T>> {
const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT;
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
@@ -125,7 +132,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
computedStyles: profile.computedStyles,
includePaintOrder: true,
includeDOMRects: true,
});
@@ -146,6 +153,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
issues,
signal,
observed.rootFrameId,
profile,
);
const formsAvailable = await enrichFormControlStates(
scoped,
@@ -384,7 +392,15 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
return {
visualFactsCollected: profile.includeVisualFacts,
rootFrameId,
viewport,
documents: facts,
issues,
startedAt,
finishedAt: Date.now(),
};
}
/** Existing semantic consumers get a narrow view; no raw snapshot
apps/extension/src/tools/vom/facts.ts
+12 -7
View File
@@ -30,7 +30,7 @@ export interface NodeFacts extends CapturedNode {
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly ancestryComplete: ReadonlyMap<number, boolean>;
readonly ancestryComplete?: ReadonlyMap<number, boolean>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
@@ -43,11 +43,12 @@ export interface DecodedDocument {
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
includeVisualFacts = false,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const ancestryComplete = new Map<number, boolean>();
const ancestryComplete = includeVisualFacts ? new Map<number, boolean>() : undefined;
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) {
@@ -84,7 +85,7 @@ export async function buildDocumentIndex<T extends DecodedNode>(
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
complete = !current.parentMissing && current.nodeType === 9;
if (ancestryComplete) complete = !current.parentMissing && current.nodeType === 9;
current = undefined;
break;
}
@@ -92,7 +93,7 @@ export async function buildDocumentIndex<T extends DecodedNode>(
}
if (current && overlayByNode.has(current.backendNodeId)) {
overlay = overlayByNode.get(current.backendNodeId)!;
complete = ancestryComplete.get(current.backendNodeId) === true;
if (ancestryComplete) complete = ancestryComplete.get(current.backendNodeId) === true;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
@@ -102,12 +103,14 @@ export async function buildDocumentIndex<T extends DecodedNode>(
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
complete = complete && !item.parentMissing;
ancestryComplete.set(item.backendNodeId, complete);
if (ancestryComplete) {
complete = complete && !item.parentMissing;
ancestryComplete.set(item.backendNodeId, complete);
}
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, ancestryComplete, excludedBackendNodeIds };
return { nodes, ...(ancestryComplete ? { ancestryComplete } : {}), excludedBackendNodeIds };
}
export interface DocumentIdentity {
@@ -158,6 +161,8 @@ export interface DocumentFacts<T extends FrameOwnedAxNode> {
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
/** Whether visual collection was enabled; partial capture failures remain in issues. */
readonly visualFactsCollected: boolean;
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
apps/extension/src/tools/vom/normalize.ts
+12 -4
View File
@@ -26,8 +26,10 @@ import {
type NodeFacts,
} from "./facts";
import {
BASIC_SNAPSHOT,
decodeDocument,
type SnapshotDocument,
type SnapshotProfile,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
@@ -55,8 +57,9 @@ export async function normalizeDocument(
strings: string[],
context: FrameContext,
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal);
const decoded = await decodeDocument(doc, strings, signal, profile);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
@@ -101,13 +104,14 @@ export async function normalizeDocument(
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
...(context.coordinates &&
...(profile.includeVisualFacts &&
context.coordinates &&
context.projection?.status === "available" &&
context.targetProjection !== null
? {
@@ -144,6 +148,7 @@ export async function normalizeSnapshot(
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedFrameDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
@@ -335,12 +340,15 @@ export async function normalizeSnapshot(
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
projection: state,
pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale,
...(profile.includeVisualFacts
? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale }
: {}),
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
},
signal,
profile,
);
result.push({ ...normalized, frame });
}
apps/extension/src/tools/vom/snapshot.ts
+19 -6
View File
@@ -8,6 +8,9 @@ export const REQUESTED_STYLES = [
"cursor",
"visibility",
"opacity",
] as const;
export const VISUAL_STYLES = [
...REQUESTED_STYLES,
"display",
"overflow-x",
"overflow-y",
@@ -22,9 +25,14 @@ export const REQUESTED_STYLES = [
"contain",
"overflow-clip-margin",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** The same profile owns request columns, decoding and visual-only derived facts. */
export const BASIC_SNAPSHOT = {
includeVisualFacts: false,
computedStyles: REQUESTED_STYLES,
} as const;
export const VISUAL_SNAPSHOT = { includeVisualFacts: true, computedStyles: VISUAL_STYLES } as const;
export type SnapshotProfile = typeof BASIC_SNAPSHOT | typeof VISUAL_SNAPSHOT;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
@@ -117,6 +125,7 @@ export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
@@ -184,14 +193,17 @@ export async function decodeDocument(
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
for (let i = 0; i < profile.computedStyles.length; i++)
styles[profile.computedStyles[i]] = str(strings, styleRow[i]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
...(dl?.clientRects?.[li] ? { clientRect: dl.clientRects[li] } : {}),
...(profile.includeVisualFacts && dl?.clientRects?.[li]
? { clientRect: dl.clientRects[li] }
: {}),
styles,
};
@@ -215,7 +227,8 @@ export async function decodeDocument(
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null)
...(profile.includeVisualFacts &&
(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null))
? { parentMissing: true }
: {}),
apps/extension/src/tools/vom/visual-discovery.ts
+21 -7
View File
@@ -28,12 +28,19 @@ export interface VisualCandidate {
readonly region: Extract<VisualRegionResult, { status: "available" }>;
}
export interface VisualDiscoveryIssue {
readonly target: CdpTarget;
readonly frameId: string;
readonly backendNodeId?: number;
readonly reason: VisualIssueReason;
}
export type VisualDiscoveryIssue =
| {
readonly reason: "visual-facts-not-collected";
readonly target?: never;
readonly frameId?: never;
readonly backendNodeId?: never;
}
| {
readonly target: CdpTarget;
readonly frameId: string;
readonly backendNodeId?: number;
readonly reason: VisualIssueReason;
};
export interface VisualDiscoveryResult {
readonly candidates: readonly VisualCandidate[];
@@ -95,6 +102,13 @@ export async function discoverVisualCandidates(
signal?: AbortSignal,
): Promise<VisualDiscoveryResult> {
const checkpoint = createCaptureCheckpoint(signal);
if (!facts.visualFactsCollected)
return {
candidates: [],
complete: false,
captureIssues: facts.issues,
issues: [{ reason: "visual-facts-not-collected" }],
};
let work = 0;
const candidates: VisualCandidate[] = [];
const issues: VisualDiscoveryIssue[] = [];
@@ -112,7 +126,7 @@ export async function discoverVisualCandidates(
if (pending) await pending;
}
if (
!document.index.ancestryComplete.get(node.backendNodeId) ||
!document.index.ancestryComplete?.get(node.backendNodeId) ||
path.length >= document.index.nodes.size
) {
const incomplete = {
apps/extension/src/tools/vom/visual-region.ts
+2
View File
@@ -260,6 +260,8 @@ export function extendVisualContext(
};
}
/** A page screenshot rectangle, not an exact mask of visible Canvas pixels.
* Rounded corners are rendered by the browser and do not invalidate this range. */
export type VisualRegionResult =
| { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource }
| { status: "empty" }