Compare commits

..
Author SHA1 Message Date
Ljy-0827 fa0952ac1b refactor(vom): preserve complete canvas candidates with exact deduplication 2026-09-09 20:06:30 +08:00
Ljy-0827 f6e7a98508 fix(vom): fix border radius incorrectly reject canvas 2026-09-09 18:27:02 +08:00
Ljy-0827 972232bba9 feat(vom): discover canvas candidates and resolve visible regions 2026-09-09 16:36:10 +08:00
Hannah Liu 896c7944d8 Merge pull request #203 from Tencent/fix/capture-document-consistency
fix(observation): validate DOM identity across capture phases | 校验跨采集阶段的 DOM 身份一致性
2026-09-09 16:26:24 +08:00
Hannah Liu 7e5f3bae26 Merge pull request #200 from Tencent/fix/observation-document-consistency
refactor(observation):  unify cross-frame capture coordination and data ownership | 统一跨 frame 采集协调与数据归属
2026-09-09 16:25:52 +08:00
Hannah Liu af4c2fa229 Merge pull request #198 from Tencent/refactor/document-capture-facts
refactor(observation): decouple DOM processing from capture orchestration | 解耦 DOM 处理与采集流程
2026-09-09 16:24:42 +08:00
Hannah Liu ee10da6fe9 Merge pull request #195 from Tencent/fix/frame-geometry-contracts
fix(geometry): unify cross-frame coordinates and screenshot conversion | 统一跨 frame 坐标与截图转换
2026-09-09 16:24:06 +08:00
drakezhang 4834d8b5fa test(geometry): identify the regression browser attachment 2026-09-09 16:11:39 +08:00
Ljy-0827 b4006d5b85 perf(observation): validate document identity against snapshot roots 2026-09-09 16:09:09 +08:00
Ljy-0827 2d1e5c6d9d fix(observation): validate document consistency with bounded identity checks 2026-09-09 16:09:09 +08:00
20 changed files with 2345 additions and 41 deletions

No files matched your search

@@ -29,6 +29,24 @@ function fakeApi() {
}
describe("ChromiumCdp", () => {
it("changes attachment identity only across real debugger attachments", async () => {
const { api, onEvent, onDetach } = fakeApi();
const cdp = new ChromiumCdp(api);
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
const first = cdp.getAttachmentId(4);
expect(first).toBeTruthy();
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
expect(cdp.getAttachmentId(4)).toBe(first);
onDetach.fire({ tabId: 4 }, "target_closed");
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
expect(cdp.getAttachmentId(4)).not.toBe(first);
await cdp.detach(4);
expect(cdp.getAttachmentId(4)).toBeUndefined();
});
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -164,6 +164,7 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
export class ChromiumCdp {
private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>();
private readonly attachmentIds = new Map<number, string>();
private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>();
@@ -198,6 +199,11 @@ export class ChromiumCdp {
this.bindFrameTargetHandler();
}
/** Identity of this actual debugger attachment, independent of frame topology. */
getAttachmentId(tabId: number): string | undefined {
return this.attachmentIds.get(tabId);
}
/** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching.
@@ -217,6 +223,7 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId);
this.attachmentIds.set(tabId, crypto.randomUUID());
await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
});
@@ -444,6 +451,7 @@ export class ChromiumCdp {
this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId);
this.attachmentIds.delete(tabId);
this.clearDialogState(tabId);
this.clearConsoleState(tabId);
this.clearNetworkState(tabId);
@@ -502,6 +510,7 @@ export class ChromiumCdp {
this.attachInFlight.clear();
this.tabOwners.clear();
this.attachedTabs.clear();
this.attachmentIds.clear();
this.dialogBuffers.clear();
this.dialogSequences.clear();
this.consoleBuffers.clear();
@@ -848,6 +857,7 @@ export class ChromiumCdp {
const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
if (typeof source.tabId === "number") {
this.attachedTabs.delete(source.tabId);
this.attachmentIds.delete(source.tabId);
this.attachInFlight.delete(source.tabId);
this.tabOwners.delete(source.tabId);
this.clearDialogState(source.tabId);
@@ -3695,7 +3695,8 @@ describe("handleSnapshot", () => {
const result = await captureVomObservation(makeFrameAwareDeps().cdp, 4, "https://example.com", {
maxTokens,
});
expect(result.text.match(/@warning/g)).toHaveLength(1);
expect(result.text.match(/@warning/g)).toHaveLength(2);
expect(result.text).toContain("document identity unverified");
expect(result.text).toContain("geometry incomplete");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
@@ -3722,7 +3723,8 @@ describe("handleSnapshot", () => {
for (const maxTokens of [undefined, 100, 0]) {
const result = await captureVomObservation(cdp, 4, "https://example.com", { maxTokens });
expect(result.text).toContain(`@warning observation incomplete: some ${stage} data`);
expect(result.text.match(/@warning/g)).toHaveLength(1);
expect(result.text.match(/@warning/g)).toHaveLength(2);
expect(result.text).toContain("document identity unverified");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.frames.map((frame) => frame.frameId)).toEqual(["main", "child"]);
@@ -3770,7 +3772,8 @@ describe("handleSnapshot", () => {
4,
"https://example.com",
);
expect(result.text).not.toContain("@warning");
expect(result.text).not.toContain("iframe geometry incomplete");
expect(result.text).toContain("document identity unverified");
const node = result.matchNodes.find((item) => item.backendNodeId === 22);
if (ownerGeometry === "clipped") expect(node?.rect).toBeNull();
else expect(node?.rect).toEqual({ x: 120, y: 130, w: 120, h: 40 });
@@ -130,6 +130,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
getAttachmentId: () => rootSession,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
@@ -18,8 +18,13 @@ import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./c
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
cssVisualViewport?: {
zoom?: number;
scale?: number;
clientWidth?: number;
clientHeight?: number;
};
visualViewport?: { zoom?: number; scale?: number; clientWidth?: number; clientHeight?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
+12
View File
@@ -938,6 +938,18 @@ export async function captureVomObservation(
notices.push(
`@warning observation incomplete: some ${incompleteStages.join(", ")} data is unavailable or omitted.`,
);
if (
facts.issues.some(
(issue) => issue.stage === "identity" && issue.reason !== "identity-unverified",
)
)
notices.push(
"@warning observation incomplete: some documents were omitted because their identity changed or could not be revalidated.",
);
if (facts.issues.some((issue) => issue.reason === "identity-unverified"))
notices.push(
"@warning document identity unverified: some retained documents could not be checked for changes during capture.",
);
const captureNotice = notices.join("\n");
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
+1
View File
@@ -55,6 +55,7 @@ export interface CdpRunner {
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
detach?(tabId: number): Promise<void>;
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
getAttachmentId?(tabId: number): string | undefined;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void;
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
@@ -2,13 +2,25 @@ import { describe, expect, it, vi } from "vitest";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { captureVomObservation } from "../../observation";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot";
import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph";
import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot";
import { deduplicateVisualCandidates } from "../visual-dedup";
import { discoverVisualCandidates } from "../visual-discovery";
function fixture(
options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {},
options: {
frames?: CdpFrame[];
canvas?: boolean;
after?: Record<string, { element?: number; missing?: boolean; unreadable?: boolean }>;
fail?: string;
missingIdentity?: boolean;
omitDocument?: string;
attachmentChanged?: boolean;
overlay?: string;
} = {},
) {
const frames = options.frames ?? [
{ frameId: "main", target: { tabId: 4 } },
@@ -34,7 +46,10 @@ function fixture(
const elements = new Map(
frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]),
);
const contexts = new Map(frames.map((frame, i) => [i + 1, frame.frameId]));
const logs: Array<{ target: CdpTarget; method: string; params: Record<string, unknown> }> = [];
let snapshots = 0;
const targetCount = new Set(frames.map((frame) => cdpTargetKey(frame.target))).size;
const send = vi.fn(
async <T>(target: CdpTarget, method: string, params: object = {}): Promise<T> => {
logs.push({ target, method, params: params as Record<string, unknown> });
@@ -42,25 +57,51 @@ function fixture(
if (options.fail === `${target.sessionId ?? "main"}:${method}`)
throw new Error("fixture failure");
let result: unknown = {};
if (method === "Page.createIsolatedWorld")
result = {
executionContextId: frames.findIndex((frame) => frame.frameId === args.frameId) + 1,
};
if (method === "Runtime.evaluate" && args.contextId) {
const id = contexts.get(Number(args.contextId))!;
const change = options.after?.[id];
result =
options.missingIdentity || change?.unreadable
? {}
: {
result: {
deepSerializedValue: change?.missing
? { type: "null" }
: {
type: "node",
value: { backendNodeId: change?.element ?? elements.get(id) },
},
},
};
}
if (method === "Page.getLayoutMetrics")
result = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOMSnapshot.captureSnapshot") {
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
snapshots++;
const requested = (params as { computedStyles: readonly string[] }).computedStyles;
expect([REQUESTED_STYLES, VISUAL_STYLES]).toContainEqual(requested);
result = {
strings: [
"#document",
"html",
"button",
options.canvas ? "canvas" : "button",
OVERLAY_HOST_MARKER_ATTR,
"",
"visible",
"1",
"static",
"auto",
"block",
"none",
"0px",
],
documents: frames
.filter(
@@ -110,10 +151,18 @@ function fixture(
[0, 0, 1000, 800],
[10, 20, 100, 40],
],
styles: [
[7, 8, 8, 5, 6],
[7, 8, 8, 5, 6],
clientRects: [
[0, 0, 1000, 800],
[0, 0, 100, 40],
],
styles: Array.from({ length: 2 }, () =>
requested.map(
(name) =>
[7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11][
VISUAL_STYLES.indexOf(name as (typeof VISUAL_STYLES)[number])
],
),
),
},
};
}),
@@ -137,6 +186,12 @@ function fixture(
},
);
const cdp: CdpRunner = {
getAttachmentId: () =>
options.missingIdentity
? undefined
: options.attachmentChanged && snapshots === targetCount
? "new-attachment"
: "attachment",
getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }),
send: (tabId, method, params) =>
(send as NonNullable<CdpRunner["sendToTarget"]>)({ tabId }, method, params),
@@ -153,7 +208,7 @@ describe("captureObservationFacts", () => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
} as never;
const reply = await original(target, method, params);
@@ -181,6 +236,9 @@ describe("captureObservationFacts", () => {
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
expect(logs.filter((call) => call.method === "DOM.describeNode")).toHaveLength(0);
expect(logs.filter((call) => call.method === "Page.createIsolatedWorld")).toHaveLength(4);
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
@@ -191,6 +249,219 @@ describe("captureObservationFacts", () => {
expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt);
});
it.each([
{ element: 99 },
{ missing: true },
])("isolates changed child identity %j and dependent descendants", async (change) => {
const { cdp } = fixture({ after: { same: change } });
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "same", reason: "document-changed" }),
);
});
it.each([
{ after: { main: { element: 99 } } },
{ attachmentChanged: true },
])("rejects a root replacement or reattachment %j", async (options) => {
await expect(captureObservationFacts(fixture(options).cdp, 4)).rejects.toThrow(
"document identity",
);
});
it.each([
"main",
"same",
])("does not publish a previously identified %s document after identity reads fail", async (frameId) => {
const { cdp } = fixture({ after: { [frameId]: { unreadable: true } } });
if (frameId === "main")
await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity");
else {
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId, reason: "identity-unavailable" }),
);
}
});
it("validates newly discovered snapshot documents without a pre-snapshot identity read", async () => {
const { cdp, logs } = fixture();
const graph = await cdp.getFrameGraph!(4);
cdp.getFrameGraph = async () => ({ ...graph, frames: graph.frames.slice(0, 1) });
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "same", "nested"]);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
expect(logs.filter((call) => call.method === "Page.createIsolatedWorld")).toHaveLength(3);
expect(logs.findIndex((call) => call.method === "Page.createIsolatedWorld")).toBeGreaterThan(
logs.findIndex((call) => call.method === "Accessibility.getFullAXTree"),
);
});
it("rejects a snapshot root that no longer matches the current frame root", async () => {
const { cdp } = fixture();
const original = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
const result = await original(target, method, params);
if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) {
const snapshot = result as {
documents: { frameId: string; nodes: { backendNodeId: number[] } }[];
};
snapshot.documents[0].nodes.backendNodeId[2] = 999;
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity");
});
it("starts after-identity reads only after every target finishes and propagates cancellation during identity cleanup", async () => {
const { cdp, logs } = fixture();
const controller = new AbortController();
const original = cdp.sendToTarget!;
let identityCleanup = 0;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.createIsolatedWorld") {
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(
4,
);
}
const result = await original(target, method, params);
if (
method === "Runtime.releaseObjectGroup" &&
String((params as { objectGroup?: string })?.objectGroup).startsWith(
"bsk-document-identity-",
) &&
target.sessionId === "remote"
) {
if (++identityCleanup === 1) controller.abort();
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(identityCleanup).toBe(1);
});
it.each([
1, 10, 100,
])("bounds identity reads for %i same-target documents after snapshot and AX collection", async (count) => {
const frames: CdpFrame[] = Array.from({ length: count }, (_, i) => ({
frameId: `frame-${i}`,
target: { tabId: 4 },
...(i ? { parentFrameId: "frame-0", ownerBackendNodeId: 90000 + i } : {}),
}));
const { cdp, logs } = fixture({ frames });
const original = cdp.sendToTarget!;
let active = 0;
let peak = 0;
let released = 0;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.createIsolatedWorld") {
peak = Math.max(peak, ++active);
await new Promise((resolve) => setTimeout(resolve, 1));
}
if (method === "DOMSnapshot.captureSnapshot") {
expect(active).toBe(0);
expect(released).toBe(0);
}
if (method === "Page.createIsolatedWorld") {
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(
count,
);
}
const result = await original(target, method, params);
if (
method === "Runtime.releaseObjectGroup" &&
String((params as { objectGroup?: string })?.objectGroup).startsWith(
"bsk-document-identity-",
)
) {
active--;
released++;
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
expect(peak).toBe(Math.min(count, 4));
expect(active).toBe(0);
expect(released).toBe(count);
for (const method of [
"Page.createIsolatedWorld",
"Runtime.evaluate",
"Runtime.releaseObjectGroup",
])
expect(logs.filter((call) => call.method === method)).toHaveLength(count);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(1);
expect(logs.filter((call) => call.method === "Page.getFrameTree")).toHaveLength(0);
});
it("joins identity cleanup on cancellation without starting queued frames", async () => {
const frames: CdpFrame[] = Array.from({ length: 5 }, (_, i) => ({
frameId: `frame-${i}`,
target: { tabId: 4 },
...(i ? { parentFrameId: "frame-0", ownerBackendNodeId: 90000 + i } : {}),
}));
const { cdp, logs } = fixture({ frames });
const original = cdp.sendToTarget!;
const controller = new AbortController();
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
let ready!: () => void;
const blocked = new Promise<void>((resolve) => {
ready = resolve;
});
let held = 0;
cdp.sendToTarget = async (target, method, params) => {
const result = await original(target, method, params);
if (
method === "Runtime.releaseObjectGroup" &&
String((params as { objectGroup?: string })?.objectGroup).startsWith(
"bsk-document-identity-",
)
) {
if (++held === 4) ready();
await gate;
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
let settled = false;
const pending = captureObservationFacts(cdp, 4, controller.signal).finally(() => {
settled = true;
});
const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" });
await blocked;
controller.abort();
await Promise.resolve();
expect(settled).toBe(false);
release();
await rejected;
expect(held).toBe(4);
expect(logs.filter((call) => call.method === "Page.createIsolatedWorld")).toHaveLength(4);
});
it.each([
[{ after: { same: { element: 99 } } }, "omitted", false],
[{ after: { same: { unreadable: true } } }, "omitted", false],
[{ missingIdentity: true }, "document identity unverified", true],
[{}, "", true],
] as const)("publishes identity integrity without silently losing content %j", async (options, notice, retained) => {
const { cdp } = fixture(options);
const result = await captureVomObservation(cdp, 4, "https://example.com");
expect(result.frames.some((frame) => frame.frameId === "same")).toBe(retained);
if (notice) expect(result.text).toContain(notice);
else expect(result.text).not.toContain("identity");
if (!retained) expect(result.text).not.toContain("document identity unverified");
});
it("does not retry failed or missing documents and retains valid AX-only semantics", async () => {
const { cdp, logs } = fixture({
fail: "remote:DOMSnapshot.captureSnapshot",
@@ -209,6 +480,12 @@ describe("captureObservationFacts", () => {
);
});
it("keeps missing identity explicit without manufacturing a verified document", async () => {
const facts = await captureObservationFacts(fixture({ missingIdentity: true }).cdp, 4);
expect(facts.documents.every((doc) => !doc.identity)).toBe(true);
expect(facts.issues.filter((issue) => issue.reason === "identity-unverified")).toHaveLength(4);
});
it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => {
const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4);
const { documents } = semanticCapture(facts);
@@ -396,7 +673,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
@@ -422,7 +699,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
@@ -504,7 +781,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot")
@@ -566,7 +843,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
@@ -656,7 +933,7 @@ function siblingCaptureFixture(
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
@@ -864,7 +1141,7 @@ describe("snapshot document provenance", () => {
: method === "Page.getLayoutMetrics"
? {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 200, clientHeight: 100 },
}
: {}) as T,
@@ -906,7 +1183,7 @@ describe("sibling frame measurement scheduling", () => {
return original(tabId, method, params);
};
const captured = await captureObservationFacts(fixture.cdp, 4);
expect(captured.issues).toEqual([]);
expect(captured.issues.filter((issue) => issue.stage === "geometry")).toEqual([]);
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
@@ -1100,3 +1377,101 @@ describe("AX frame scheduling", () => {
expect(active).toBe(0);
});
});
describe("visual facts integration", () => {
it("defaults to base facts across targets and shares semantic behavior with visual capture", async () => {
const base = fixture({ canvas: true });
const visual = fixture({ canvas: true });
const baseFacts = await captureObservationFacts(base.cdp, 4);
const visualFacts = await captureObservationFacts(visual.cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
expect(baseFacts.visualFactsCollected).toBe(false);
expect(visualFacts.visualFactsCollected).toBe(true);
expect(baseFacts.documents).toHaveLength(4);
expect(visualFacts.documents).toHaveLength(4);
for (const doc of baseFacts.documents) {
expect(doc.geometry).toBeUndefined();
expect(doc.index.ancestryComplete).toBeUndefined();
for (const node of doc.index.nodes.values()) {
expect(node.parentMissing).toBeUndefined();
expect(node.layout?.clientRect).toBeUndefined();
if (node.layout) expect(Object.keys(node.layout.styles)).toEqual([...REQUESTED_STYLES]);
}
}
// This fixture has no child frame projection measurements. Enabling visual
// collection must preserve that missing evidence, not invent geometry.
expect(
visualFacts.documents.find((doc) => doc.frame.frameId === "main")?.geometry,
).toBeDefined();
expect(visualFacts.documents.filter((doc) => doc.geometry)).toHaveLength(1);
for (const doc of visualFacts.documents) {
expect(doc.index.ancestryComplete).toBeDefined();
for (const node of doc.index.nodes.values())
if (node.layout) {
expect(Object.keys(node.layout.styles)).toEqual([...VISUAL_STYLES]);
expect(Object.keys(node.layout.styles).some((key) => key.includes("radius"))).toBe(false);
}
}
for (const [capture, expected] of [
[base, REQUESTED_STYLES],
[visual, VISUAL_STYLES],
] as const) {
const calls = capture.logs.filter((c) => c.method === "DOMSnapshot.captureSnapshot");
expect(calls).toHaveLength(2);
for (const call of calls) expect(call.params.computedStyles).toEqual(expected);
}
expect(base.logs.map((c) => [c.target, c.method])).toEqual(
visual.logs.map((c) => [c.target, c.method]),
);
const scene = (input: typeof baseFacts) =>
buildSemanticVomScene({
documents: semanticCapture(input).documents,
viewport: input.viewport,
rootFrameId: input.rootFrameId,
excludedBackendNodeIds: semanticCapture(input).captured.excludedBackendNodeIds,
});
expect(scene(baseFacts)).toEqual(scene(visualFacts));
const before = base.logs.length;
expect(await discoverVisualCandidates(baseFacts)).toMatchObject({
complete: false,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(base.logs).toHaveLength(before);
});
it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => {
const { cdp, logs } = fixture({
canvas: true,
frames: [{ frameId: "main", target: { tabId: 4 } }],
});
const facts = await captureObservationFacts(cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
const before = logs.length;
const result = await discoverVisualCandidates(facts);
expect(logs).toHaveLength(before);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(1);
expect(result.complete).toBe(true);
expect(result.candidates).toHaveLength(1);
expect(result.candidates[0]).toMatchObject({
document: { frameId: "main", documentElementBackendNodeId: 1 },
region: { crop: { x: 10, y: 20, width: 100, height: 40 } },
});
const deduplicated = await deduplicateVisualCandidates(result);
expect(deduplicated.candidates).toEqual(result.candidates);
expect(deduplicated.candidates[0]).toBe(result.candidates[0]);
expect(deduplicated).toMatchObject({
candidateCount: 1,
deduplicatedCount: 0,
dedupDegraded: false,
});
expect(logs).toHaveLength(before);
expect(facts.documents[0].index.nodes.get(2)?.layout?.clientRect).toEqual([0, 0, 100, 40]);
expect(facts.documents[0].geometry?.pageScale).toBe(1);
const baseline = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] });
await captureObservationFacts(baseline.cdp, 4);
expect(logs.map((call) => call.method)).toEqual(baseline.logs.map((call) => call.method));
});
});
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
import { decodeDocument, REQUESTED_STYLES, VISUAL_SNAPSHOT } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
@@ -125,3 +125,44 @@ describe("document facts", () => {
expect(reads).toBeLessThanOrEqual(356);
});
});
describe("visual ancestry facts", () => {
it("distinguishes document roots, malformed roots, missing parents and cycles", async () => {
const input = [
{ ...node(0, null), nodeType: 9 },
{ ...node(1, 0), nodeType: 1 },
{ ...node(2, 1), nodeType: 11 },
node(3, 2),
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 5),
{ ...node(8, null), nodeType: 1 },
{ ...node(9, null), nodeType: 9, parentMissing: true },
];
const index = await buildDocumentIndex(input.reverse(), undefined, true);
for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete!.get(id)).toBe(true);
for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete!.get(id)).toBe(false);
});
it("retains client offsets in their source units and does not disguise missing parent indices", async () => {
const decoded = await decodeDocument(
{
nodes: { backendNodeId: [1, 2], nodeType: [1, 1], nodeName: [0, 0], parentIndex: [99] },
layout: {
nodeIndex: [0],
bounds: [[745, 455.625, 97, 59.5]],
clientRects: [[5, 5, 68, 38]],
},
},
["div"],
undefined,
VISUAL_SNAPSHOT,
);
expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]);
expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]);
expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true);
const index = await buildDocumentIndex(decoded.nodes, undefined, true);
expect([...index.ancestryComplete!.values()]).toEqual([false, false]);
});
});
@@ -0,0 +1,147 @@
import { describe, expect, it } from "vitest";
import { deduplicateVisualCandidates, MAX_VISUAL_DEDUP_KEYS } from "../visual-dedup";
import type { VisualCandidate, VisualDiscoveryResult } from "../visual-discovery";
function candidate(id: number, width = 100, parent = 1): VisualCandidate {
const crop = { x: 0, y: 0, width, height: 100 };
return {
document: {
attachmentId: "a",
target: { tabId: 1 },
frameId: "main",
documentElementBackendNodeId: 1,
},
backendNodeId: id,
parentBackendNodeId: parent,
region: { status: "available", borderBox: crop, crop },
};
}
function discovery(candidates: VisualCandidate[]): VisualDiscoveryResult {
return { candidates, complete: true, issues: [], captureIssues: [] };
}
describe("visual candidate deduplication", () => {
it("preserves small, unnamed and labelled candidates and their original evidence", async () => {
const small = { ...candidate(3, 1), label: ' A "label"\n😀' };
const large = candidate(2, 1000);
const input = discovery([small, large]);
Object.freeze(input.candidates);
Object.freeze(small);
Object.freeze(large);
for (let i = 0; i < 2; i++) {
const result = await deduplicateVisualCandidates(input);
expect(result.candidates).toEqual([small, large]);
expect(result.candidates[0]).toBe(small);
expect(result.candidates[1]).toBe(large);
expect(result).toMatchObject({
candidateCount: 2,
deduplicatedCount: 0,
dedupDegraded: false,
});
}
});
it("only deduplicates exact crops within the same full DOM identity and direct parent", async () => {
const base = candidate(20);
const variants = [
candidate(30, 100, 2),
{ ...candidate(40), parentBackendNodeId: null },
...["x", "y", "width", "height"].map((axis) => ({
...base,
region: {
...base.region,
crop: {
...base.region.crop,
[axis]: base.region.crop[axis as keyof typeof base.region.crop] + 0.001,
},
},
})),
...[
{ frameId: "child" },
{ target: { tabId: 2 } },
{ target: { tabId: 1, sessionId: "remote" } },
{ documentElementBackendNodeId: 9 },
{ attachmentId: "b" },
].map((identity) => ({ ...base, document: { ...base.document, ...identity } })),
];
const duplicate = { ...candidate(10), label: "Not a new target" };
const result = await deduplicateVisualCandidates(discovery([base, ...variants, duplicate]));
expect(result.candidates).toEqual([base, ...variants]);
expect(result.candidates[0]).toBe(base);
expect(result.deduplicatedCount).toBe(1);
});
it.each([
50_001, 100_000,
])("keeps all %s distinct candidates including the last tiny Canvas", async (count) => {
const input = Array.from({ length: count }, (_, i) => candidate(i + 1, 100, i + 1));
input[count - 1] = candidate(count, 1, count);
// A remembered key still deduplicates; an unremembered key passes through.
input.push({ ...input[0] }, { ...input[count - 1] });
const result = await deduplicateVisualCandidates(discovery(input));
expect(result.candidates).toEqual([...input.slice(0, count), input[count + 1]]);
expect(result.candidates[count - 1]).toBe(input[count - 1]);
expect(result).toMatchObject({
candidateCount: count + 2,
deduplicatedCount: 1,
dedupDegraded: true,
});
});
it("does not report degradation when every key fits", async () => {
const input = Array.from({ length: MAX_VISUAL_DEDUP_KEYS }, (_, i) => candidate(i, 1, i));
input.push({ ...input[0] });
const result = await deduplicateVisualCandidates(discovery(input));
expect(result.candidates).toHaveLength(MAX_VISUAL_DEDUP_KEYS);
expect(result.dedupDegraded).toBe(false);
expect(result.deduplicatedCount).toBe(1);
});
it("preserves upstream unknowns without counting issues as Canvas nodes", async () => {
const input: VisualDiscoveryResult = {
...discovery([candidate(2)]),
complete: false,
issues: [{ reason: "geometry-unavailable", target: { tabId: 1 }, frameId: "missing" }],
captureIssues: [{ stage: "ax", reason: "capture-unavailable", target: { tabId: 1 } }],
};
const result = await deduplicateVisualCandidates(input);
expect(result.complete).toBe(false);
expect(result.candidateCount).toBe(1);
expect(result.issues).toBe(input.issues);
expect(result.captureIssues).toBe(input.captureIssues);
const absent = await deduplicateVisualCandidates({
...input,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(absent).toMatchObject({
complete: false,
candidateCount: 0,
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(await deduplicateVisualCandidates(discovery([]))).toMatchObject({
complete: true,
candidates: [],
candidateCount: 0,
dedupDegraded: false,
});
});
it("drains cancellation through the shared checkpoint", async () => {
const controller = new AbortController();
controller.abort();
await expect(
deduplicateVisualCandidates(discovery([]), controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
const during = new AbortController();
const input = discovery(Array.from({ length: 100_000 }, (_, i) => candidate(i, 100, i)));
const timer = setTimeout(() => during.abort(), 0);
try {
await expect(deduplicateVisualCandidates(input, during.signal)).rejects.toMatchObject({
name: "AbortError",
});
} finally {
clearTimeout(timer);
}
});
});
@@ -0,0 +1,502 @@
import { describe, expect, it } from "vitest";
import type { CdpFrame } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { GeometryProjection } from "../../geometry";
import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts";
import type { FrameOwnedAxNode } from "../frame-document";
import { normalizeDocument } from "../normalize";
import { VISUAL_SNAPSHOT, VISUAL_STYLES } from "../snapshot";
import { discoverVisualCandidates } from "../visual-discovery";
const defaults: Record<string, string> = {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "1",
display: "block",
"overflow-x": "visible",
"overflow-y": "visible",
transform: "none",
zoom: "1",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const projection: GeometryProjection = {
sourceClips: [],
edges: [],
topViewport: { width: 1000, height: 800 },
};
interface Spec {
id: number;
parent?: number;
tag?: string;
styles?: Record<string, string>;
bounds?: number[] | null;
client?: number[];
attrs?: Record<string, string>;
}
async function document(
nodes: Spec[],
options: {
frame?: CdpFrame;
projection?: GeometryProjection;
pageScale?: number;
scrollY?: number;
layoutScale?: number;
} = {},
): Promise<DocumentFacts<FrameOwnedAxNode>> {
const frame = options.frame ?? { frameId: "main", target: { tabId: 1 } };
const strings: string[] = [];
const str = (value: string) => {
const index = strings.indexOf(value);
if (index >= 0) return index;
strings.push(value);
return strings.length - 1;
};
const input: Spec[] = [
{ id: 0, tag: "#document", bounds: null },
{ id: 1, parent: 0, tag: "html", bounds: [0, 0, 1000, 800] },
...nodes,
];
const layouts = input.filter((node) => node.bounds !== null);
const normalized = await normalizeDocument(
{
nodes: {
backendNodeId: input.map((n) => n.id),
nodeType: input.map((n) =>
n.tag === "#document" ? 9 : n.tag === "#document-fragment" ? 11 : 1,
),
parentIndex: input.map((n) =>
n.parent === undefined ? -1 : input.findIndex((p) => p.id === n.parent),
),
nodeName: input.map((n) => str(n.tag ?? "canvas")),
attributes: input.map((n) =>
Object.entries(n.attrs ?? {}).flatMap(([k, v]) => [str(k), str(v)]),
),
},
layout: {
nodeIndex: layouts.map((n) => input.indexOf(n)),
bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]),
clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]),
styles: layouts.map((n) =>
VISUAL_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])),
),
},
},
strings,
{
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
target: frame.target,
projection: {
status: "available",
projection: {
source: { target: frame.target, frameId: frame.frameId },
geometry: options.projection ?? projection,
},
},
coordinates: {
layoutUnitsPerCssPixel: options.layoutScale ?? 1,
scrollCss: { x: 0, y: options.scrollY ?? 0 },
},
pageScale: options.pageScale ?? 1,
},
undefined,
VISUAL_SNAPSHOT,
);
return {
frame,
identity: {
attachmentId: "attached",
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: 1,
},
index: normalized.index,
geometry: normalized.geometry,
domNodes: normalized.nodes,
axNodes: [],
};
}
function facts(
documents: DocumentFacts<FrameOwnedAxNode>[],
issues: ObservationFacts<FrameOwnedAxNode>["issues"] = [],
): ObservationFacts<FrameOwnedAxNode> {
return {
visualFactsCollected: true,
rootFrameId: "main",
documents,
viewport: { width: 1000, height: 800 },
issues,
startedAt: 1,
finishedAt: 2,
};
}
describe("Canvas discovery", () => {
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
const empty = facts([]);
expect(await discoverVisualCandidates(empty)).toMatchObject({
complete: true,
candidates: [],
issues: [],
});
expect(await discoverVisualCandidates({ ...empty, visualFactsCollected: false })).toMatchObject(
{ complete: false, candidates: [], issues: [{ reason: "visual-facts-not-collected" }] },
);
const controller = new AbortController();
controller.abort();
await expect(
discoverVisualCandidates({ ...empty, visualFactsCollected: false }, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
});
it("discovers Canvas and frame contents without corner radius facts", async () => {
const own = await document([{ id: 2, parent: 1, styles: {} }]);
expect(await discoverVisualCandidates(facts([own]))).toMatchObject({
candidates: [{ backendNodeId: 2 }],
complete: true,
issues: [],
});
const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]);
const child = await document([{ id: 3, parent: 1 }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 2,
target: { tabId: 1 },
},
});
expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({
candidates: [{ backendNodeId: 3 }],
complete: true,
issues: [],
});
});
it.each([
0.8, 1, 1.25, 2,
])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => {
const doc = await document(
[
{
id: 2,
parent: 1,
tag: "div",
bounds: [100, 200, 64, 34].map((n) => n * scale),
client: [2, 2, 60, 30],
styles: { "overflow-x": "hidden", "overflow-y": "hidden" },
},
{ id: 3, parent: 2, bounds: [102, 202, 120, 40].map((n) => n * scale) },
],
{ layoutScale: scale, scrollY: 75 },
);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.issues).toEqual([]);
const expected = {
borderBox: { x: 102, y: 127, width: 120, height: 40 },
crop: { x: 102, y: 127, width: 60, height: 30 },
};
for (const box of ["borderBox", "crop"] as const)
for (const key of ["x", "y", "width", "height"] as const)
expect(result.candidates[0].region[box][key]).toBeCloseTo(expected[box][key], 8);
});
it("normalizes a zero-width ancestor origin before one-axis clipping", async () => {
const doc = await document(
[
{
id: 2,
parent: 1,
tag: "div",
bounds: [200, 400, 0, 60],
client: [0, 0, 0, 30],
styles: { "overflow-y": "hidden" },
},
{ id: 3, parent: 2, bounds: [200, 400, 240, 80] },
],
{ layoutScale: 2, scrollY: 75 },
);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.issues).toEqual([]);
expect(result.candidates[0].region.crop).toEqual({ x: 100, y: 125, width: 120, height: 30 });
});
it("keeps unnamed, semantic, aria-hidden, inert, pointer-none and visibility override Canvas", async () => {
const doc = await document([
{ id: 2, parent: 1 },
{ id: 3, parent: 1, attrs: { "aria-label": "Chart" } },
{
id: 4,
parent: 1,
attrs: { "aria-hidden": "true", inert: "" },
styles: { "pointer-events": "none" },
},
{ id: 5, parent: 1, tag: "div", styles: { visibility: "hidden" } },
{ id: 6, parent: 5, styles: { visibility: "visible" } },
{ id: 7, parent: 1, attrs: { hidden: "" }, styles: { display: "block" } },
]);
doc.axNodes.push({ nodeId: "canvas", backendDOMNodeId: 2 } as FrameOwnedAxNode);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.candidates.map((c) => c.backendNodeId)).toEqual([2, 3, 4, 6, 7]);
expect(result.candidates[1].label).toBe("Chart");
expect(result.complete).toBe(true);
});
it("excludes CSS transparent subtrees, hidden/zero/no-layout/fully clipped nodes and overlays", async () => {
const doc = await document([
{ id: 2, parent: 1, tag: "div", styles: { opacity: "0" } },
{ id: 3, parent: 2 },
{ id: 4, parent: 1, styles: { visibility: "hidden" } },
{ id: 5, parent: 1, bounds: [0, 0, 0, 40] },
{ id: 6, parent: 1, bounds: null },
{ id: 7, parent: 1, bounds: [1200, 0, 100, 50] },
{ id: 8, parent: 1, tag: "div", attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } },
{ id: 9, parent: 8, tag: "#document-fragment", bounds: null },
{ id: 10, parent: 9 },
]);
expect(await discoverVisualCandidates(facts([doc]))).toMatchObject({
candidates: [],
issues: [],
complete: true,
});
});
it.each([
["both", "hidden", "hidden", [0, 0, 60, 30], [60, 30]],
["x", "clip", "visible", [0, 0, 60, 20], [60, 40]],
["border", "hidden", "hidden", [5, 5, 68, 38], [64, 34]],
])("uses client box for %s clipping", async (_name, x, y, client, expected) => {
const doc = await document([
{
id: 2,
parent: 1,
tag: "div",
bounds: [0, 0, 78, 48],
client: client as number[],
styles: { "overflow-x": x as string, "overflow-y": y as string },
},
{
id: 3,
parent: 2,
bounds: x === "clip" || _name === "both" ? [0, 0, 120, 40] : [9, 9, 120, 40],
},
]);
const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates;
expect([candidate.region.crop.width, candidate.region.crop.height]).toEqual(expected);
expect(candidate.region.clips?.backendNodeId).toBe(2);
});
it("keeps Canvas border-box and clips partial viewport intersections", async () => {
const doc = await document([{ id: 2, parent: 1, bounds: [950, 20, 130, 50] }]);
const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates;
expect(candidate.region.borderBox.width).toBe(130);
expect(candidate.region.crop).toEqual({ x: 950, y: 20, width: 50, height: 50 });
});
it("propagates owner ancestry across targets, without depending on semantic parents", async () => {
const parent = await document([
{
id: 2,
parent: 1,
tag: "div",
bounds: [0, 0, 200, 100],
client: [0, 0, 200, 100],
styles: { "overflow-x": "hidden", "overflow-y": "hidden" },
},
{ id: 3, parent: 2, tag: "iframe", bounds: [100, 0, 200, 200] },
]);
parent.domNodes.splice(0); // Visual discovery never reads the semantic subset.
const child = await document([{ id: 2, parent: 1, bounds: [0, 0, 200, 200] }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 3,
target: { tabId: 1, sessionId: "remote" },
},
projection: {
...projection,
edges: [
{
sourceViewport: { width: 200, height: 200 },
destinationQuad: [
{ x: 100, y: 0 },
{ x: 300, y: 0 },
{ x: 300, y: 200 },
{ x: 100, y: 200 },
],
},
],
},
});
const result = await discoverVisualCandidates(facts([child, parent]));
expect(result.complete).toBe(true);
expect(result.candidates[0]).toMatchObject({
document: { frameId: "child", target: { sessionId: "remote" } },
region: { crop: { x: 100, y: 0, width: 100, height: 100 } },
});
const owner = parent.index.nodes.get(3)!;
owner.layout!.styles = { ...owner.layout!.styles, visibility: "hidden" };
expect((await discoverVisualCandidates(facts([parent, child]))).candidates).toHaveLength(0);
});
it.each<Record<string, string>>([
{ transform: "matrix(0,1,-1,0,0,0)" },
{ "clip-path": "circle(50%)" },
{ "mask-image": "url(mask)" },
{ zoom: "1.25", "overflow-x": "hidden" },
])("reports unsupported ancestor geometry: %o", async (styles) => {
const doc = await document([
{ id: 2, parent: 1, tag: "div", styles },
{ id: 3, parent: 2 },
]);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("geometry-unsupported");
});
it("supports positive scale without overflow, rejects pinch and non-rectangular frame projection", async () => {
const nodes = [
{
id: 2,
parent: 1,
styles: { transform: "matrix(1.25,0,0,1.25,0,0)" },
bounds: [10, 20, 150, 50],
},
];
expect(
(await discoverVisualCandidates(facts([await document(nodes)]))).candidates,
).toHaveLength(1);
expect(
(await discoverVisualCandidates(facts([await document(nodes, { pageScale: 1.5 })]))).issues[0]
.reason,
).toBe("geometry-unsupported");
const rotated = {
...projection,
edges: [
{
sourceViewport: { width: 200, height: 200 },
destinationQuad: [
{ x: 10, y: 0 },
{ x: 200, y: 10 },
{ x: 190, y: 200 },
{ x: 0, y: 190 },
] as [
{ x: number; y: number },
{ x: number; y: number },
{ x: number; y: number },
{ x: number; y: number },
],
},
],
};
expect(
(await discoverVisualCandidates(facts([await document(nodes, { projection: rotated })])))
.issues[0].reason,
).toBe("geometry-unsupported");
});
it("keeps partial capture separate from valid geometry, rejects unverified identities", async () => {
const doc = await document([{ id: 2, parent: 1 }]);
const partial = facts(
[doc],
[{ target: doc.frame.target, frameId: "main", stage: "ax", reason: "capture-unavailable" }],
);
const result = await discoverVisualCandidates(partial);
expect(result.candidates).toHaveLength(1);
expect(result.captureIssues).toBe(partial.issues);
expect(result.complete).toBe(false);
const unverified = { ...doc, identity: undefined };
const failed = await discoverVisualCandidates(facts([unverified]));
expect(failed.candidates).toHaveLength(0);
expect(failed.issues[0].reason).toBe("identity-unverified");
});
it("does not trust orphan, cyclic, boxless or missing frame-owner ancestry", async () => {
const doc = await document([
{ id: 2, parent: 1, tag: "div" },
{ id: 3, parent: 2 },
]);
for (const parent of [99, 3]) {
doc.index.nodes.get(2)!.parentBackendNodeId = parent;
const index = await buildDocumentIndex([...doc.index.nodes.values()], undefined, true);
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("ancestry-incomplete");
}
const boxless = await document([
{ id: 2, parent: 1, tag: "div", bounds: null },
{ id: 3, parent: 2 },
]);
expect((await discoverVisualCandidates(facts([boxless]))).issues[0].reason).toBe(
"facts-unavailable",
);
const child = {
...doc,
frame: { ...doc.frame, frameId: "child", parentFrameId: "absent", ownerBackendNodeId: 2 },
};
expect((await discoverVisualCandidates(facts([child]))).issues[0].reason).toBe(
"ownership-unresolved",
);
});
it("is deterministic, does not deduplicate or mutate inputs", async () => {
const input = facts([
await document([
{ id: 2, parent: 1 },
{ id: 3, parent: 1 },
]),
]);
const freeze = (value: unknown): void => {
if (!value || typeof value !== "object" || Object.isFrozen(value)) return;
if (value instanceof Map) for (const item of value.values()) freeze(item);
for (const item of Object.values(value)) freeze(item);
Object.freeze(value);
};
freeze(input);
const first = await discoverVisualCandidates(input);
expect(await discoverVisualCandidates(input)).toEqual(first);
expect(first.candidates).toHaveLength(2);
});
it.each([
1000, 10000, 100000,
])("memoizes deep ancestry across %i nodes and checks cancellation", async (count) => {
const doc = await document([{ id: 2, parent: 1 }]);
const template = doc.index.nodes.get(2)!;
let reads = 0;
const input = [doc.index.nodes.get(0)!, doc.index.nodes.get(1)!];
for (let id = 2; id < count + 2; id++) {
const node = {
...template,
backendNodeId: id,
tag: id < count / 2 ? "div" : "canvas",
parentBackendNodeId: id < count / 2 ? id - 1 : count / 2 - 1,
};
Object.defineProperty(node, "parentBackendNodeId", {
get: () => {
reads++;
return id < count / 2 ? id - 1 : count / 2 - 1;
},
});
input.push(node);
}
const index = await buildDocumentIndex(input, undefined, true);
reads = 0;
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(count / 2 + 2);
expect(reads).toBeLessThan(count * 5);
const controller = new AbortController();
controller.abort();
await expect(discoverVisualCandidates(facts([doc]), controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
});
});
@@ -0,0 +1,250 @@
import { describe, expect, it } from "vitest";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualAncestor,
} from "../visual-region";
const styles = {
position: "static",
visibility: "visible",
opacity: "1",
display: "block",
transform: "none",
zoom: "1",
"overflow-x": "visible",
"overflow-y": "visible",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const ancestor: VisualAncestor = {
document: {
attachmentId: "a",
target: { tabId: 1 },
frameId: "main",
documentElementBackendNodeId: 1,
},
backendNodeId: 2,
styles,
clientBox: { x: 5, y: 5, width: 60, height: 30 },
};
const box = { x: 0, y: 0, width: 120, height: 40 };
describe("shared visual region rules", () => {
it.each([
"visible",
"hidden",
"auto",
"scroll",
"clip",
])("keeps %s screenshot bounds independent of rounded styling", (overflow) => {
for (const clipContents of [true, false]) {
const node = {
...ancestor,
styles: { ...styles, "overflow-x": overflow, "overflow-y": overflow },
};
const baseline = extendVisualContext(EMPTY_VISUAL_CONTEXT, node, clipContents);
for (const radius of ["0px", "8px", "50%", "4px 8px"]) {
const rounded = extendVisualContext(
EMPTY_VISUAL_CONTEXT,
{
...node,
styles: {
...node.styles,
"border-top-left-radius": radius,
"border-top-right-radius": radius,
"border-bottom-left-radius": radius,
"border-bottom-right-radius": radius,
},
},
clipContents,
);
expect(rounded).toEqual(baseline);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: rounded,
}),
).toMatchObject({ status: "available" });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "hidden",
context: rounded,
}),
).toEqual({ status: "empty" });
if (clipContents && overflow !== "visible")
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: { x: 80, y: 0, width: 20, height: 20 },
visibility: "visible",
context: rounded,
}),
).toEqual({ status: "empty" });
}
}
});
it("accepts independently supplied normalized facts and preserves clipping policy", () => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, "overflow-x": "hidden" },
});
expect(
resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context }),
).toMatchObject({
status: "available",
crop: { x: 5, y: 0, width: 60, height: 40 },
clips: { overflowX: "hidden", overflowY: "visible" },
});
const differentPolicy = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, "overflow-x": "scroll" },
});
expect(differentPolicy.clip).toEqual(context.clip);
expect(differentPolicy.clips).not.toEqual(context.clips);
});
it("distinguishes fully clipped, missing client geometry and invalid rectangles", () => {
const clipping = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } };
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...clipping,
clientBox: { x: 0, y: 0, width: 0, height: 20 },
});
expect(
resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context })
.status,
).toBe("empty");
const unavailable = extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...clipping, clientBox: null });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: unavailable,
}),
).toEqual({ status: "unavailable", reason: "geometry-unavailable" });
expect(
resolveVisualRegion({
borderBox: { ...box, x: NaN },
frameVisibleBox: box,
visibility: "visible",
context: EMPTY_VISUAL_CONTEXT,
}).status,
).toBe("unavailable");
expect(projectVisualBox({ x: 5, y: 5, width: 0, height: 20 }, [])).toEqual({
x: 5,
y: 5,
width: 0,
height: 20,
});
});
it.each<Record<string, string>>([
{ rotate: "45deg" },
{ scale: "-1 1" },
{ perspective: "100px" },
{ clip: "rect(0px, 20px, 20px, 0px)" },
{ contain: "paint" },
{ "overflow-x": "clip", "overflow-clip-margin": "10px" },
{ transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0.01,0,0,0,1)" },
])("refuses unsupported CSS evidence without changing DOM geometry: %o", (change) => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, ...change },
});
expect(context.issue).toBe("geometry-unsupported");
});
it("accepts axis-preserving matrix3d translation and independent positive scale", () => {
for (const change of [
{ transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,10,20,0,1)" },
{ scale: "1.25" },
])
expect(
extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...ancestor, styles: { ...styles, ...change } })
.issue,
).toBeUndefined();
});
it.each([
"matrix(1,0,0,1,0,0)",
"matrix(1,0,0,1,10,20)",
])("accepts unscaled overflow transform %s", (transform) => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: {
...styles,
transform,
position: "relative",
"overflow-x": "hidden",
"overflow-y": "hidden",
},
});
expect(context.issue).toBeUndefined();
const child = extendVisualContext(
context,
{
...ancestor,
styles: {
...styles,
position: "absolute",
"overflow-x": "clip",
"overflow-y": "clip",
"overflow-clip-margin": "content-box",
},
},
false,
);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: child,
}),
).toMatchObject({ status: "available", crop: { x: 5, y: 5, width: 60, height: 30 } });
});
it("supports positioned overflow containers but refuses uncertain out-of-flow clipping", () => {
const overflow = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } };
const staticParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, overflow);
expect(
extendVisualContext(
staticParent,
{ ...ancestor, styles: { ...styles, position: "absolute" } },
false,
).issue,
).toBe("geometry-unsupported");
const positionedParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...overflow,
styles: { ...overflow.styles, position: "relative" },
});
const child = extendVisualContext(
positionedParent,
{ ...ancestor, styles: { ...styles, position: "absolute" } },
false,
);
expect(child.issue).toBeUndefined();
expect(child.clip).toEqual(positionedParent.clip);
expect(
extendVisualContext(
positionedParent,
{ ...ancestor, styles: { ...styles, position: "fixed" } },
false,
).issue,
).toBe("geometry-unsupported");
});
});
@@ -11,7 +11,8 @@ import {
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import type { CapturedSceneInput } from "./facts";
import { verifyDocumentIdentity } from "./document-identity";
import type { CapturedSceneInput, DocumentIdentity } from "./facts";
import { buildDocumentIndex, type CaptureIssue, type ObservationFacts } from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
@@ -21,11 +22,17 @@ import {
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
import {
BASIC_SNAPSHOT,
describeSnapshotFrames,
type SnapshotReply,
VISUAL_SNAPSHOT,
} from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
snapshotAttachmentId?: string;
documents: NormalizedFrameDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
@@ -67,7 +74,9 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
options: { includeVisualFacts?: boolean } = {},
): Promise<ObservationFacts<T>> {
const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT;
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
@@ -121,8 +130,9 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
computedStyles: profile.computedStyles,
includePaintOrder: true,
includeDOMRects: true,
});
@@ -143,6 +153,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
issues,
signal,
observed.rootFrameId,
profile,
);
const formsAvailable = await enrichFormControlStates(
scoped,
@@ -246,6 +257,53 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
const rootBatch = batches.find((batch) => !batch.target.sessionId);
const rootFrameId =
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
const ownershipInvalid = new Set(invalid);
const identities = new Map<string, DocumentIdentity>();
const selected = new Map(currentFrames.map((frame) => [frame.frameId, frame]));
const checks = batches.flatMap((batch) => {
const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc]));
return batch.frames
.filter(
(frame) =>
!invalid.has(frame.frameId) &&
cdpTargetKey(selected.get(frame.frameId)!.target) === cdpTargetKey(batch.target),
)
.map((frame) => ({ batch, frame, doc: docs.get(frame.frameId) }));
});
await collectTasks(
checks,
async ({ batch, frame, doc }) => {
throwCaptureAborted(signal);
if (!batch.snapshotAttachmentId || doc?.documentElementBackendNodeId === undefined) {
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "identity-unverified",
});
return;
}
const identity: DocumentIdentity = {
attachmentId: batch.snapshotAttachmentId,
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: doc.documentElementBackendNodeId,
};
const verified = await verifyDocumentIdentity(cdp, identity, signal);
const status = cdp.getAttachmentId?.(tabId) === identity.attachmentId ? verified : "changed";
if (status === "current") identities.set(frame.frameId, identity);
else {
invalid.add(frame.frameId);
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: status === "changed" ? "document-changed" : "identity-unavailable",
});
}
},
signal,
);
const children = new Map<string, string[]>();
for (const frame of currentFrames) {
if (!frame.parentFrameId) continue;
@@ -263,7 +321,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
}
}
for (const frame of currentFrames) {
if (invalid.has(frame.frameId))
if (ownershipInvalid.has(frame.frameId))
issues.push({
target: frame.target,
frameId: frame.frameId,
@@ -271,7 +329,12 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
reason: "frame-ownership-unresolved",
});
}
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
if (invalid.has(rootFrameId))
throw new Error(
ownershipInvalid.has(rootFrameId)
? "observation root document ownership is ambiguous"
: "observation document identity changed or could not be verified; observe again",
);
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const belongs = (frame: CdpFrame) => {
@@ -319,6 +382,8 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
} = document;
facts.push({
frame,
identity: identities.get(document.frameId),
...(doc?.geometry ? { geometry: doc.geometry } : {}),
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
@@ -327,7 +392,15 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
return {
visualFactsCollected: profile.includeVisualFacts,
rootFrameId,
viewport,
documents: facts,
issues,
startedAt,
finishedAt: Date.now(),
};
}
/** Existing semantic consumers get a narrow view; no raw snapshot
@@ -0,0 +1,53 @@
import type { CdpRunner } from "../shared";
import { sendToCdpTarget } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { DocumentIdentity } from "./facts";
/** Compare the snapshot root with the current root in that exact frame.
* Deep serialization supplies the backend ID without a separate describeNode. */
export async function verifyDocumentIdentity(
cdp: CdpRunner,
identity: DocumentIdentity,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
throwIfAborted(signal);
if (!attached()) return "changed";
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
const send = <T>(method: string, params: object) => {
throwIfAborted(signal);
return sendToCdpTarget<T>(cdp, identity.target, method, params);
};
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: identity.frameId,
worldName: "bsk-document-identity",
});
const reply = await send<{
result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
}>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
objectGroup,
serializationOptions: {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
if (!attached()) return "changed";
const root = reply.result?.deepSerializedValue;
if (root?.type === "null") return "changed";
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
if (id === undefined) return "unavailable";
return id === identity.documentElementBackendNodeId ? "current" : "changed";
} catch (error) {
throwIfAborted(signal);
if (isAbortError(error)) throw error;
return attached() ? "unavailable" : "changed";
} finally {
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
+46 -5
View File
@@ -1,7 +1,8 @@
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import type { GeometryProjection } from "../geometry";
import type { FrameProjectionIssue, SnapshotCoordinates } from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
@@ -10,21 +11,26 @@ import type { FrameOwnedAxNode } from "./frame-document";
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-layout";
bounds?: number[];
/** [clientLeft, clientTop, clientWidth, clientHeight], untransformed local CSS units. */
clientRect?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly ancestryComplete?: ReadonlyMap<number, boolean>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
@@ -37,10 +43,12 @@ export interface DecodedDocument {
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
includeVisualFacts = false,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const ancestryComplete = includeVisualFacts ? new Map<number, boolean>() : undefined;
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) {
@@ -61,6 +69,7 @@ export async function buildDocumentIndex<T extends DecodedNode>(
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
let complete = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
@@ -76,13 +85,16 @@ export async function buildDocumentIndex<T extends DecodedNode>(
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
if (ancestryComplete) complete = !current.parentMissing && current.nodeType === 9;
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId))
if (current && overlayByNode.has(current.backendNodeId)) {
overlay = overlayByNode.get(current.backendNodeId)!;
if (ancestryComplete) complete = ancestryComplete.get(current.backendNodeId) === true;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
@@ -91,10 +103,21 @@ export async function buildDocumentIndex<T extends DecodedNode>(
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (ancestryComplete) {
complete = complete && !item.parentMissing;
ancestryComplete.set(item.backendNodeId, complete);
}
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, excludedBackendNodeIds };
return { nodes, ...(ancestryComplete ? { ancestryComplete } : {}), excludedBackendNodeIds };
}
export interface DocumentIdentity {
attachmentId: string;
target: CdpTarget;
frameId: string;
documentElementBackendNodeId: number;
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
@@ -110,18 +133,36 @@ export interface CaptureIssue {
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "ownership" | "geometry" | "forms";
reason: "capture-unavailable" | "frame-ownership-unresolved" | "geometry-unavailable";
stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms";
reason:
| "document-changed"
| "identity-unavailable"
| "identity-unverified"
| "capture-unavailable"
| "frame-ownership-unresolved"
| "geometry-unavailable";
}
/** Shared per document; never retain the live GeometryContext in published facts. */
export interface DocumentGeometry {
readonly projections: readonly GeometryProjection[];
readonly coordinates: SnapshotCoordinates;
/** Visual viewport scale (pinch), distinct from browser UI zoom. */
readonly pageScale?: number;
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly geometry?: DocumentGeometry;
readonly frame: CdpFrame;
readonly identity?: DocumentIdentity;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
/** Whether visual collection was enabled; partial capture failures remain in issues. */
readonly visualFactsCollected: boolean;
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
+41 -3
View File
@@ -18,10 +18,18 @@ import {
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
type DocumentGeometry,
type DocumentIndex,
type NodeFacts,
} from "./facts";
import {
BASIC_SNAPSHOT,
decodeDocument,
type SnapshotDocument,
type SnapshotProfile,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
@@ -33,11 +41,14 @@ export interface FrameContext {
targetProjection?: GeometryProjection | null;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
pageScale?: number;
}
export interface NormalizedDocument {
geometry?: DocumentGeometry;
nodes: NodeFacts[];
index: DocumentIndex;
documentElementBackendNodeId?: number;
}
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
@@ -46,8 +57,9 @@ export async function normalizeDocument(
strings: string[],
context: FrameContext,
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal);
const decoded = await decodeDocument(doc, strings, signal, profile);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
@@ -92,12 +104,33 @@ export async function normalizeDocument(
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
...(profile.includeVisualFacts &&
context.coordinates &&
context.projection?.status === "available" &&
context.targetProjection !== null
? {
geometry: {
projections: [
context.projection.projection.geometry,
...(context.targetProjection ? [context.targetProjection] : []),
],
coordinates: context.coordinates,
pageScale: context.pageScale,
},
}
: {}),
documentElementBackendNodeId: nodes.find(
(node) =>
node.nodeType === 1 &&
node.parentBackendNodeId !== null &&
index.nodes.get(node.parentBackendNodeId)?.nodeType === 9,
)?.backendNodeId,
};
}
@@ -115,6 +148,7 @@ export async function normalizeSnapshot(
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedFrameDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
@@ -306,11 +340,15 @@ export async function normalizeSnapshot(
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
projection: state,
...(profile.includeVisualFacts
? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale }
: {}),
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
},
signal,
profile,
);
result.push({ ...normalized, frame });
}
+35 -4
View File
@@ -9,9 +9,30 @@ export const REQUESTED_STYLES = [
"visibility",
"opacity",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export const VISUAL_STYLES = [
...REQUESTED_STYLES,
"display",
"overflow-x",
"overflow-y",
"transform",
"zoom",
"clip-path",
"mask-image",
"rotate",
"scale",
"perspective",
"clip",
"contain",
"overflow-clip-margin",
] as const;
/** The same profile owns request columns, decoding and visual-only derived facts. */
export const BASIC_SNAPSHOT = {
includeVisualFacts: false,
computedStyles: REQUESTED_STYLES,
} as const;
export const VISUAL_SNAPSHOT = { includeVisualFacts: true, computedStyles: VISUAL_STYLES } as const;
export type SnapshotProfile = typeof BASIC_SNAPSHOT | typeof VISUAL_SNAPSHOT;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
@@ -49,6 +70,7 @@ export interface SnapshotDocument {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
clientRects?: number[][];
paintOrders?: number[];
};
}
@@ -103,6 +125,7 @@ export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
@@ -170,13 +193,17 @@ export async function decodeDocument(
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
for (let i = 0; i < profile.computedStyles.length; i++)
styles[profile.computedStyles[i]] = str(strings, styleRow[i]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
...(profile.includeVisualFacts && dl?.clientRects?.[li]
? { clientRect: dl.clientRects[li] }
: {}),
styles,
};
@@ -200,6 +227,10 @@ export async function decodeDocument(
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(profile.includeVisualFacts &&
(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null))
? { parentMissing: true }
: {}),
tag,
attrs,
@@ -0,0 +1,64 @@
import { createCaptureCheckpoint } from "./capture-abort";
import type { VisualCandidate, VisualDiscoveryResult } from "./visual-discovery";
export const MAX_VISUAL_DEDUP_KEYS = 50_000;
export interface VisualDedupResult extends VisualDiscoveryResult {
/** Known valid candidates in this capture, not the total Canvas count on the page. */
readonly candidateCount: number;
/** Candidates actually removed by exact coverage deduplication. */
readonly deduplicatedCount: number;
readonly dedupDegraded: boolean;
}
/** Keep the first real anchor for each exact DOM/parent/crop key, in encounter order.
* Once the key set is full, unknown keys pass through: capacity never drops candidates. */
export async function deduplicateVisualCandidates(
discovery: VisualDiscoveryResult,
signal?: AbortSignal,
): Promise<VisualDedupResult> {
const checkpoint = createCaptureCheckpoint(signal);
const keys = new Set<string>();
const candidates: VisualCandidate[] = [];
let deduplicatedCount = 0;
let dedupDegraded = false;
for (let i = 0; i < discovery.candidates.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const candidate = discovery.candidates[i];
const {
document,
region: { crop },
} = candidate;
const key = JSON.stringify([
document.attachmentId,
document.target.tabId,
document.target.sessionId ?? null,
document.frameId,
document.documentElementBackendNodeId,
candidate.parentBackendNodeId,
crop.x,
crop.y,
crop.width,
crop.height,
]);
if (keys.has(key)) {
deduplicatedCount++;
continue;
}
if (keys.size < MAX_VISUAL_DEDUP_KEYS) keys.add(key);
else dedupDegraded = true;
candidates.push(candidate);
}
const pending = checkpoint();
if (pending) await pending;
return {
...discovery,
candidates,
candidateCount: discovery.candidates.length,
deduplicatedCount,
dedupDegraded,
};
}
@@ -0,0 +1,321 @@
import { type CdpTarget } from "@/browser-driver/frame-graph";
import type { ViewportRect } from "../geometry";
import { createCaptureCheckpoint } from "./capture-abort";
import type {
CaptureIssue,
DocumentFacts,
DocumentIdentity,
NodeFacts,
ObservationFacts,
} from "./facts";
import type { FrameOwnedAxNode } from "./frame-document";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualContext,
type VisualIssueReason,
type VisualRegionResult,
visualProjectionIssue,
} from "./visual-region";
export interface VisualCandidate {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly parentBackendNodeId: number | null;
readonly label?: string;
readonly region: Extract<VisualRegionResult, { status: "available" }>;
}
export type VisualDiscoveryIssue =
| {
readonly reason: "visual-facts-not-collected";
readonly target?: never;
readonly frameId?: never;
readonly backendNodeId?: never;
}
| {
readonly target: CdpTarget;
readonly frameId: string;
readonly backendNodeId?: number;
readonly reason: VisualIssueReason;
};
export interface VisualDiscoveryResult {
readonly candidates: readonly VisualCandidate[];
readonly issues: readonly VisualDiscoveryIssue[];
/** Preserve upstream failures, including those that do not invalidate Canvas geometry. */
readonly captureIssues: readonly CaptureIssue[];
readonly complete: boolean;
}
type Document = DocumentFacts<FrameOwnedAxNode>;
interface NodeContext {
self: VisualContext;
children: VisualContext;
}
function viewportRect(
rect: { x: number; y: number; w: number; h: number } | null | undefined,
): ViewportRect | null {
return rect ? { x: rect.x, y: rect.y, width: rect.w, height: rect.h } : null;
}
/** Client offsets are unscaled local CSS units. Only ordinary, untransformed
* overflow chains consume this adapter; other combinations are rejected by policy. */
function clientBox(node: NodeFacts, document: Document): ViewportRect | null {
const client = node.layout?.clientRect;
const bounds = node.layout?.bounds;
const border =
node.localRect ??
(bounds?.length === 4 && bounds.every(Number.isFinite) && document.geometry
? {
x:
bounds[0] / document.geometry.coordinates.layoutUnitsPerCssPixel -
document.geometry.coordinates.scrollCss.x,
y:
bounds[1] / document.geometry.coordinates.layoutUnitsPerCssPixel -
document.geometry.coordinates.scrollCss.y,
}
: null);
if (
!client ||
client.length !== 4 ||
!client.every(Number.isFinite) ||
client[2] < 0 ||
client[3] < 0 ||
!border ||
!document.geometry
)
return null;
return projectVisualBox(
{ x: border.x + client[0], y: border.y + client[1], width: client[2], height: client[3] },
document.geometry.projections,
);
}
/** Pure Facts consumer: no CDP, ref registration, selection, or semantic keep/drop.
* Both frame and node ancestry are memoized in this call, using iterative walks. */
export async function discoverVisualCandidates(
facts: ObservationFacts<FrameOwnedAxNode>,
signal?: AbortSignal,
): Promise<VisualDiscoveryResult> {
const checkpoint = createCaptureCheckpoint(signal);
if (!facts.visualFactsCollected)
return {
candidates: [],
complete: false,
captureIssues: facts.issues,
issues: [{ reason: "visual-facts-not-collected" }],
};
let work = 0;
const candidates: VisualCandidate[] = [];
const issues: VisualDiscoveryIssue[] = [];
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
const contexts = new Map<string, Map<number, NodeContext>>();
const roots = new Map<string, VisualContext>();
async function nodeContext(document: Document, id: number): Promise<NodeContext> {
const cache = contexts.get(document.frame.frameId)!;
const path: NodeFacts[] = [];
let node = document.index.nodes.get(id);
while (node && !cache.has(node.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (
!document.index.ancestryComplete?.get(node.backendNodeId) ||
path.length >= document.index.nodes.size
) {
const incomplete = {
...roots.get(document.frame.frameId)!,
issue: roots.get(document.frame.frameId)!.issue ?? ("ancestry-incomplete" as const),
};
cache.set(node.backendNodeId, { self: incomplete, children: incomplete });
break;
}
path.push(node);
node =
node.parentBackendNodeId === null
? undefined
: document.index.nodes.get(node.parentBackendNodeId);
}
let parent = node
? cache.get(node.backendNodeId)!.children
: roots.get(document.frame.frameId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const current = path[i];
let self = parent;
let children = parent;
if (document.index.excludedBackendNodeIds.has(current.backendNodeId)) {
self = children = { ...parent, hidden: true };
} else if (current.nodeType === 1) {
if (!current.layout) {
// An element without layout is not evidence that descendants are hidden
// (e.g. a boxless ancestor). Keep the missing style evidence explicit.
self = children = { ...parent, issue: parent.issue ?? "facts-unavailable" };
} else if (document.identity) {
const ancestor = {
document: document.identity,
backendNodeId: current.backendNodeId,
styles: current.layout.styles,
clientBox: clientBox(current, document),
};
children = extendVisualContext(
parent,
ancestor,
current.tag !== "iframe" && current.tag !== "frame",
);
self = current.tag === "canvas" ? extendVisualContext(parent, ancestor, false) : children;
}
}
const result = { self, children };
cache.set(current.backendNodeId, result);
parent = children;
}
return (
cache.get(id) ?? {
self: { ...parent, issue: "ancestry-incomplete" },
children: { ...parent, issue: "ancestry-incomplete" },
}
);
}
// Process parent documents first, independent of snapshot/target completion order.
for (const document of facts.documents) {
const path: Document[] = [];
const visiting = new Set<string>();
let current: Document | undefined = document;
while (current && !roots.has(current.frame.frameId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.frame.frameId)) {
roots.set(current.frame.frameId, {
...EMPTY_VISUAL_CONTEXT,
issue: "ownership-unresolved",
});
contexts.set(current.frame.frameId, new Map());
break;
}
visiting.add(current.frame.frameId);
path.push(current);
current = current.frame.parentFrameId
? documents.get(current.frame.parentFrameId)
: undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
const doc = path[i];
const { frame } = doc;
let root: VisualContext = roots.get(frame.frameId) ?? EMPTY_VISUAL_CONTEXT;
if (frame.parentFrameId) {
const parent = documents.get(frame.parentFrameId);
if (
!parent ||
!roots.has(parent.frame.frameId) ||
frame.ownerBackendNodeId === undefined ||
!parent.identity
) {
root = { ...root, issue: "ownership-unresolved" };
} else {
const owner = parent.index.nodes.get(frame.ownerBackendNodeId);
const ownerState = await nodeContext(parent, frame.ownerBackendNodeId);
root = {
...ownerState.children,
// CSS visibility inside a child document cannot override its hidden owner.
hidden:
ownerState.children.hidden ||
owner?.layout?.styles.visibility === "hidden" ||
owner?.layout?.styles.visibility === "collapse",
// Frame scaling is already interpreted by the shared content-quad projection.
transformed: false,
localClip: false,
unpositionedClip: false,
};
}
} else if (frame.frameId !== facts.rootFrameId)
root = { ...root, issue: "ownership-unresolved" };
root = {
...root,
issue:
root.issue ??
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
};
roots.set(frame.frameId, root);
contexts.set(frame.frameId, new Map());
}
}
for (const document of facts.documents) {
for (const node of document.index.nodes.values()) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (node.tag !== "canvas" || document.index.excludedBackendNodeIds.has(node.backendNodeId))
continue;
// No layout/zero size is positive evidence of no screenshot area for this node.
if (!node.layout) continue;
const bounds = node.layout.bounds;
if (
bounds?.length === 4 &&
bounds.every(Number.isFinite) &&
(bounds[2] <= 0 || bounds[3] <= 0)
)
continue;
const context = (await nodeContext(document, node.backendNodeId)).self;
const local = viewportRect(node.localRect);
const borderBox =
local && document.geometry ? projectVisualBox(local, document.geometry.projections) : null;
const region = resolveVisualRegion({
borderBox,
frameVisibleBox: viewportRect(node.rect),
context,
visibility: node.layout.styles.visibility,
});
if (region.status === "unavailable") {
issues.push({
target: document.frame.target,
frameId: document.frame.frameId,
backendNodeId: node.backendNodeId,
reason: region.reason,
});
} else if (region.status === "available" && document.identity) {
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
candidates.push({
document: document.identity,
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
...(label ? { label } : {}),
region,
});
}
}
}
// AX-only/missing documents have no Canvas nodes to carry the missing evidence.
const reportedFrames = new Set(issues.map((issue) => issue.frameId));
for (const document of facts.documents) {
const reason = roots.get(document.frame.frameId)?.issue;
if (
reason &&
!reportedFrames.has(document.frame.frameId) &&
(!document.identity || !document.index.nodes.size)
)
issues.push({ target: document.frame.target, frameId: document.frame.frameId, reason });
}
const pending = checkpoint();
if (pending) await pending;
return {
candidates,
issues,
captureIssues: facts.issues,
complete: issues.length === 0 && facts.issues.length === 0,
};
}
@@ -0,0 +1,318 @@
import {
type GeometryProjection,
type Polygon,
projectPolygon,
rectPolygon,
regionBounds,
type ViewportRect,
} from "../geometry";
import type { DocumentGeometry, DocumentIdentity } from "./facts";
export type VisualIssueReason =
| "ancestry-incomplete"
| "facts-unavailable"
| "geometry-unavailable"
| "geometry-unsupported"
| "identity-unverified"
| "ownership-unresolved";
/** Top viewport CSS coordinates. Unconstrained axes have infinite bounds. */
export interface VisualClip {
readonly left: number;
readonly top: number;
readonly right: number;
readonly bottom: number;
}
/** Shared persistent chain: one entry per clipping ancestor, not per Canvas. */
export interface VisualClipSource {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly x: boolean;
readonly y: boolean;
readonly overflowX: string;
readonly overflowY: string;
readonly box: ViewportRect;
readonly parent?: VisualClipSource;
}
export interface VisualContext {
readonly clip: VisualClip;
readonly clips?: VisualClipSource;
readonly hidden: boolean;
/** A local CSS scale/zoom makes snapshot client offsets ambiguous. */
readonly transformed: boolean;
readonly localClip?: boolean;
/** Overflow between an absolute target and its nearest positioned ancestor. */
readonly unpositionedClip?: boolean;
readonly issue?: VisualIssueReason;
}
export const EMPTY_VISUAL_CONTEXT: VisualContext = {
clip: { left: -Infinity, top: -Infinity, right: Infinity, bottom: Infinity },
hidden: false,
transformed: false,
};
function axisRect(polygon: Polygon): boolean {
if (polygon.length !== 4 || polygon.some((p) => !Number.isFinite(p.x) || !Number.isFinite(p.y)))
return false;
const [a, b, c, d] = polygon;
return a.y === b.y && b.x === c.x && c.y === d.y && d.x === a.x && b.x > a.x && d.y > a.y;
}
/** Check evidence before projectRectToViewport can reduce a polygon to its bounds. */
export function visualProjectionIssue(
geometry: DocumentGeometry | undefined,
): VisualIssueReason | undefined {
if (!geometry || !geometry.projections.length || geometry.pageScale === undefined)
return "geometry-unavailable";
if (!Number.isFinite(geometry.pageScale) || geometry.pageScale <= 0)
return "geometry-unavailable";
if (geometry.pageScale !== 1) return "geometry-unsupported";
const { layoutUnitsPerCssPixel, scrollCss } = geometry.coordinates;
if (
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0 ||
![scrollCss.x, scrollCss.y].every(Number.isFinite)
)
return "geometry-unavailable";
for (const projection of geometry.projections) {
if (
![projection.topViewport.width, projection.topViewport.height].every(
(n) => Number.isFinite(n) && n > 0,
)
)
return "geometry-unavailable";
if (projection.sourceClips.some((clip) => !axisRect(clip))) return "geometry-unsupported";
for (const edge of projection.edges) {
if (
![edge.sourceViewport.width, edge.sourceViewport.height].every(
(n) => Number.isFinite(n) && n > 0,
)
)
return "geometry-unavailable";
if (!axisRect(edge.destinationQuad) || edge.destinationClips?.some((clip) => !axisRect(clip)))
return "geometry-unsupported";
}
}
}
/** Uses the same projection math as DOM geometry, without clipping away box origins. */
export function projectVisualBox(
box: ViewportRect,
projections: readonly GeometryProjection[],
): ViewportRect | null {
let polygon: Polygon = rectPolygon({ x: box.x, y: box.y, w: box.width, h: box.height });
for (const projection of projections)
for (const edge of projection.edges) polygon = projectPolygon(polygon, edge);
// Preserve zero-size client boxes: they can prove a fully clipped axis.
if (box.width === 0 || box.height === 0) {
const [a, , c] = polygon;
return { x: a.x, y: a.y, width: c.x - a.x, height: c.y - a.y };
}
return regionBounds([polygon]);
}
function axisTransformScale(value: string): [number, number] | null {
if (value === "none") return [1, 1];
const match = /^matrix(3d)?\(([^)]+)\)$/.exec(value);
if (!match) return null;
const values = match[2].split(",").map((part) => Number(part.trim()));
if (!values.every(Number.isFinite)) return null;
if (match[1])
return values.length === 16 &&
values[0] > 0 &&
values[5] > 0 &&
values[10] === 1 &&
values[15] === 1 &&
[1, 2, 3, 4, 6, 7, 8, 9, 11, 14].every((index) => values[index] === 0)
? [values[0], values[5]]
: null;
return values.length === 6 && values[0] > 0 && values[3] > 0 && values[1] === 0 && values[2] === 0
? [values[0], values[3]]
: null;
}
export interface VisualAncestor {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly styles: Readonly<Record<string, string>>;
/** Already in top viewport CSS units, from a source-specific adapter. */
readonly clientBox?: ViewportRect | null;
}
/** One ancestor step, reusable by discovery and the future live local adapter. */
export function extendVisualContext(
parent: VisualContext,
node: VisualAncestor,
clipContents = true,
): VisualContext {
const styles = node.styles;
const opacity = styles.opacity?.trim() ? Number(styles.opacity) : NaN;
const zoom = styles.zoom === "normal" ? 1 : styles.zoom?.trim() ? Number(styles.zoom) : NaN;
const scale =
styles.scale === "none" ? [1] : (styles.scale ?? "").trim().split(/\s+/).map(Number);
const scaleSupported =
scale.length >= 1 &&
scale.length <= 3 &&
scale.every((n) => Number.isFinite(n) && n > 0) &&
(scale.length < 3 || scale[2] === 1);
const transformScale = axisTransformScale(styles.transform ?? "");
// Identity/translation changes origins, which bounds already capture, but
// leaves client offsets in the same units. Only scaling makes them ambiguous.
const transformed =
parent.transformed ||
transformScale?.some((n) => n !== 1) === true ||
zoom !== 1 ||
scale.some((n) => n !== 1);
const hidden = parent.hidden || opacity === 0 || styles.display === "none";
let issue = parent.issue;
if (
!Number.isFinite(opacity) ||
!Number.isFinite(zoom) ||
zoom <= 0 ||
!styles.display ||
!styles.visibility
)
issue ??= "facts-unavailable";
if (!styles.transform || !styles["clip-path"] || !styles["mask-image"])
issue ??= "facts-unavailable";
else if (!transformScale || styles["clip-path"] !== "none" || styles["mask-image"] !== "none")
issue ??= "geometry-unsupported";
if (
!["rotate", "scale", "perspective", "clip", "contain", "overflow-clip-margin"].every(
(key) => styles[key],
)
)
issue ??= "facts-unavailable";
else if (
!scaleSupported ||
!["none", "0deg"].includes(styles.rotate) ||
styles.perspective !== "none" ||
styles.clip !== "auto" ||
/(?:^|\s)(paint|strict|content)(?:$|\s)/.test(styles.contain)
)
issue ??= "geometry-unsupported";
// Do not pretend every DOM ancestor clips out-of-flow descendants. Support
// the ordinary positioned-container case; other containing-block cases stay explicit.
if (
(styles.position === "absolute" && parent.unpositionedClip) ||
(styles.position === "fixed" && parent.localClip)
)
issue ??= "geometry-unsupported";
const overflow = [styles["overflow-x"], styles["overflow-y"]];
if (clipContents && overflow.includes("clip") && styles["overflow-clip-margin"] !== "0px")
issue ??= "geometry-unsupported";
if (
overflow.some(
(value) => !["visible", "hidden", "clip", "scroll", "auto", "overlay"].includes(value),
)
)
issue ??= "facts-unavailable";
const [x, y] = overflow.map((value) => value !== "visible");
let clip = parent.clip;
let clips = parent.clips;
if (clipContents && (x || y)) {
if (transformed) issue ??= "geometry-unsupported";
const box = node.clientBox;
if (
!box ||
![box.x, box.y, box.width, box.height].every(Number.isFinite) ||
box.width < 0 ||
box.height < 0
)
issue ??= "geometry-unavailable";
else {
clip = {
left: x ? Math.max(clip.left, box.x) : clip.left,
right: x ? Math.min(clip.right, box.x + box.width) : clip.right,
top: y ? Math.max(clip.top, box.y) : clip.top,
bottom: y ? Math.min(clip.bottom, box.y + box.height) : clip.bottom,
};
clips = {
document: node.document,
backendNodeId: node.backendNodeId,
x,
y,
overflowX: styles["overflow-x"],
overflowY: styles["overflow-y"],
box,
parent: clips,
};
}
}
const clipsHere = clipContents && (x || y);
return {
hidden,
transformed,
issue,
clip,
clips,
localClip: parent.localClip || clipsHere,
unpositionedClip:
styles.position !== "static" ||
styles.transform !== "none" ||
/(?:^|\s)layout(?:$|\s)/.test(styles.contain ?? "")
? false
: parent.unpositionedClip || clipsHere,
};
}
/** A page screenshot rectangle, not an exact mask of visible Canvas pixels.
* Rounded corners are rendered by the browser and do not invalidate this range. */
export type VisualRegionResult =
| { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource }
| { status: "empty" }
| { status: "unavailable"; reason: VisualIssueReason };
/** Box and frameVisibleBox must come from the same box kind (Canvas border-box).
* Frame/viewport clipping is supplied by the shared geometry adapter. */
export function resolveVisualRegion(input: {
borderBox: ViewportRect | null;
frameVisibleBox: ViewportRect | null;
context: VisualContext;
visibility: string;
}): VisualRegionResult {
const { borderBox, frameVisibleBox, context, visibility } = input;
if (context.hidden || visibility === "hidden" || visibility === "collapse")
return { status: "empty" };
if (context.issue) return { status: "unavailable", reason: context.issue };
if (visibility !== "visible") return { status: "unavailable", reason: "facts-unavailable" };
if (!borderBox) return { status: "unavailable", reason: "geometry-unavailable" };
if (!frameVisibleBox) return { status: "empty" };
if (
![
borderBox.x,
borderBox.y,
borderBox.width,
borderBox.height,
frameVisibleBox.x,
frameVisibleBox.y,
frameVisibleBox.width,
frameVisibleBox.height,
].every(Number.isFinite)
)
return { status: "unavailable", reason: "geometry-unavailable" };
if (
borderBox.width <= 0 ||
borderBox.height <= 0 ||
frameVisibleBox.width < 0 ||
frameVisibleBox.height < 0 ||
Object.values(context.clip).some(Number.isNaN)
)
return { status: "unavailable", reason: "geometry-unavailable" };
const x = Math.max(frameVisibleBox.x, context.clip.left);
const y = Math.max(frameVisibleBox.y, context.clip.top);
const right = Math.min(frameVisibleBox.x + frameVisibleBox.width, context.clip.right);
const bottom = Math.min(frameVisibleBox.y + frameVisibleBox.height, context.clip.bottom);
return right > x && bottom > y
? {
status: "available",
borderBox,
crop: { x, y, width: right - x, height: bottom - y },
clips: context.clips,
}
: { status: "empty" };
}