fix(geometry): unify frame projection and screenshot coordinates

This commit is contained in:
Ljy-0827
2026-09-09 13:20:21 +08:00
committed by GitHub
parent 90b0ff301b
commit 1485addfd5
11 changed files with 899 additions and 305 deletions
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; import { resolveNodeGeometry } from "../frame-geometry";
import { import {
clipPolygon, clipPolygon,
polygonArea, polygonArea,
@@ -10,6 +10,7 @@ import {
rectPolygon, rectPolygon,
regionBounds, regionBounds,
} from "../geometry"; } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => { describe("frame geometry projection", () => {
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected child command ${method}`); throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"], }) as CdpRunner["sendToTarget"],
}; };
const projection = await resolveFrameProjection( const projection = await new GeometryContext(cdp, 4, {
cdp, rootFrameId: "main",
{ frames: [
rootFrameId: "main", { frameId: "main", target: { tabId: 4 } },
frames: [ {
{ frameId: "main", target: { tabId: 4 } }, frameId: "same-process-parent",
{ parentFrameId: "main",
frameId: "same-process-parent", ownerBackendNodeId: 10,
parentFrameId: "main", target: { tabId: 4 },
ownerBackendNodeId: 10, },
target: { tabId: 4 }, {
}, frameId: "oopif",
{ parentFrameId: "same-process-parent",
frameId: "oopif", ownerBackendNodeId: 20,
parentFrameId: "same-process-parent", target: { tabId: 4, sessionId: "oopif-session" },
ownerBackendNodeId: 20, },
target: { tabId: 4, sessionId: "oopif-session" }, ],
}, }).targetProjection("oopif");
],
},
"oopif",
);
expect(projection?.edges).toEqual([ expect(projection?.edges).toEqual([
{ {
@@ -0,0 +1,284 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target },
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
],
};
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 300, height: 200 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
});
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
}
describe("measurement geometry context", () => {
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const results = await Promise.all(
Array.from({ length: 20 }, () => context.targetProjection("child")),
);
expect(results.every((value) => value === results[0])).toBe(true);
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"Page.getLayoutMetrics",
"DOM.getBoxModel",
]);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
expect(cdp.calls.mock.calls).toHaveLength(4);
});
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const source = { target, frameId: "child" };
const projection = await context.snapshotProjection(source, 10, [], {
width: 1000,
height: 1000,
});
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
expect(projectSnapshotRect(input!, projection!)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
expect(projectSnapshotRect(scrolled!, projection!)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
1,
);
expect(
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
).toHaveLength(1);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ x: 10, y: 20 },
)!;
const projection = {
source: { target, frameId: "main" },
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
};
expect(projectSnapshotRect(input, projection)).toEqual({
x: 20,
y: 30,
width: 120,
height: 40,
});
expect(
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
).toBeNull();
expect(
projectSnapshotRect(input, {
...projection,
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
})) as CdpRunner["send"],
};
expect(
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 100,
height: 100,
}),
).toBeNull();
});
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
const cdp = driver();
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
{ target, frameId: "nested" },
10,
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
{ width: 1000, height: 1000 },
);
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ x: 0, y: 0 },
);
expect(projectSnapshotRect(input!, projection!)).toEqual({
x: 60,
y: 620,
width: 100,
height: 100,
});
});
it("rejects malformed frame ancestry without looping", async () => {
const cdp = driver();
const cycle: CdpFrameGraph = {
rootFrameId: "a",
frames: [
{ frameId: "a", parentFrameId: "b", target },
{ frameId: "b", parentFrameId: "a", target },
],
};
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
expect(cdp.calls).not.toHaveBeenCalled();
const orphan: CdpFrameGraph = {
rootFrameId: "a",
frames: [{ frameId: "a", parentFrameId: "missing", target }],
};
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
});
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
const cdp = driver();
expect(
await resolveNodeGeometry(
cdp,
4,
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ code: "cdp_failed" });
expect(cdp.calls).not.toHaveBeenCalled();
});
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
let active = 0;
let peak = 0;
const release: Array<() => void> = [];
const send = vi.fn(async () => {
active++;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
});
const controller = new AbortController();
const context = new GeometryContext(
{ send: send as CdpRunner["send"] },
4,
undefined,
controller.signal,
);
const tasks = Array.from({ length: 8 }, (_, index) =>
context.viewport({ tabId: 4, sessionId: String(index) }),
);
const settled = Promise.allSettled(tasks);
expect(send).toHaveBeenCalledTimes(4);
controller.abort();
for (const done of release) done();
const results = await settled;
expect(peak).toBe(4);
expect(send).toHaveBeenCalledTimes(4);
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
});
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
const controller = new AbortController();
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
cdp.calls.mockImplementation(async (tab, method) => {
const result = await original(tab, method);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
await expect(
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
}),
).rejects.toMatchObject({ name: "AbortError" });
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.releaseObject",
]);
});
it("reads live owner geometry only after scrolling", async () => {
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
let scrolled = false;
cdp.calls.mockImplementation(async (tab, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
scrolled = true;
return {};
}
if (method === "DOM.getContentQuads") {
expect(scrolled).toBe(true);
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
}
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
return original(tab, method);
});
expect(
await resolveNodeGeometry(
cdp,
4,
{ target, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
1,
);
});
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
).toBeNull();
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
).toBeNull();
});
it("retries a rejected measurement only in a new operation", async () => {
const cdp = driver();
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
const context = new GeometryContext(cdp, 4);
await expect(context.viewport(target)).rejects.toThrow("unavailable");
await expect(context.viewport(target)).rejects.toThrow("unavailable");
expect(cdp.calls).toHaveBeenCalledTimes(1);
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
width: 1000,
height: 1000,
});
});
});
@@ -266,7 +266,15 @@ describe("handleScreenshot", () => {
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined(); expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
}); });
it("captures a clipped PNG when ref is given", async () => { it.each([
{ zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } },
{ zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } },
{ zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } },
])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({
zoom,
scroll,
expected,
}) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11"); const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, { tabId: 7 }); ctx.refStore.set("e5", 999, { tabId: 7 });
@@ -274,6 +282,10 @@ describe("handleScreenshot", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }), "Page.captureScreenshot": () => ({ data: TINY_PNG }),
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll },
cssVisualViewport: { zoom },
}),
}); });
const captureVisibleTab = vi.fn(); const captureVisibleTab = vi.fn();
const res = await handleScreenshot( const res = await handleScreenshot(
@@ -297,7 +309,8 @@ describe("handleScreenshot", () => {
clip?: { x: number; y: number; width: number; height: number }; clip?: { x: number; y: number; width: number; height: number };
} }
)?.clip; )?.clip;
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); expect(clip).toMatchObject(expected);
expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1);
}); });
it("returns not_found for unknown ref", async () => { it("returns not_found for unknown ref", async () => {
+46 -151
View File
@@ -1,24 +1,21 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types"; import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import { import {
clipPolygon, clipPolygon,
type GeometryProjection,
type Point, type Point,
type Polygon, type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea, polygonArea,
polygonCentroid, polygonCentroid,
projectRegionToViewport, projectRegionToViewport,
type Quad,
type Region, type Region,
rectPolygon, rectPolygon,
regionBounds, regionBounds,
type Size,
type ViewportRect, type ViewportRect,
} from "./geometry"; } from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; import type { CssViewport } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
export interface NodeAddress { export interface NodeAddress {
target: CdpTarget; target: CdpTarget;
@@ -27,6 +24,8 @@ export interface NodeAddress {
} }
export interface ResolvedNodeGeometry { export interface ResolvedNodeGeometry {
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
topViewport: CssViewport;
topVisibleRegions: Region; topVisibleRegions: Region;
topBounds: ViewportRect; topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */ /** Point in the top-level tab viewport, used by root-target input events. */
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
return { code: "cdp_failed", message }; return { code: "cdp_failed", message };
} }
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners( async function scrollFrameOwners(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
graph: CdpFrameGraph, context: GeometryContext,
frameId: string, frameId: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const byId = frameMap(graph); const ancestry = await context.ancestry(frameId);
const ancestry = frameAncestry(graph, frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) { for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) { if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`); return geometryError(`could not resolve frame owner for ${child.frameId}`);
} }
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
return null; return null;
} }
async function scrollElementWithFrameGraph( async function scrollElementWithContext(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
target: CdpTarget, target: CdpTarget,
backendNodeId: number, backendNodeId: number,
frameId: string | undefined, frameId: string | undefined,
graph: CdpFrameGraph | null, context: GeometryContext,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
if (frameId) { if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); const frame = await context.frame(frameId);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId); if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
return geometryError("node frame does not belong to its target");
}
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
if (error) return error; if (error) return error;
} else if (target.sessionId) { } else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
backendNodeId: number, backendNodeId: number,
frameId?: string, frameId?: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); return scrollElementWithContext(
cdp,
tabId,
target,
backendNodeId,
frameId,
new GeometryContext(cdp, tabId),
);
} }
function largestRegion(regions: Region): Polygon | null { function largestRegion(regions: Region): Polygon | null {
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
if (address.target.sessionId && !address.frameId) { if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
} }
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
const context = new GeometryContext(cdp, tabId);
const graph = address.frameId ? await context.graph() : null;
if (address.frameId && graph) {
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
return geometryError("node frame does not belong to its target");
}
}
if (address.frameId && !graph) { if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`); return geometryError(`could not resolve frame graph for ${address.frameId}`);
} }
if (options.scrollIntoView) { if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph( const scrollError = await scrollElementWithContext(
cdp, cdp,
tabId, tabId,
address.target, address.target,
address.backendNodeId, address.backendNodeId,
address.frameId, address.frameId,
graph, context,
); );
if (scrollError) return scrollError; if (scrollError) return scrollError;
} }
// Only topology was read above. Begin live measurements after scrolling.
const localRegion = await nodeContentRegion( const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target), cdpRunnerForTarget(cdp, address.target),
tabId, tabId,
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
let topVisibleRegions: Region; let topVisibleRegions: Region;
let targetActionPoint: Point | null = null; let targetActionPoint: Point | null = null;
if (address.frameId && graph) { if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId); const projection = await context.targetProjection(address.frameId);
if (!projection) if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`); return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection); topVisibleRegions = projectRegionToViewport(localRegion, projection);
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
} }
} else { } else {
const viewport = await targetViewport(cdp, address.target); const viewport = await context.viewport(address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry"); if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion topVisibleRegions = localRegion
.map((polygon) => .map((polygon) =>
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
return { code: "permission_denied", message: "element not visible in its target" }; return { code: "permission_denied", message: "element not visible in its target" };
} }
targetActionPoint ??= actionPoint; targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; return {
topVisibleRegions,
topBounds,
actionPoint,
targetActionPoint,
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
};
} catch (error) { } catch (error) {
return geometryError(error instanceof Error ? error.message : String(error)); return geometryError(error instanceof Error ? error.message : String(error));
} }
-19
View File
@@ -245,22 +245,3 @@ export function projectRectToViewport(
if (!rect) return null; if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
} }
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
@@ -0,0 +1,84 @@
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { GeometryProjection, ViewportRect } from "../geometry";
import { projectRectToViewport } from "../geometry";
export interface CoordinateOwner {
target: CdpTarget;
frameId?: string;
}
export interface FrameViewportRect {
space: "frame-viewport-css";
owner: CoordinateOwner;
rect: ViewportRect;
}
export interface SnapshotProjection {
source: CoordinateOwner;
geometry: GeometryProjection;
}
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
scroll: { x: number; y: number },
): FrameViewportRect | null {
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const [x, y, width, height] = bounds;
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
};
}
export function projectSnapshotRect(
input: FrameViewportRect,
projection: SnapshotProjection,
): ViewportRect | null {
if (
input.owner.frameId !== projection.source.frameId ||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
) {
return null;
}
const { x, y, width: w, height: h } = input.rect;
return projectRectToViewport({ x, y, w, h }, projection.geometry);
}
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
export interface CssViewport {
width: number;
height: number;
scrollX: number;
scrollY: number;
cssToDip: number;
}
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
export function screenshotPageRect(
rect: ViewportRect,
viewport: CssViewport,
): { space: "page-dip"; rect: ViewportRect } | null {
const { cssToDip, scrollX, scrollY } = viewport;
if (
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
cssToDip <= 0 ||
rect.width <= 0 ||
rect.height <= 0
)
return null;
return {
space: "page-dip",
rect: {
x: (rect.x + scrollX) * cssToDip,
y: (rect.y + scrollY) * cssToDip,
width: rect.width * cssToDip,
height: rect.height * cssToDip,
},
};
}
@@ -0,0 +1,297 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import {
type GeometryProjection,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
type Quad,
type Size,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjection } from "./coordinate-types";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
return {
width: value?.clientWidth ?? 0,
height: value?.clientHeight ?? 0,
scrollX: value?.pageX ?? 0,
scrollY: value?.pageY ?? 0,
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
};
}
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
private active = 0;
private readonly waiting: Array<() => void> = [];
private graphPromise?: Promise<CdpFrameGraph | null>;
private framesPromise?: Promise<Map<string, CdpFrame>>;
constructor(
private readonly cdp: CdpRunner,
private readonly tabId: number,
graph?: CdpFrameGraph,
private readonly signal?: AbortSignal,
) {
if (graph) this.graphPromise = Promise.resolve(graph);
}
private async request<T>(
target: CdpTarget,
method: string,
params: object,
cleanup = false,
): Promise<T> {
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
else this.active++;
try {
if (!cleanup && this.signal?.aborted)
throw new DOMException("geometry aborted", "AbortError");
return await sendToCdpTarget<T>(this.cdp, target, method, params);
} finally {
const next = this.waiting.shift();
if (next) next();
else this.active--;
}
}
graph(): Promise<CdpFrameGraph | null> {
this.graphPromise ??=
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
return this.graphPromise;
}
private frames(): Promise<Map<string, CdpFrame>> {
this.framesPromise ??= this.graph().then(
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
);
return this.framesPromise;
}
async frame(frameId: string): Promise<CdpFrame | undefined> {
return (await this.frames()).get(frameId);
}
/** Topology only: safe to resolve before the caller scrolls frame owners. */
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
const frames = await this.frames();
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = frames.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
const key = cdpTargetKey(target);
let promise = this.metrics.get(key);
if (!promise) {
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
this.metrics.set(key, promise);
}
return promise;
}
async viewport(target: CdpTarget): Promise<Size | null> {
const { width, height } = cssViewport(await this.layoutMetrics(target));
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
? { width, height }
: null;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
if (!promise) {
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
backendNodeId,
}).then((reply) => parseCdpQuad(reply.model?.content));
this.owners.set(key, promise);
}
return promise;
}
/** Same-target owner quads are already target-relative, including nested owners. */
async snapshotProjection(
source: CoordinateOwner,
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
): Promise<SnapshotProjection | null> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
const owner = await promise;
return owner
? {
source,
geometry: {
sourceClips: [],
edges: [
{
sourceViewport: owner.size,
destinationQuad: owner.quad,
destinationClips: ancestorClips,
},
],
topViewport: viewport,
},
}
: null;
}
private async snapshotOwner(
target: CdpTarget,
backendNodeId: number,
): Promise<{ quad: Quad; size: Size } | null> {
const quad = await this.ownerContent(target, backendNodeId);
if (!quad) return null;
const resolved = await this.request<{ object?: { objectId?: string } }>(
target,
"DOM.resolveNode",
{ backendNodeId },
);
const objectId = resolved.object?.objectId;
if (!objectId) return null;
try {
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this.isConnected) return null;
const style = getComputedStyle(this);
return {
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
};
}`,
returnByValue: true,
},
);
const size = reply.result?.value;
return !reply.exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? { quad, size }
: null;
} finally {
// Cleanup must still run if cancellation arrived after resolving the object.
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
}
}
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
current = parent;
}
return current;
}
private async clips(
frames: Map<string, CdpFrame>,
frame: CdpFrame,
root: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== root.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
if (!quad) return null;
clips.push(quad);
current = parent;
}
return clips;
}
/** Input is already in the frame's CDP target viewport, not its local viewport. */
targetProjection(frameId: string): Promise<GeometryProjection | null> {
let promise = this.projections.get(frameId);
if (!promise) {
promise = this.buildTargetProjection(frameId);
this.projections.set(frameId, promise);
}
return promise;
}
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
const frames = await this.frames();
const frame = frames.get(frameId);
if (!frame) return null;
let root = this.targetRoot(frames, frame);
if (!root) return null;
const sourceViewport = await this.viewport(root.target);
if (!sourceViewport) return null;
const sourceClips = await this.clips(frames, frame, root);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
const seen = new Set<string>();
while (root.parentFrameId) {
if (seen.has(root.frameId)) return null;
seen.add(root.frameId);
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
}
+21 -19
View File
@@ -34,6 +34,8 @@ import type {
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry"; import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { import {
type ChromeTabsApi, type ChromeTabsApi,
enforceToolTargetScope, enforceToolTargetScope,
@@ -197,15 +199,14 @@ async function captureElementScreenshot(
if (isRpcError(geometry)) return geometry; if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds; const rect = geometry.topBounds;
const clip = screenshotPageRect(rect, geometry.topViewport);
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
try { try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
format: "png", format: "png",
clip: { clip: {
x: rect.x, ...clip.rect,
y: rect.y,
width: rect.width,
height: rect.height,
scale: 1, scale: 1,
}, },
}); });
@@ -984,26 +985,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
} }
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel( async function fallbackCapturedViewModel(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
geometry: GeometryContext,
signal?: AbortSignal, signal?: AbortSignal,
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation"); throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 }; let viewport = { width: 0, height: 0 };
try { try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {}); const metrics = await geometry.layoutMetrics({ tabId });
throwIfAborted(signal, "observation"); throwIfAborted(signal, "observation");
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const source = cssViewport(metrics);
viewport = { viewport = { width: source.width, height: source.height };
width: source.clientWidth ?? 0,
height: source.clientHeight ?? 0,
};
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
} }
@@ -1016,12 +1010,13 @@ async function captureForVom(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
options: CaptureVomObservationOptions, options: CaptureVomObservationOptions,
geometry: GeometryContext,
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
try { try {
return await captureViewModel(cdp, tabId, { signal: options.signal }); return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal); return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
} }
} }
@@ -1096,9 +1091,16 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url); await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options); const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
const captured = await captureForVom(cdp, tabId, options, geometry);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal); const documents = await captureFrameData<CdpAxNode>(
cdp,
tabId,
captured,
options.signal,
geometry,
);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const normalizedDocuments = const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0 documents.length === 1 && captured.iframeNodes.size > 0
@@ -754,7 +754,7 @@ describe("captureViewModel", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 }); expect(div?.rect).toMatchObject({ y: 0, h: 600 });
}); });
it("normalizes device-pixel bounds by devicePixelRatio", async () => { it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
@@ -802,6 +802,12 @@ describe("captureViewModel", () => {
w: 1000, w: 1000,
h: 800, h: 800,
}); });
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 2000,
h: 1600,
});
}); });
it("extracts textContent from #text child nodes via nodeValue", async () => { it("extracts textContent from #text child nodes via nodeValue", async () => {
@@ -950,6 +956,12 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
@@ -979,7 +991,7 @@ describe("captureViewModel", () => {
expect(frameParentIds?.get("child-frame")).toBe("main-frame"); expect(frameParentIds?.get("child-frame")).toBe("main-frame");
}); });
it("captures iframe documents without requiring owner-frame geometry", async () => { it("projects iframe documents independently of owner snapshot geometry", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -1040,6 +1052,12 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
@@ -1048,7 +1066,7 @@ describe("captureViewModel", () => {
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toBeNull(); expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
}); });
it("recursively normalizes nested iframe sub-documents", async () => { it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1127,16 +1145,43 @@ describe("captureViewModel", () => {
], ],
}; };
const cdp = { const cdp = {
send: vi.fn(async (_t: number, method: string) => { send: vi.fn(
if (method === "DOMSnapshot.enable") return {}; async (
if (method === "DOMSnapshot.captureSnapshot") return snapshot; _t: number,
if (method === "Page.getLayoutMetrics") { method: string,
return { params?: { backendNodeId?: number; objectId?: string },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, ) => {
}; if (method === "DOMSnapshot.enable") return {};
} if (method === "DOMSnapshot.captureSnapshot") return snapshot;
throw new Error(method); if (method === "Page.getLayoutMetrics") {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return {
model: {
content:
params?.backendNodeId === 13
? [10, 20, 310, 20, 310, 220, 10, 220]
: [15, 26, 115, 26, 115, 106, 15, 106],
},
};
if (method === "DOM.resolveNode")
return { object: { objectId: String(params?.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return {
result: {
value:
params?.objectId === "13"
? { width: 300, height: 200 }
: { width: 100, height: 80 },
},
};
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
},
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { iframeNodes } = await captureViewModel(cdp, 4);
@@ -1150,7 +1195,7 @@ describe("captureViewModel", () => {
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
}); });
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
@@ -1192,7 +1237,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = await captureViewModel(cdp, 4);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
}); });
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => { it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
+64 -62
View File
@@ -5,9 +5,15 @@
// STYLE_COL so capture and visibility policy share one explicit contract. // STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom"; import type { Rect, Viewport } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; import {
projectSnapshotRect,
type SnapshotProjection,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
@@ -77,6 +83,8 @@ export interface CapturedViewModel {
export interface CaptureViewModelOptions { export interface CaptureViewModelOptions {
signal?: AbortSignal; signal?: AbortSignal;
geometry?: GeometryContext;
target?: CdpTarget;
} }
function captureAbortError(): Error { function captureAbortError(): Error {
@@ -147,16 +155,6 @@ interface SnapshotReply {
documents?: SnapshotDocument[]; documents?: SnapshotDocument[];
} }
interface LayoutMetricsReply {
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
function isFormControlTag(tag: string): boolean { function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select"; return tag === "input" || tag === "textarea" || tag === "select";
} }
@@ -353,7 +351,8 @@ interface ParseDocumentResult {
interface FrameContext { interface FrameContext {
frameId?: string; frameId?: string;
ownerFrameBackendNodeId: number | null; ownerFrameBackendNodeId: number | null;
projection: GeometryProjection | null; projection: SnapshotProjection | null;
target: CdpTarget;
scrollX: number; scrollX: number;
scrollY: number; scrollY: number;
} }
@@ -363,15 +362,6 @@ function str(strings: string[], idx: number | undefined): string {
return strings[idx] ?? ""; return strings[idx] ?? "";
} }
function devicePixelRatio(metrics: LayoutMetricsReply): number {
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
if (!layoutW || !cssW) return 1;
const dpr = layoutW / cssW;
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
return dpr >= 1 ? dpr : 1;
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> { function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>(); const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out; if (!sparse?.index || !sparse.value) return out;
@@ -797,13 +787,11 @@ export async function collectOverlayExcludedBackendIds(
* *
* @param doc - the raw DOMSnapshot document object * @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot * @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param context - frame coordinate context; output rects are top viewport-relative * @param context - frame coordinate context; output rects are top viewport-relative
*/ */
function parseDocumentNodes( function parseDocumentNodes(
doc: SnapshotDocument, doc: SnapshotDocument,
strings: string[], strings: string[],
dpr: number,
context: FrameContext, context: FrameContext,
): ParseDocumentResult { ): ParseDocumentResult {
const dn = doc.nodes; const dn = doc.nodes;
@@ -898,15 +886,17 @@ function parseDocumentNodes(
if (li !== undefined) { if (li !== undefined) {
const b = dl?.bounds?.[li]; const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) { if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
localRect = { const input = snapshotViewportRect(
x: b[0] / dpr - context.scrollX, b,
y: b[1] / dpr - context.scrollY, { target: context.target, frameId: context.frameId },
w: b[2] / dpr, { x: context.scrollX, y: context.scrollY },
h: b[3] / dpr, );
}; if (input) {
const bounds = context.projection const { x, y, width: w, height: h } = input.rect;
? projectRectToViewport(localRect, context.projection) localRect = { x, y, w, h };
: null; }
const bounds =
input && context.projection ? projectSnapshotRect(input, context.projection) : null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
} }
paintOrder = dl?.paintOrders?.[li] ?? 0; paintOrder = dl?.paintOrders?.[li] ?? 0;
@@ -980,15 +970,14 @@ interface ParseFrameDocumentsResult {
excludedBackendNodeIds: Set<number>; excludedBackendNodeIds: Set<number>;
} }
function parseChildFrameDocuments( async function parseChildFrameDocuments(
documents: SnapshotDocument[], documents: SnapshotDocument[],
strings: string[], strings: string[],
dpr: number,
parentDocIndex: number, parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext, parentContext: FrameContext,
geometry: GeometryContext,
visited = new Set<number>(), visited = new Set<number>(),
): ParseFrameDocumentsResult { ): Promise<ParseFrameDocumentsResult> {
const iframeNodes: CapturedIframeNodes = new Map(); const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>(); const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>(); const frameParentIds = new Map<string, string>();
@@ -1000,7 +989,6 @@ function parseChildFrameDocuments(
} }
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) { for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue; if (visited.has(childDocIndex)) continue;
@@ -1008,14 +996,31 @@ function parseChildFrameDocuments(
if (!childDoc) continue; if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx]; const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue; if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId); let projection: SnapshotProjection | null = null;
const projection = const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
parentContext.projection && iframeNode?.localRect const parentProjection = parentContext.projection?.geometry;
? childFrameProjection(parentContext.projection, iframeNode.localRect) if (parentProjection) {
: null; // Each owner quad is target-relative; do not apply the parent's transform twice.
const edge = parentProjection.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: parentProjection.sourceClips;
try {
projection = await geometry.snapshotProjection(
source,
iframeBackendId,
clips,
parentProjection.topViewport,
);
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] frame owner geometry unavailable", error);
}
}
const childContext: FrameContext = { const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings), frameId: source.frameId,
target: source.target,
ownerFrameBackendNodeId: iframeBackendId, ownerFrameBackendNodeId: iframeBackendId,
projection, projection,
scrollX: childDoc.scrollOffsetX ?? 0, scrollX: childDoc.scrollOffsetX ?? 0,
@@ -1023,7 +1028,7 @@ function parseChildFrameDocuments(
}; };
const nextVisited = new Set(visited); const nextVisited = new Set(visited);
nextVisited.add(childDocIndex); nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext); const parsed = parseDocumentNodes(childDoc, strings, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes); iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) { if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId); frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
@@ -1031,13 +1036,12 @@ function parseChildFrameDocuments(
} }
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments( const nested = await parseChildFrameDocuments(
documents, documents,
strings, strings,
dpr,
childDocIndex, childDocIndex,
parsed.nodes,
childContext, childContext,
geometry,
nextVisited, nextVisited,
); );
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) { for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
@@ -1061,22 +1065,20 @@ export async function captureViewModel(
options: CaptureViewModelOptions = {}, options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
throwIfAborted(options.signal); throwIfAborted(options.signal);
let metrics: LayoutMetricsReply = {}; const target = options.target ?? { tabId };
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
let metrics: LayoutMetrics = {};
try { try {
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {}); metrics = await geometry.layoutMetrics(target);
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error); console.debug("[bsk capture] layout metrics unavailable", error);
} }
throwIfAborted(options.signal); throwIfAborted(options.signal);
const dpr = devicePixelRatio(metrics); const measured = cssViewport(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const viewport: Viewport = { width: measured.width, height: measured.height };
const viewport: Viewport = { const scrollX = measured.scrollX;
width: vpSrc.clientWidth ?? 0, const scrollY = measured.scrollY;
height: vpSrc.clientHeight ?? 0,
};
const scrollX = vpSrc.pageX ?? 0;
const scrollY = vpSrc.pageY ?? 0;
await cdp.send(tabId, "DOMSnapshot.enable", {}); await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal); throwIfAborted(options.signal);
@@ -1102,19 +1104,19 @@ export async function captureViewModel(
const topContext: FrameContext = { const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings), frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null, ownerFrameBackendNodeId: null,
target,
projection: { projection: {
sourceClips: [], source: { target, frameId: snapshotFrameId(doc0, strings) },
edges: [], geometry: { sourceClips: [], edges: [], topViewport: viewport },
topViewport: viewport,
}, },
scrollX, scrollX,
scrollY, scrollY,
}; };
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext); const mainParsed = parseDocumentNodes(doc0, strings, topContext);
const nodes = mainParsed.nodes; const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds); const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext); const frameParsed = await parseChildFrameDocuments(documents, strings, 0, topContext, geometry);
const iframeNodes = frameParsed.iframeNodes; const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) { for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id); excludedBackendNodeIds.add(id);
+8 -14
View File
@@ -1,6 +1,6 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveFrameProjection } from "../frame-geometry";
import { type GeometryProjection, projectRectToViewport } from "../geometry"; import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import { import {
@@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError"; return err instanceof Error && err.name === "AbortError";
} }
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes( function transformFrameNodes(
nodes: CapturedNode[], nodes: CapturedNode[],
projection: GeometryProjection | null, projection: GeometryProjection | null,
@@ -51,6 +41,7 @@ async function captureMissingFrameDocuments(
tabId: number, tabId: number,
graph: CdpFrameGraph, graph: CdpFrameGraph,
captured: CapturedViewModel, captured: CapturedViewModel,
geometry: GeometryContext,
signal?: AbortSignal, signal?: AbortSignal,
): Promise<void> { ): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map(); if (!captured.frameNodes) captured.frameNodes = new Map();
@@ -62,10 +53,12 @@ async function captureMissingFrameDocuments(
try { try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal, signal,
geometry,
target: frame.target,
}); });
let projection: GeometryProjection | null = null; let projection: GeometryProjection | null = null;
try { try {
projection = await resolveFrameProjection(cdp, graph, frame.frameId); projection = await geometry.targetProjection(frame.frameId);
} catch (err) { } catch (err) {
if (isAbortError(err)) throw err; if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", { console.debug("[bsk observation] child frame geometry projection failed", {
@@ -171,10 +164,11 @@ export async function captureFrameData<T extends FrameAxNode>(
tabId: number, tabId: number,
captured: CapturedViewModel, captured: CapturedViewModel,
signal?: AbortSignal, signal?: AbortSignal,
geometry = new GeometryContext(cdp, tabId, undefined, signal),
): Promise<CapturedFrameDocument<T>[]> { ): Promise<CapturedFrameDocument<T>[]> {
const graph = await discoverFrameGraph(cdp, tabId); const graph = await geometry.graph();
const frames = graph?.frames ?? []; const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal); if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
throwIfAborted(signal); throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal); const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured); return buildFrameDocuments(graph, batches, captured);