fix(geometry): preserve OOPIF scale with occupied scrollbars

This commit is contained in:
drakezhang
2026-09-09 16:58:25 +08:00
parent 896c7944d8
commit a62a28d8cd
14 changed files with 504 additions and 65 deletions
@@ -436,6 +436,8 @@ describe("file transfer tools", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "Runtime.evaluate")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
}
@@ -13,7 +13,138 @@ import {
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
function scrollbarDriver(
visible = { width: 185, height: 89 },
frameSize: unknown = { width: 200, height: 100 },
) {
const send = vi.fn(async (target: { sessionId?: string }, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: frameSize } };
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: {
clientWidth: target.sessionId ? visible.width : 800,
clientHeight: target.sessionId ? visible.height : 600,
},
};
if (method === "DOM.getBoxModel")
return {
model: {
content: target.sessionId
? [0, 0, 200, 0, 200, 100, 0, 100]
: [100, 100, 500, 100, 500, 300, 100, 300],
},
};
if (method === "DOM.getContentQuads")
return { quads: [[180, 80, 230, 80, 230, 120, 180, 120]] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: (tabId, method) => send({ tabId } as never, method) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child" },
},
{
frameId: "same-child",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "child" },
},
],
}),
};
return { cdp, send };
}
describe("frame geometry projection", () => {
it.each([
{ width: 200, height: 100 },
{ width: 185, height: 100 },
{ width: 200, height: 89 },
{ width: 185, height: 89 },
])("preserves scale and clips scrollbar strips for visible viewport $width × $height", async (visible) => {
const { cdp, send } = scrollbarDriver(visible);
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: {
x: 460,
y: 260,
width: (visible.width - 180) * 2,
height: (visible.height - 80) * 2,
},
actionPoint: { x: 460 + visible.width - 180, y: 260 + visible.height - 80 },
targetActionPoint: { x: (180 + visible.width) / 2, y: (80 + visible.height) / 2 },
});
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(send.mock.calls.filter(([, method]) => method === "Page.getLayoutMetrics")).toHaveLength(
2,
);
});
it("shares the full viewport read across same-target frames, and refreshes it next operation", async () => {
const { cdp, send } = scrollbarDriver();
const context = new GeometryContext(cdp, 4);
await Promise.all([context.targetProjection("child"), context.targetProjection("same-child")]);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(2);
});
it.each([
null,
{},
{ width: 0, height: 100 },
{ width: 200, height: NaN },
])("rejects unavailable full viewport dimensions: %j", async (size) => {
const { cdp } = scrollbarDriver(undefined, size);
expect(await new GeometryContext(cdp, 4).targetProjection("child")).toBeNull();
});
it("clips an inner OOPIF against an intermediate OOPIF's occupied scrollbar strips", async () => {
const { cdp } = scrollbarDriver();
const outer = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "DOM.getBoxModel")
return { model: { content: [150, 60, 250, 60, 250, 120, 150, 120] } } as never;
if (target.sessionId !== "inner") return outer(target, method, params);
if (method === "Runtime.evaluate")
return { result: { value: { width: 100, height: 60 } } } as never;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 90, clientHeight: 50 } } as never;
if (method === "DOM.getContentQuads")
return { quads: [[0, 0, 100, 0, 100, 60, 0, 60]] } as never;
throw new Error(`unexpected ${method}`);
};
const graph = await cdp.getFrameGraph!(4);
graph.frames[2] = {
frameId: "inner",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "inner" },
};
cdp.getFrameGraph = async () => graph;
expect(
await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "inner" },
frameId: "inner",
backendNodeId: 101,
}),
).toMatchObject({
topBounds: { x: 400, y: 220, width: 70, height: 58 },
});
});
it("keeps region bounds separate from polygon area", () => {
const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
@@ -77,6 +208,8 @@ describe("frame geometry projection", () => {
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -135,6 +268,8 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -196,6 +331,8 @@ describe("frame geometry projection", () => {
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -433,6 +433,7 @@ describe("handleRequestHelp", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -201,6 +201,7 @@ describe("handleClick", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -220,6 +221,7 @@ describe("handleClick", () => {
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
{ sessionId: "child-session", method: "Runtime.evaluate" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
@@ -3380,6 +3380,7 @@ describe("handleSnapshot", () => {
throw new Error(`unexpected root CDP method ${method}`);
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return {
visualViewport: { clientWidth: 1000 },
@@ -1,6 +1,7 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import type { CdpRunner } from "../shared";
import { captureObservationFacts } from "../vom/capture-coordinator";
@@ -11,12 +12,28 @@ type Send = <T = Record<string, unknown>>(
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> };
type Oracle = {
probes: Record<string, Rect>;
viewport: { width: number; height: number };
owners: Record<string, { x: number; y: number; scale: number }>;
};
function clip(rect: Rect, viewport: Oracle["viewport"]): Rect | null {
const x = Math.max(0, rect.x),
y = Math.max(0, rect.y);
const right = Math.min(viewport.width, rect.x + rect.w);
const bottom = Math.min(viewport.height, rect.y + rect.h);
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
}
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const box = document.querySelector('#probe').getBoundingClientRect();
const probes = {};
for (const node of document.querySelectorAll('[data-geometry-probe]')) {
const box = node.getBoundingClientRect();
probes[node.id] = { x: box.x, y: box.y, w: box.width, h: box.height };
}
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
@@ -28,17 +45,31 @@ const oracleExpression = `(() => {
scale,
};
}
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners };
return { probes, viewport: { width: document.documentElement.clientWidth, height: document.documentElement.clientHeight }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each([
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => {
it.each(
[
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
]
.flatMap((configuration) => [
{ ...configuration, fixture: "snapshot-coordinates", scrollbars: "none" },
{ ...configuration, fixture: "oopif-scrollbars", scrollbars: "both" },
])
.concat(
["vertical", "horizontal", "none"].map((scrollbars) => ({
deviceScale: 1,
zoom: 1,
fixture: "oopif-scrollbars",
scrollbars,
})),
),
)("$fixture: device scale $deviceScale, zoom $zoom, scrollbars $scrollbars", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
@@ -59,7 +90,9 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
);
await send(
"Page.navigate",
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` },
{
url: `${baseUrl}/${configuration.fixture}?run=coordinates&scrollbars=${configuration.scrollbars}`,
},
rootSession,
);
await expect
@@ -86,7 +119,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF.
expect(sessions.length).toBe(configuration.fixture === "oopif-scrollbars" ? 3 : 2);
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
@@ -107,7 +140,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
};
visit(frameTree);
}
expect(frames).toHaveLength(5);
expect(frames).toHaveLength(configuration.fixture === "oopif-scrollbars" ? 3 : 5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
@@ -127,6 +160,11 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
if (
method === "Runtime.evaluate" &&
(params as { expression?: string })?.expression?.includes("window.innerWidth")
)
calls.push(`${target.sessionId ?? target.tabId}:viewport-size`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
@@ -179,47 +217,130 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
);
oracles.set(frame.frameId, reply.result.value);
}
const childViewport = await send<{
result: {
value: { width: number; height: number; clientWidth: number; clientHeight: number };
};
}>(
"Runtime.evaluate",
{
expression:
"({ width: innerWidth, height: innerHeight, clientWidth: document.documentElement.clientWidth, clientHeight: document.documentElement.clientHeight })",
returnByValue: true,
},
sessions[1],
);
for (const dimension of ["Width", "Height"] as const) {
const occupied =
childViewport.result.value[dimension === "Width" ? "width" : "height"] -
childViewport.result.value[`client${dimension}`];
if (
configuration.scrollbars === "both" ||
configuration.scrollbars === (dimension === "Width" ? "vertical" : "horizontal")
)
expect(occupied).toBeGreaterThan(0);
else expect(occupied).toBe(0);
}
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
for (const frame of frames) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === "probe");
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
const local = oracles.get(frame.frameId)!.probe;
const top = { ...local };
let current = frame;
while (current.parentFrameId) {
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!];
top.x = owner.x + top.x * owner.scale;
top.y = owner.y + top.y * owner.scale;
top.w *= owner.scale;
top.h *= owner.scale;
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
expect(
node!.rect,
JSON.stringify({
frame: names.get(frame.frameId),
local,
top,
viewport: facts.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} local ${key}`,
).toBeLessThan(2);
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} top ${key}`,
).toBeLessThan(2);
}
}
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
// Identity verification also uses Runtime.evaluate; count the full viewport reads separately.
const viewportReadsBeforeLive = calls.filter((call) => call.endsWith(":viewport-size"));
expect(new Set(viewportReadsBeforeLive).size).toBe(viewportReadsBeforeLive.length);
expect(viewportReadsBeforeLive.length).toBeLessThanOrEqual(sessions.length - 1);
for (const frame of frames) {
for (const [id, local] of Object.entries(oracles.get(frame.frameId)!.probes)) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === id);
expect(node, `missing ${id} in ${names.get(frame.frameId) || "root"}`).toBeDefined();
let top = clip(local, oracles.get(frame.frameId)!.viewport);
let current = frame;
while (top && current.parentFrameId) {
const parentOracle = oracles.get(current.parentFrameId)!;
const owner = parentOracle.owners[names.get(current.frameId)!];
top = clip(
{
x: owner.x + top.x * owner.scale,
y: owner.y + top.y * owner.scale,
w: top.w * owner.scale,
h: top.h * owner.scale,
},
parentOracle.viewport,
);
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} ${id} local ${key}`,
).toBeLessThan(2);
if (!top) {
expect(node!.rect, `${id} must be clipped out`).toBeNull();
if (configuration.fixture === "oopif-scrollbars")
expect(
await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
}),
).toMatchObject({ code: "permission_denied" });
continue;
}
expect(node!.rect).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} ${id} top ${key}`,
).toBeLessThan(2);
if (configuration.fixture !== "oopif-scrollbars") continue;
const live = await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
});
if ("code" in live) throw new Error(live.message);
for (const [key, liveKey] of [
["x", "x"],
["y", "y"],
["w", "width"],
["h", "height"],
] as const)
expect(
Math.abs(live.topBounds[liveKey] - top[key]),
`${id} live ${key}`,
).toBeLessThan(2);
const before = server.snapshot("coordinates").events.length;
for (const type of ["mousePressed", "mouseReleased"])
await send(
"Input.dispatchMouseEvent",
{ type, ...live.actionPoint, button: "left", clickCount: 1 },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.slice(before)
.some(
(event: { type: string; path: string; data: { probe?: string } }) =>
event.type === "geometry.clicked" &&
event.data.probe === id &&
event.path ===
(frame.parentFrameId
? names.get(frame.frameId) === "nested"
? "/oopif-scrollbars/nested"
: "/oopif-scrollbars/frame"
: "/oopif-scrollbars"),
),
{ timeout: 3000 },
)
.toBe(true);
}
}
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
+2 -1
View File
@@ -161,7 +161,8 @@ export async function resolveNodeGeometry(
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
if (address.target.sessionId) {
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
const localViewport = await context.viewport(address.target);
if (!localViewport) return geometryError("could not resolve target viewport geometry");
const localVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(
@@ -9,8 +9,10 @@ import {
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
projectPolygon,
type Quad,
type Size,
viewportPolygon,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
@@ -67,6 +69,7 @@ export function cssViewport(metrics: LayoutMetrics): CssViewport {
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly frameViewports = new Map<string, Promise<Size | null>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
@@ -157,6 +160,35 @@ export class GeometryContext {
: null;
}
/** The full OOPIF viewport maps to its owner's content quad. Layout metrics
* exclude occupied scrollbars and remain the clipping boundary, not the scale. */
private frameViewport(target: CdpTarget): Promise<Size | null> {
const key = cdpTargetKey(target);
let promise = this.frameViewports.get(key);
if (!promise) {
promise = this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.evaluate",
{
expression: "({ width: window.innerWidth, height: window.innerHeight })",
returnByValue: true,
},
).then(({ result, exceptionDetails }) => {
const size = result?.value;
return !exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? size
: null;
});
this.frameViewports.set(key, promise);
}
return promise;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
@@ -342,12 +374,19 @@ export class GeometryContext {
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
if (!destinationQuad) return null;
const source = await this.frameViewport(root.target);
const visible = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
if (!source || !visible || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
const edge = { sourceViewport: source, destinationQuad, destinationClips };
// Keep scrollbar strips clipped at every target boundary, including
// intermediate OOPIFs. Project the clip with the same full-viewport scale.
if (source.width !== visible.width || source.height !== visible.height)
destinationClips.push(projectPolygon(viewportPolygon(visible), edge));
edges.push(edge);
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
@@ -655,6 +655,7 @@ function childSnapshot(frameId: string, backendNodeId: number) {
describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Runtime.evaluate") return { result: { value: { width: 300, height: 200 } } };
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },