mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(geometry): unify frame projection and screenshot coordinates
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import { resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
const projection = await new GeometryContext(cdp, 4, {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
}).targetProjection("oopif");
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveNodeGeometry } from "../frame-geometry";
|
||||
import { rectPolygon } from "../geometry";
|
||||
import {
|
||||
projectSnapshotRect,
|
||||
screenshotPageRect,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const target = { tabId: 4 };
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target },
|
||||
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
|
||||
],
|
||||
};
|
||||
|
||||
function driver() {
|
||||
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 300, height: 200 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
});
|
||||
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
|
||||
}
|
||||
|
||||
describe("measurement geometry context", () => {
|
||||
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
|
||||
const cdp = driver();
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 20 }, () => context.targetProjection("child")),
|
||||
);
|
||||
expect(results.every((value) => value === results[0])).toBe(true);
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||
"Page.getLayoutMetrics",
|
||||
"DOM.getBoxModel",
|
||||
]);
|
||||
await new GeometryContext(cdp, 4).targetProjection("child");
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.calls.mock.calls).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
|
||||
const cdp = driver();
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
const source = { target, frameId: "child" };
|
||||
const projection = await context.snapshotProjection(source, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
|
||||
expect(projectSnapshotRect(input!, projection!)).toEqual({
|
||||
x: 73.75,
|
||||
y: 641.25,
|
||||
width: 150,
|
||||
height: 50,
|
||||
});
|
||||
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
|
||||
expect(projectSnapshotRect(scrolled!, projection!)?.y).toBe(628.75);
|
||||
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
|
||||
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rejects owner mismatches and invalid numeric bounds", () => {
|
||||
const input = snapshotViewportRect(
|
||||
[30, 50, 120, 40],
|
||||
{ target, frameId: "main" },
|
||||
{ x: 10, y: 20 },
|
||||
)!;
|
||||
const projection = {
|
||||
source: { target, frameId: "main" },
|
||||
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
|
||||
};
|
||||
expect(projectSnapshotRect(input, projection)).toEqual({
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 120,
|
||||
height: 40,
|
||||
});
|
||||
expect(
|
||||
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
projectSnapshotRect(input, {
|
||||
...projection,
|
||||
source: { target: { tabId: 5 }, frameId: "main" },
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
|
||||
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
|
||||
});
|
||||
|
||||
it("does not turn a border quad or a failed owner read into a content projection", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
|
||||
})) as CdpRunner["send"],
|
||||
};
|
||||
expect(
|
||||
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 100,
|
||||
height: 100,
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
|
||||
const cdp = driver();
|
||||
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
|
||||
{ target, frameId: "nested" },
|
||||
10,
|
||||
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
|
||||
{ width: 1000, height: 1000 },
|
||||
);
|
||||
const input = snapshotViewportRect(
|
||||
[0, 0, 300, 200],
|
||||
{ target, frameId: "nested" },
|
||||
{ x: 0, y: 0 },
|
||||
);
|
||||
expect(projectSnapshotRect(input!, projection!)).toEqual({
|
||||
x: 60,
|
||||
y: 620,
|
||||
width: 100,
|
||||
height: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed frame ancestry without looping", async () => {
|
||||
const cdp = driver();
|
||||
const cycle: CdpFrameGraph = {
|
||||
rootFrameId: "a",
|
||||
frames: [
|
||||
{ frameId: "a", parentFrameId: "b", target },
|
||||
{ frameId: "b", parentFrameId: "a", target },
|
||||
],
|
||||
};
|
||||
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
|
||||
expect(cdp.calls).not.toHaveBeenCalled();
|
||||
const orphan: CdpFrameGraph = {
|
||||
rootFrameId: "a",
|
||||
frames: [{ frameId: "a", parentFrameId: "missing", target }],
|
||||
};
|
||||
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
|
||||
const cdp = driver();
|
||||
expect(
|
||||
await resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).toMatchObject({ code: "cdp_failed" });
|
||||
expect(cdp.calls).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const release: Array<() => void> = [];
|
||||
const send = vi.fn(async () => {
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise<void>((resolve) => release.push(resolve));
|
||||
active--;
|
||||
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const context = new GeometryContext(
|
||||
{ send: send as CdpRunner["send"] },
|
||||
4,
|
||||
undefined,
|
||||
controller.signal,
|
||||
);
|
||||
const tasks = Array.from({ length: 8 }, (_, index) =>
|
||||
context.viewport({ tabId: 4, sessionId: String(index) }),
|
||||
);
|
||||
const settled = Promise.allSettled(tasks);
|
||||
expect(send).toHaveBeenCalledTimes(4);
|
||||
controller.abort();
|
||||
for (const done of release) done();
|
||||
const results = await settled;
|
||||
expect(peak).toBe(4);
|
||||
expect(send).toHaveBeenCalledTimes(4);
|
||||
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
|
||||
const controller = new AbortController();
|
||||
const cdp = driver();
|
||||
const original = cdp.calls.getMockImplementation()!;
|
||||
cdp.calls.mockImplementation(async (tab, method) => {
|
||||
const result = await original(tab, method);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
|
||||
await expect(
|
||||
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||
"DOM.getBoxModel",
|
||||
"DOM.resolveNode",
|
||||
"Runtime.releaseObject",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads live owner geometry only after scrolling", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.calls.getMockImplementation()!;
|
||||
let scrolled = false;
|
||||
cdp.calls.mockImplementation(async (tab, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") {
|
||||
scrolled = true;
|
||||
return {};
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
expect(scrolled).toBe(true);
|
||||
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
|
||||
return original(tab, method);
|
||||
});
|
||||
expect(
|
||||
await resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{ target, frameId: "child", backendNodeId: 20 },
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
|
||||
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
|
||||
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
|
||||
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
|
||||
expect(
|
||||
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("retries a rejected measurement only in a new operation", async () => {
|
||||
const cdp = driver();
|
||||
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||
expect(cdp.calls).toHaveBeenCalledTimes(1);
|
||||
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -266,7 +266,15 @@ describe("handleScreenshot", () => {
|
||||
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("captures a clipped PNG when ref is given", async () => {
|
||||
it.each([
|
||||
{ zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } },
|
||||
{ zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } },
|
||||
{ zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } },
|
||||
])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({
|
||||
zoom,
|
||||
scroll,
|
||||
expected,
|
||||
}) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, { tabId: 7 });
|
||||
@@ -274,6 +282,10 @@ describe("handleScreenshot", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll },
|
||||
cssVisualViewport: { zoom },
|
||||
}),
|
||||
});
|
||||
const captureVisibleTab = vi.fn();
|
||||
const res = await handleScreenshot(
|
||||
@@ -297,7 +309,8 @@ describe("handleScreenshot", () => {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
)?.clip;
|
||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
||||
expect(clip).toMatchObject(expected);
|
||||
expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import type { CssViewport } from "./geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
@@ -27,6 +24,8 @@ export interface NodeAddress {
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
|
||||
topViewport: CssViewport;
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
/** Point in the top-level tab viewport, used by root-target input events. */
|
||||
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
context: GeometryContext,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
const ancestry = await context.ancestry(frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithFrameGraph(
|
||||
async function scrollElementWithContext(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
graph: CdpFrameGraph | null,
|
||||
context: GeometryContext,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
const frame = await context.frame(frameId);
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
|
||||
return geometryError("node frame does not belong to its target");
|
||||
}
|
||||
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||
return scrollElementWithContext(
|
||||
cdp,
|
||||
tabId,
|
||||
target,
|
||||
backendNodeId,
|
||||
frameId,
|
||||
new GeometryContext(cdp, tabId),
|
||||
);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||
const context = new GeometryContext(cdp, tabId);
|
||||
const graph = address.frameId ? await context.graph() : null;
|
||||
if (address.frameId && graph) {
|
||||
const frame = await context.frame(address.frameId);
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
|
||||
return geometryError("node frame does not belong to its target");
|
||||
}
|
||||
}
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
const scrollError = await scrollElementWithContext(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
graph,
|
||||
context,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
// Only topology was read above. Begin live measurements after scrolling.
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
|
||||
let topVisibleRegions: Region;
|
||||
let targetActionPoint: Point | null = null;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
const projection = await context.targetProjection(address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
|
||||
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
||||
}
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
const viewport = await context.viewport(address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
|
||||
return { code: "permission_denied", message: "element not visible in its target" };
|
||||
}
|
||||
targetActionPoint ??= actionPoint;
|
||||
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
|
||||
return {
|
||||
topVisibleRegions,
|
||||
topBounds,
|
||||
actionPoint,
|
||||
targetActionPoint,
|
||||
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
|
||||
};
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
@@ -245,22 +245,3 @@ export function projectRectToViewport(
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { GeometryProjection, ViewportRect } from "../geometry";
|
||||
import { projectRectToViewport } from "../geometry";
|
||||
|
||||
export interface CoordinateOwner {
|
||||
target: CdpTarget;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface FrameViewportRect {
|
||||
space: "frame-viewport-css";
|
||||
owner: CoordinateOwner;
|
||||
rect: ViewportRect;
|
||||
}
|
||||
|
||||
export interface SnapshotProjection {
|
||||
source: CoordinateOwner;
|
||||
geometry: GeometryProjection;
|
||||
}
|
||||
|
||||
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
|
||||
export function snapshotViewportRect(
|
||||
bounds: number[],
|
||||
owner: CoordinateOwner,
|
||||
scroll: { x: number; y: number },
|
||||
): FrameViewportRect | null {
|
||||
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
|
||||
const [x, y, width, height] = bounds;
|
||||
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
space: "frame-viewport-css",
|
||||
owner,
|
||||
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
|
||||
};
|
||||
}
|
||||
|
||||
export function projectSnapshotRect(
|
||||
input: FrameViewportRect,
|
||||
projection: SnapshotProjection,
|
||||
): ViewportRect | null {
|
||||
if (
|
||||
input.owner.frameId !== projection.source.frameId ||
|
||||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const { x, y, width: w, height: h } = input.rect;
|
||||
return projectRectToViewport({ x, y, w, h }, projection.geometry);
|
||||
}
|
||||
|
||||
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
|
||||
export interface CssViewport {
|
||||
width: number;
|
||||
height: number;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
cssToDip: number;
|
||||
}
|
||||
|
||||
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
|
||||
export function screenshotPageRect(
|
||||
rect: ViewportRect,
|
||||
viewport: CssViewport,
|
||||
): { space: "page-dip"; rect: ViewportRect } | null {
|
||||
const { cssToDip, scrollX, scrollY } = viewport;
|
||||
if (
|
||||
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
|
||||
cssToDip <= 0 ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
)
|
||||
return null;
|
||||
return {
|
||||
space: "page-dip",
|
||||
rect: {
|
||||
x: (rect.x + scrollX) * cssToDip,
|
||||
y: (rect.y + scrollY) * cssToDip,
|
||||
width: rect.width * cssToDip,
|
||||
height: rect.height * cssToDip,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type GeometryProjection,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
type Quad,
|
||||
type Size,
|
||||
} from "../geometry";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { CoordinateOwner, CssViewport, SnapshotProjection } from "./coordinate-types";
|
||||
|
||||
export interface LayoutMetrics {
|
||||
cssVisualViewport?: { zoom?: number };
|
||||
visualViewport?: { zoom?: number };
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
|
||||
export function cssViewport(metrics: LayoutMetrics): CssViewport {
|
||||
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
|
||||
return {
|
||||
width: value?.clientWidth ?? 0,
|
||||
height: value?.clientHeight ?? 0,
|
||||
scrollX: value?.pageX ?? 0,
|
||||
scrollY: value?.pageY ?? 0,
|
||||
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** One read-only measurement phase. Discard after scrolling or any later operation. */
|
||||
export class GeometryContext {
|
||||
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
|
||||
private readonly owners = new Map<string, Promise<Quad | null>>();
|
||||
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
|
||||
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
|
||||
private active = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
private graphPromise?: Promise<CdpFrameGraph | null>;
|
||||
private framesPromise?: Promise<Map<string, CdpFrame>>;
|
||||
|
||||
constructor(
|
||||
private readonly cdp: CdpRunner,
|
||||
private readonly tabId: number,
|
||||
graph?: CdpFrameGraph,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
if (graph) this.graphPromise = Promise.resolve(graph);
|
||||
}
|
||||
|
||||
private async request<T>(
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params: object,
|
||||
cleanup = false,
|
||||
): Promise<T> {
|
||||
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
|
||||
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||
else this.active++;
|
||||
try {
|
||||
if (!cleanup && this.signal?.aborted)
|
||||
throw new DOMException("geometry aborted", "AbortError");
|
||||
return await sendToCdpTarget<T>(this.cdp, target, method, params);
|
||||
} finally {
|
||||
const next = this.waiting.shift();
|
||||
if (next) next();
|
||||
else this.active--;
|
||||
}
|
||||
}
|
||||
|
||||
graph(): Promise<CdpFrameGraph | null> {
|
||||
this.graphPromise ??=
|
||||
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
|
||||
return this.graphPromise;
|
||||
}
|
||||
|
||||
private frames(): Promise<Map<string, CdpFrame>> {
|
||||
this.framesPromise ??= this.graph().then(
|
||||
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
|
||||
);
|
||||
return this.framesPromise;
|
||||
}
|
||||
|
||||
async frame(frameId: string): Promise<CdpFrame | undefined> {
|
||||
return (await this.frames()).get(frameId);
|
||||
}
|
||||
|
||||
/** Topology only: safe to resolve before the caller scrolls frame owners. */
|
||||
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
|
||||
const frames = await this.frames();
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = frames.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
|
||||
const key = cdpTargetKey(target);
|
||||
let promise = this.metrics.get(key);
|
||||
if (!promise) {
|
||||
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
|
||||
this.metrics.set(key, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
async viewport(target: CdpTarget): Promise<Size | null> {
|
||||
const { width, height } = cssViewport(await this.layoutMetrics(target));
|
||||
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
|
||||
? { width, height }
|
||||
: null;
|
||||
}
|
||||
|
||||
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
|
||||
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
let promise = this.owners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
|
||||
backendNodeId,
|
||||
}).then((reply) => parseCdpQuad(reply.model?.content));
|
||||
this.owners.set(key, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** Same-target owner quads are already target-relative, including nested owners. */
|
||||
async snapshotProjection(
|
||||
source: CoordinateOwner,
|
||||
ownerBackendNodeId: number,
|
||||
ancestorClips: Polygon[],
|
||||
viewport: Size,
|
||||
): Promise<SnapshotProjection | null> {
|
||||
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
||||
let promise = this.snapshotOwners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||
this.snapshotOwners.set(key, promise);
|
||||
}
|
||||
const owner = await promise;
|
||||
return owner
|
||||
? {
|
||||
source,
|
||||
geometry: {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: owner.size,
|
||||
destinationQuad: owner.quad,
|
||||
destinationClips: ancestorClips,
|
||||
},
|
||||
],
|
||||
topViewport: viewport,
|
||||
},
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
private async snapshotOwner(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
): Promise<{ quad: Quad; size: Size } | null> {
|
||||
const quad = await this.ownerContent(target, backendNodeId);
|
||||
if (!quad) return null;
|
||||
const resolved = await this.request<{ object?: { objectId?: string } }>(
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{ backendNodeId },
|
||||
);
|
||||
const objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
try {
|
||||
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
|
||||
target,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (!this.isConnected) return null;
|
||||
const style = getComputedStyle(this);
|
||||
return {
|
||||
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
|
||||
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
|
||||
};
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const size = reply.result?.value;
|
||||
return !reply.exceptionDetails &&
|
||||
size &&
|
||||
Number.isFinite(size.width) &&
|
||||
Number.isFinite(size.height) &&
|
||||
size.width > 0 &&
|
||||
size.height > 0
|
||||
? { quad, size }
|
||||
: null;
|
||||
} finally {
|
||||
// Cleanup must still run if cancellation arrived after resolving the object.
|
||||
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private async clips(
|
||||
frames: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
root: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== root.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
|
||||
if (!quad) return null;
|
||||
clips.push(quad);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
/** Input is already in the frame's CDP target viewport, not its local viewport. */
|
||||
targetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||
let promise = this.projections.get(frameId);
|
||||
if (!promise) {
|
||||
promise = this.buildTargetProjection(frameId);
|
||||
this.projections.set(frameId, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||
const frames = await this.frames();
|
||||
const frame = frames.get(frameId);
|
||||
if (!frame) return null;
|
||||
let root = this.targetRoot(frames, frame);
|
||||
if (!root) return null;
|
||||
const sourceViewport = await this.viewport(root.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await this.clips(frames, frame, root);
|
||||
if (!sourceClips) return null;
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
const seen = new Set<string>();
|
||||
while (root.parentFrameId) {
|
||||
if (seen.has(root.frameId)) return null;
|
||||
seen.add(root.frameId);
|
||||
const parent = frames.get(root.parentFrameId);
|
||||
if (!parent || root.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
|
||||
const source = await this.viewport(root.target);
|
||||
const parentRoot = this.targetRoot(frames, parent);
|
||||
if (!destinationQuad || !source || !parentRoot) return null;
|
||||
const destinationClips = await this.clips(frames, parent, parentRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
root = parentRoot;
|
||||
}
|
||||
const topViewport = await this.viewport(root.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,8 @@ import type {
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { screenshotPageRect } from "./geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -197,15 +199,14 @@ async function captureElementScreenshot(
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
const clip = screenshotPageRect(rect, geometry.topViewport);
|
||||
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
...clip.rect,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
@@ -984,26 +985,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
geometry: GeometryContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const metrics = await geometry.layoutMetrics({ tabId });
|
||||
throwIfAborted(signal, "observation");
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
const source = cssViewport(metrics);
|
||||
viewport = { width: source.width, height: source.height };
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
@@ -1016,12 +1010,13 @@ async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureVomObservationOptions,
|
||||
geometry: GeometryContext,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1096,9 +1091,16 @@ export async function captureVomObservation(
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
|
||||
const captured = await captureForVom(cdp, tabId, options, geometry);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
const documents = await captureFrameData<CdpAxNode>(
|
||||
cdp,
|
||||
tabId,
|
||||
captured,
|
||||
options.signal,
|
||||
geometry,
|
||||
);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const normalizedDocuments =
|
||||
documents.length === 1 && captured.iframeNodes.size > 0
|
||||
|
||||
@@ -754,7 +754,7 @@ describe("captureViewModel", () => {
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
@@ -802,6 +802,12 @@ describe("captureViewModel", () => {
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 2000,
|
||||
h: 1600,
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||
@@ -950,6 +956,12 @@ describe("captureViewModel", () => {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 800, height: 400 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
@@ -979,7 +991,7 @@ describe("captureViewModel", () => {
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
it("projects iframe documents independently of owner snapshot geometry", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -1040,6 +1052,12 @@ describe("captureViewModel", () => {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 800, height: 400 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
@@ -1048,7 +1066,7 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toBeNull();
|
||||
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1127,16 +1145,43 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
send: vi.fn(
|
||||
async (
|
||||
_t: number,
|
||||
method: string,
|
||||
params?: { backendNodeId?: number; objectId?: string },
|
||||
) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return {
|
||||
model: {
|
||||
content:
|
||||
params?.backendNodeId === 13
|
||||
? [10, 20, 310, 20, 310, 220, 10, 220]
|
||||
: [15, 26, 115, 26, 115, 106, 15, 106],
|
||||
},
|
||||
};
|
||||
if (method === "DOM.resolveNode")
|
||||
return { object: { objectId: String(params?.backendNodeId) } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
params?.objectId === "13"
|
||||
? { width: 300, height: 200 }
|
||||
: { width: 100, height: 80 },
|
||||
},
|
||||
};
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
},
|
||||
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
@@ -1150,7 +1195,7 @@ describe("captureViewModel", () => {
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
@@ -1192,7 +1237,7 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
|
||||
});
|
||||
|
||||
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
||||
|
||||
@@ -5,9 +5,15 @@
|
||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import {
|
||||
projectSnapshotRect,
|
||||
type SnapshotProjection,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
@@ -77,6 +83,8 @@ export interface CapturedViewModel {
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
signal?: AbortSignal;
|
||||
geometry?: GeometryContext;
|
||||
target?: CdpTarget;
|
||||
}
|
||||
|
||||
function captureAbortError(): Error {
|
||||
@@ -147,16 +155,6 @@ interface SnapshotReply {
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
interface LayoutMetricsReply {
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
@@ -353,7 +351,8 @@ interface ParseDocumentResult {
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: GeometryProjection | null;
|
||||
projection: SnapshotProjection | null;
|
||||
target: CdpTarget;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
@@ -363,15 +362,6 @@ function str(strings: string[], idx: number | undefined): string {
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
||||
if (!layoutW || !cssW) return 1;
|
||||
const dpr = layoutW / cssW;
|
||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
@@ -797,13 +787,11 @@ export async function collectOverlayExcludedBackendIds(
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
@@ -898,15 +886,17 @@ function parseDocumentNodes(
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
const input = snapshotViewportRect(
|
||||
b,
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
{ x: context.scrollX, y: context.scrollY },
|
||||
);
|
||||
if (input) {
|
||||
const { x, y, width: w, height: h } = input.rect;
|
||||
localRect = { x, y, w, h };
|
||||
}
|
||||
const bounds =
|
||||
input && context.projection ? projectSnapshotRect(input, context.projection) : null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
@@ -980,15 +970,14 @@ interface ParseFrameDocumentsResult {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function parseChildFrameDocuments(
|
||||
async function parseChildFrameDocuments(
|
||||
documents: SnapshotDocument[],
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
geometry: GeometryContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
): Promise<ParseFrameDocumentsResult> {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
@@ -1000,7 +989,6 @@ function parseChildFrameDocuments(
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
@@ -1008,14 +996,31 @@ function parseChildFrameDocuments(
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
let projection: SnapshotProjection | null = null;
|
||||
const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
|
||||
const parentProjection = parentContext.projection?.geometry;
|
||||
if (parentProjection) {
|
||||
// Each owner quad is target-relative; do not apply the parent's transform twice.
|
||||
const edge = parentProjection.edges[0];
|
||||
const clips = edge
|
||||
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
|
||||
: parentProjection.sourceClips;
|
||||
try {
|
||||
projection = await geometry.snapshotProjection(
|
||||
source,
|
||||
iframeBackendId,
|
||||
clips,
|
||||
parentProjection.topViewport,
|
||||
);
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.debug("[bsk capture] frame owner geometry unavailable", error);
|
||||
}
|
||||
}
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
frameId: source.frameId,
|
||||
target: source.target,
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
@@ -1023,7 +1028,7 @@ function parseChildFrameDocuments(
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
@@ -1031,13 +1036,12 @@ function parseChildFrameDocuments(
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
const nested = await parseChildFrameDocuments(
|
||||
documents,
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
geometry,
|
||||
nextVisited,
|
||||
);
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
@@ -1061,22 +1065,20 @@ export async function captureViewModel(
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
let metrics: LayoutMetricsReply = {};
|
||||
const target = options.target ?? { tabId };
|
||||
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
|
||||
let metrics: LayoutMetrics = {};
|
||||
try {
|
||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
metrics = await geometry.layoutMetrics(target);
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
const scrollX = vpSrc.pageX ?? 0;
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
const measured = cssViewport(metrics);
|
||||
const viewport: Viewport = { width: measured.width, height: measured.height };
|
||||
const scrollX = measured.scrollX;
|
||||
const scrollY = measured.scrollY;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwIfAborted(options.signal);
|
||||
@@ -1102,19 +1104,19 @@ export async function captureViewModel(
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
target,
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
source: { target, frameId: snapshotFrameId(doc0, strings) },
|
||||
geometry: { sourceClips: [], edges: [], topViewport: viewport },
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, topContext);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const frameParsed = await parseChildFrameDocuments(documents, strings, 0, topContext, geometry);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
@@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection | null,
|
||||
@@ -51,6 +41,7 @@ async function captureMissingFrameDocuments(
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
geometry: GeometryContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
@@ -62,10 +53,12 @@ async function captureMissingFrameDocuments(
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
geometry,
|
||||
target: frame.target,
|
||||
});
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
projection = await geometry.targetProjection(frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
@@ -171,10 +164,11 @@ export async function captureFrameData<T extends FrameAxNode>(
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
geometry = new GeometryContext(cdp, tabId, undefined, signal),
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const graph = await geometry.graph();
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
|
||||
Reference in New Issue
Block a user