mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(geometry): unify frame projection and screenshot coordinates
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
import { resolveNodeGeometry } from "../frame-geometry";
|
||||||
import {
|
import {
|
||||||
clipPolygon,
|
clipPolygon,
|
||||||
polygonArea,
|
polygonArea,
|
||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
rectPolygon,
|
rectPolygon,
|
||||||
regionBounds,
|
regionBounds,
|
||||||
} from "../geometry";
|
} from "../geometry";
|
||||||
|
import { GeometryContext } from "../geometry/frame-context";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
describe("frame geometry projection", () => {
|
describe("frame geometry projection", () => {
|
||||||
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
|
|||||||
throw new Error(`unexpected child command ${method}`);
|
throw new Error(`unexpected child command ${method}`);
|
||||||
}) as CdpRunner["sendToTarget"],
|
}) as CdpRunner["sendToTarget"],
|
||||||
};
|
};
|
||||||
const projection = await resolveFrameProjection(
|
const projection = await new GeometryContext(cdp, 4, {
|
||||||
cdp,
|
rootFrameId: "main",
|
||||||
{
|
frames: [
|
||||||
rootFrameId: "main",
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
frames: [
|
{
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
frameId: "same-process-parent",
|
||||||
{
|
parentFrameId: "main",
|
||||||
frameId: "same-process-parent",
|
ownerBackendNodeId: 10,
|
||||||
parentFrameId: "main",
|
target: { tabId: 4 },
|
||||||
ownerBackendNodeId: 10,
|
},
|
||||||
target: { tabId: 4 },
|
{
|
||||||
},
|
frameId: "oopif",
|
||||||
{
|
parentFrameId: "same-process-parent",
|
||||||
frameId: "oopif",
|
ownerBackendNodeId: 20,
|
||||||
parentFrameId: "same-process-parent",
|
target: { tabId: 4, sessionId: "oopif-session" },
|
||||||
ownerBackendNodeId: 20,
|
},
|
||||||
target: { tabId: 4, sessionId: "oopif-session" },
|
],
|
||||||
},
|
}).targetProjection("oopif");
|
||||||
],
|
|
||||||
},
|
|
||||||
"oopif",
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(projection?.edges).toEqual([
|
expect(projection?.edges).toEqual([
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,284 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||||
|
import { resolveNodeGeometry } from "../frame-geometry";
|
||||||
|
import { rectPolygon } from "../geometry";
|
||||||
|
import {
|
||||||
|
projectSnapshotRect,
|
||||||
|
screenshotPageRect,
|
||||||
|
snapshotViewportRect,
|
||||||
|
} from "../geometry/coordinate-types";
|
||||||
|
import { GeometryContext } from "../geometry/frame-context";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
const target = { tabId: 4 };
|
||||||
|
const graph: CdpFrameGraph = {
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target },
|
||||||
|
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
function driver() {
|
||||||
|
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
|
||||||
|
if (method === "Page.getLayoutMetrics")
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
|
||||||
|
if (method === "DOM.getBoxModel")
|
||||||
|
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
|
||||||
|
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||||
|
if (method === "Runtime.callFunctionOn")
|
||||||
|
return { result: { value: { width: 300, height: 200 } } };
|
||||||
|
if (method === "Runtime.releaseObject") return {};
|
||||||
|
throw new Error(method);
|
||||||
|
});
|
||||||
|
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("measurement geometry context", () => {
|
||||||
|
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
const context = new GeometryContext(cdp, 4);
|
||||||
|
const results = await Promise.all(
|
||||||
|
Array.from({ length: 20 }, () => context.targetProjection("child")),
|
||||||
|
);
|
||||||
|
expect(results.every((value) => value === results[0])).toBe(true);
|
||||||
|
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||||
|
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||||
|
"Page.getLayoutMetrics",
|
||||||
|
"DOM.getBoxModel",
|
||||||
|
]);
|
||||||
|
await new GeometryContext(cdp, 4).targetProjection("child");
|
||||||
|
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
|
||||||
|
expect(cdp.calls.mock.calls).toHaveLength(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
const context = new GeometryContext(cdp, 4);
|
||||||
|
const source = { target, frameId: "child" };
|
||||||
|
const projection = await context.snapshotProjection(source, 10, [], {
|
||||||
|
width: 1000,
|
||||||
|
height: 1000,
|
||||||
|
});
|
||||||
|
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
|
||||||
|
expect(projectSnapshotRect(input!, projection!)).toEqual({
|
||||||
|
x: 73.75,
|
||||||
|
y: 641.25,
|
||||||
|
width: 150,
|
||||||
|
height: 50,
|
||||||
|
});
|
||||||
|
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
|
||||||
|
expect(projectSnapshotRect(scrolled!, projection!)?.y).toBe(628.75);
|
||||||
|
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
|
||||||
|
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
|
||||||
|
).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects owner mismatches and invalid numeric bounds", () => {
|
||||||
|
const input = snapshotViewportRect(
|
||||||
|
[30, 50, 120, 40],
|
||||||
|
{ target, frameId: "main" },
|
||||||
|
{ x: 10, y: 20 },
|
||||||
|
)!;
|
||||||
|
const projection = {
|
||||||
|
source: { target, frameId: "main" },
|
||||||
|
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
|
||||||
|
};
|
||||||
|
expect(projectSnapshotRect(input, projection)).toEqual({
|
||||||
|
x: 20,
|
||||||
|
y: 30,
|
||||||
|
width: 120,
|
||||||
|
height: 40,
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
|
||||||
|
).toBeNull();
|
||||||
|
expect(
|
||||||
|
projectSnapshotRect(input, {
|
||||||
|
...projection,
|
||||||
|
source: { target: { tabId: 5 }, frameId: "main" },
|
||||||
|
}),
|
||||||
|
).toBeNull();
|
||||||
|
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
|
||||||
|
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not turn a border quad or a failed owner read into a content projection", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
|
||||||
|
})) as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
expect(
|
||||||
|
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||||
|
width: 100,
|
||||||
|
height: 100,
|
||||||
|
}),
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
|
||||||
|
{ target, frameId: "nested" },
|
||||||
|
10,
|
||||||
|
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
|
||||||
|
{ width: 1000, height: 1000 },
|
||||||
|
);
|
||||||
|
const input = snapshotViewportRect(
|
||||||
|
[0, 0, 300, 200],
|
||||||
|
{ target, frameId: "nested" },
|
||||||
|
{ x: 0, y: 0 },
|
||||||
|
);
|
||||||
|
expect(projectSnapshotRect(input!, projection!)).toEqual({
|
||||||
|
x: 60,
|
||||||
|
y: 620,
|
||||||
|
width: 100,
|
||||||
|
height: 100,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects malformed frame ancestry without looping", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
const cycle: CdpFrameGraph = {
|
||||||
|
rootFrameId: "a",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "a", parentFrameId: "b", target },
|
||||||
|
{ frameId: "b", parentFrameId: "a", target },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
|
||||||
|
expect(cdp.calls).not.toHaveBeenCalled();
|
||||||
|
const orphan: CdpFrameGraph = {
|
||||||
|
rootFrameId: "a",
|
||||||
|
frames: [{ frameId: "a", parentFrameId: "missing", target }],
|
||||||
|
};
|
||||||
|
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
expect(
|
||||||
|
await resolveNodeGeometry(
|
||||||
|
cdp,
|
||||||
|
4,
|
||||||
|
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
),
|
||||||
|
).toMatchObject({ code: "cdp_failed" });
|
||||||
|
expect(cdp.calls).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
|
||||||
|
let active = 0;
|
||||||
|
let peak = 0;
|
||||||
|
const release: Array<() => void> = [];
|
||||||
|
const send = vi.fn(async () => {
|
||||||
|
active++;
|
||||||
|
peak = Math.max(peak, active);
|
||||||
|
await new Promise<void>((resolve) => release.push(resolve));
|
||||||
|
active--;
|
||||||
|
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
|
||||||
|
});
|
||||||
|
const controller = new AbortController();
|
||||||
|
const context = new GeometryContext(
|
||||||
|
{ send: send as CdpRunner["send"] },
|
||||||
|
4,
|
||||||
|
undefined,
|
||||||
|
controller.signal,
|
||||||
|
);
|
||||||
|
const tasks = Array.from({ length: 8 }, (_, index) =>
|
||||||
|
context.viewport({ tabId: 4, sessionId: String(index) }),
|
||||||
|
);
|
||||||
|
const settled = Promise.allSettled(tasks);
|
||||||
|
expect(send).toHaveBeenCalledTimes(4);
|
||||||
|
controller.abort();
|
||||||
|
for (const done of release) done();
|
||||||
|
const results = await settled;
|
||||||
|
expect(peak).toBe(4);
|
||||||
|
expect(send).toHaveBeenCalledTimes(4);
|
||||||
|
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const cdp = driver();
|
||||||
|
const original = cdp.calls.getMockImplementation()!;
|
||||||
|
cdp.calls.mockImplementation(async (tab, method) => {
|
||||||
|
const result = await original(tab, method);
|
||||||
|
if (method === "DOM.resolveNode") controller.abort();
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
|
||||||
|
await expect(
|
||||||
|
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||||
|
width: 1000,
|
||||||
|
height: 1000,
|
||||||
|
}),
|
||||||
|
).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||||
|
"DOM.getBoxModel",
|
||||||
|
"DOM.resolveNode",
|
||||||
|
"Runtime.releaseObject",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads live owner geometry only after scrolling", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
const original = cdp.calls.getMockImplementation()!;
|
||||||
|
let scrolled = false;
|
||||||
|
cdp.calls.mockImplementation(async (tab, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") {
|
||||||
|
scrolled = true;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
expect(scrolled).toBe(true);
|
||||||
|
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
|
||||||
|
return original(tab, method);
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
await resolveNodeGeometry(
|
||||||
|
cdp,
|
||||||
|
4,
|
||||||
|
{ target, frameId: "child", backendNodeId: 20 },
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
),
|
||||||
|
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
|
||||||
|
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||||
|
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
|
||||||
|
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
|
||||||
|
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
|
||||||
|
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
|
||||||
|
expect(
|
||||||
|
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
|
||||||
|
).toBeNull();
|
||||||
|
expect(
|
||||||
|
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries a rejected measurement only in a new operation", async () => {
|
||||||
|
const cdp = driver();
|
||||||
|
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
|
||||||
|
const context = new GeometryContext(cdp, 4);
|
||||||
|
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||||
|
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||||
|
expect(cdp.calls).toHaveBeenCalledTimes(1);
|
||||||
|
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
|
||||||
|
width: 1000,
|
||||||
|
height: 1000,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -266,7 +266,15 @@ describe("handleScreenshot", () => {
|
|||||||
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
|
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("captures a clipped PNG when ref is given", async () => {
|
it.each([
|
||||||
|
{ zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } },
|
||||||
|
{ zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } },
|
||||||
|
{ zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } },
|
||||||
|
])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({
|
||||||
|
zoom,
|
||||||
|
scroll,
|
||||||
|
expected,
|
||||||
|
}) => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
ctx.refStore.set("e5", 999, { tabId: 7 });
|
ctx.refStore.set("e5", 999, { tabId: 7 });
|
||||||
@@ -274,6 +282,10 @@ describe("handleScreenshot", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||||
|
"Page.getLayoutMetrics": () => ({
|
||||||
|
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll },
|
||||||
|
cssVisualViewport: { zoom },
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
const captureVisibleTab = vi.fn();
|
const captureVisibleTab = vi.fn();
|
||||||
const res = await handleScreenshot(
|
const res = await handleScreenshot(
|
||||||
@@ -297,7 +309,8 @@ describe("handleScreenshot", () => {
|
|||||||
clip?: { x: number; y: number; width: number; height: number };
|
clip?: { x: number; y: number; width: number; height: number };
|
||||||
}
|
}
|
||||||
)?.clip;
|
)?.clip;
|
||||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
expect(clip).toMatchObject(expected);
|
||||||
|
expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
|
|||||||
@@ -1,24 +1,21 @@
|
|||||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||||
import {
|
import {
|
||||||
clipPolygon,
|
clipPolygon,
|
||||||
type GeometryProjection,
|
|
||||||
type Point,
|
type Point,
|
||||||
type Polygon,
|
type Polygon,
|
||||||
type ProjectiveEdge,
|
|
||||||
parseCdpQuad,
|
|
||||||
polygonArea,
|
polygonArea,
|
||||||
polygonCentroid,
|
polygonCentroid,
|
||||||
projectRegionToViewport,
|
projectRegionToViewport,
|
||||||
type Quad,
|
|
||||||
type Region,
|
type Region,
|
||||||
rectPolygon,
|
rectPolygon,
|
||||||
regionBounds,
|
regionBounds,
|
||||||
type Size,
|
|
||||||
type ViewportRect,
|
type ViewportRect,
|
||||||
} from "./geometry";
|
} from "./geometry";
|
||||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
import type { CssViewport } from "./geometry/coordinate-types";
|
||||||
|
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||||
|
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
|
||||||
|
|
||||||
export interface NodeAddress {
|
export interface NodeAddress {
|
||||||
target: CdpTarget;
|
target: CdpTarget;
|
||||||
@@ -27,6 +24,8 @@ export interface NodeAddress {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface ResolvedNodeGeometry {
|
export interface ResolvedNodeGeometry {
|
||||||
|
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
|
||||||
|
topViewport: CssViewport;
|
||||||
topVisibleRegions: Region;
|
topVisibleRegions: Region;
|
||||||
topBounds: ViewportRect;
|
topBounds: ViewportRect;
|
||||||
/** Point in the top-level tab viewport, used by root-target input events. */
|
/** Point in the top-level tab viewport, used by root-target input events. */
|
||||||
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
|
|||||||
return { code: "cdp_failed", message };
|
return { code: "cdp_failed", message };
|
||||||
}
|
}
|
||||||
|
|
||||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
|
||||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
|
||||||
}
|
|
||||||
|
|
||||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
|
||||||
const seen = new Set<string>();
|
|
||||||
let current = frame;
|
|
||||||
while (current.parentFrameId) {
|
|
||||||
if (seen.has(current.frameId)) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
const parent = byId.get(current.parentFrameId);
|
|
||||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return current;
|
|
||||||
}
|
|
||||||
|
|
||||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
|
||||||
const byId = frameMap(graph);
|
|
||||||
const path: CdpFrame[] = [];
|
|
||||||
const seen = new Set<string>();
|
|
||||||
let current = byId.get(frameId);
|
|
||||||
if (!current) return null;
|
|
||||||
while (current.parentFrameId) {
|
|
||||||
if (seen.has(current.frameId)) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
path.push(current);
|
|
||||||
const parent = byId.get(current.parentFrameId);
|
|
||||||
if (!parent) return null;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
|
||||||
const metrics = await sendToCdpTarget<{
|
|
||||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
|
||||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
|
||||||
const width = viewport.clientWidth ?? 0;
|
|
||||||
const height = viewport.clientHeight ?? 0;
|
|
||||||
return width > 0 && height > 0 ? { width, height } : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function ownerContentQuad(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
parent: CdpFrame,
|
|
||||||
ownerBackendNodeId: number,
|
|
||||||
): Promise<Quad | null> {
|
|
||||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
|
||||||
cdp,
|
|
||||||
parent.target,
|
|
||||||
"DOM.getBoxModel",
|
|
||||||
{ backendNodeId: ownerBackendNodeId },
|
|
||||||
);
|
|
||||||
return parseCdpQuad(result.model?.content);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function sameTargetFrameClips(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
byId: Map<string, CdpFrame>,
|
|
||||||
frame: CdpFrame,
|
|
||||||
targetRoot: CdpFrame,
|
|
||||||
): Promise<Polygon[] | null> {
|
|
||||||
const clips: Polygon[] = [];
|
|
||||||
let current = frame;
|
|
||||||
const seen = new Set<string>();
|
|
||||||
while (current.frameId !== targetRoot.frameId) {
|
|
||||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
const parent = byId.get(current.parentFrameId);
|
|
||||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
|
||||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
|
||||||
if (!clip) return null;
|
|
||||||
clips.push(clip);
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return clips;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function resolveFrameProjection(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
graph: CdpFrameGraph,
|
|
||||||
frameId: string,
|
|
||||||
): Promise<GeometryProjection | null> {
|
|
||||||
const byId = frameMap(graph);
|
|
||||||
const frame = byId.get(frameId);
|
|
||||||
if (!frame) return null;
|
|
||||||
let targetRoot = targetRootFrame(byId, frame);
|
|
||||||
if (!targetRoot) return null;
|
|
||||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
|
||||||
if (!sourceViewport) return null;
|
|
||||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
|
||||||
if (!sourceClips) return null;
|
|
||||||
|
|
||||||
const edges: ProjectiveEdge[] = [];
|
|
||||||
while (targetRoot.parentFrameId) {
|
|
||||||
const parent = byId.get(targetRoot.parentFrameId);
|
|
||||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
|
||||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
|
||||||
if (!destinationQuad) return null;
|
|
||||||
const source =
|
|
||||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
|
||||||
if (!source) return null;
|
|
||||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
|
||||||
if (!parentTargetRoot) return null;
|
|
||||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
|
||||||
if (!destinationClips) return null;
|
|
||||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
|
||||||
targetRoot = parentTargetRoot;
|
|
||||||
}
|
|
||||||
|
|
||||||
const topViewport =
|
|
||||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
|
||||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
|
||||||
if (!cdp.getFrameGraph) return null;
|
|
||||||
try {
|
|
||||||
return await cdp.getFrameGraph(tabId);
|
|
||||||
} catch (error) {
|
|
||||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function scrollFrameOwners(
|
async function scrollFrameOwners(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
graph: CdpFrameGraph,
|
context: GeometryContext,
|
||||||
frameId: string,
|
frameId: string,
|
||||||
): Promise<RpcError | null> {
|
): Promise<RpcError | null> {
|
||||||
const byId = frameMap(graph);
|
const ancestry = await context.ancestry(frameId);
|
||||||
const ancestry = frameAncestry(graph, frameId);
|
|
||||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||||
for (const child of [...ancestry].reverse()) {
|
for (const child of [...ancestry].reverse()) {
|
||||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
|
||||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||||
}
|
}
|
||||||
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrollElementWithFrameGraph(
|
async function scrollElementWithContext(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
target: CdpTarget,
|
target: CdpTarget,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
frameId: string | undefined,
|
frameId: string | undefined,
|
||||||
graph: CdpFrameGraph | null,
|
context: GeometryContext,
|
||||||
): Promise<RpcError | null> {
|
): Promise<RpcError | null> {
|
||||||
if (frameId) {
|
if (frameId) {
|
||||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
const frame = await context.frame(frameId);
|
||||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
|
||||||
|
return geometryError("node frame does not belong to its target");
|
||||||
|
}
|
||||||
|
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
|
||||||
if (error) return error;
|
if (error) return error;
|
||||||
} else if (target.sessionId) {
|
} else if (target.sessionId) {
|
||||||
return geometryError("an OOPIF node address requires frameId");
|
return geometryError("an OOPIF node address requires frameId");
|
||||||
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
|
|||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
frameId?: string,
|
frameId?: string,
|
||||||
): Promise<RpcError | null> {
|
): Promise<RpcError | null> {
|
||||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
return scrollElementWithContext(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
target,
|
||||||
|
backendNodeId,
|
||||||
|
frameId,
|
||||||
|
new GeometryContext(cdp, tabId),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function largestRegion(regions: Region): Polygon | null {
|
function largestRegion(regions: Region): Polygon | null {
|
||||||
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
|
|||||||
if (address.target.sessionId && !address.frameId) {
|
if (address.target.sessionId && !address.frameId) {
|
||||||
return geometryError("an OOPIF node address requires frameId");
|
return geometryError("an OOPIF node address requires frameId");
|
||||||
}
|
}
|
||||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||||
|
const context = new GeometryContext(cdp, tabId);
|
||||||
|
const graph = address.frameId ? await context.graph() : null;
|
||||||
|
if (address.frameId && graph) {
|
||||||
|
const frame = await context.frame(address.frameId);
|
||||||
|
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
|
||||||
|
return geometryError("node frame does not belong to its target");
|
||||||
|
}
|
||||||
|
}
|
||||||
if (address.frameId && !graph) {
|
if (address.frameId && !graph) {
|
||||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options.scrollIntoView) {
|
if (options.scrollIntoView) {
|
||||||
const scrollError = await scrollElementWithFrameGraph(
|
const scrollError = await scrollElementWithContext(
|
||||||
cdp,
|
cdp,
|
||||||
tabId,
|
tabId,
|
||||||
address.target,
|
address.target,
|
||||||
address.backendNodeId,
|
address.backendNodeId,
|
||||||
address.frameId,
|
address.frameId,
|
||||||
graph,
|
context,
|
||||||
);
|
);
|
||||||
if (scrollError) return scrollError;
|
if (scrollError) return scrollError;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only topology was read above. Begin live measurements after scrolling.
|
||||||
const localRegion = await nodeContentRegion(
|
const localRegion = await nodeContentRegion(
|
||||||
cdpRunnerForTarget(cdp, address.target),
|
cdpRunnerForTarget(cdp, address.target),
|
||||||
tabId,
|
tabId,
|
||||||
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
|
|||||||
let topVisibleRegions: Region;
|
let topVisibleRegions: Region;
|
||||||
let targetActionPoint: Point | null = null;
|
let targetActionPoint: Point | null = null;
|
||||||
if (address.frameId && graph) {
|
if (address.frameId && graph) {
|
||||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
const projection = await context.targetProjection(address.frameId);
|
||||||
if (!projection)
|
if (!projection)
|
||||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||||
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
|
|||||||
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const viewport = await targetViewport(cdp, address.target);
|
const viewport = await context.viewport(address.target);
|
||||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||||
topVisibleRegions = localRegion
|
topVisibleRegions = localRegion
|
||||||
.map((polygon) =>
|
.map((polygon) =>
|
||||||
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
|
|||||||
return { code: "permission_denied", message: "element not visible in its target" };
|
return { code: "permission_denied", message: "element not visible in its target" };
|
||||||
}
|
}
|
||||||
targetActionPoint ??= actionPoint;
|
targetActionPoint ??= actionPoint;
|
||||||
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
|
return {
|
||||||
|
topVisibleRegions,
|
||||||
|
topBounds,
|
||||||
|
actionPoint,
|
||||||
|
targetActionPoint,
|
||||||
|
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
|
||||||
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return geometryError(error instanceof Error ? error.message : String(error));
|
return geometryError(error instanceof Error ? error.message : String(error));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -245,22 +245,3 @@ export function projectRectToViewport(
|
|||||||
if (!rect) return null;
|
if (!rect) return null;
|
||||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function childFrameProjection(
|
|
||||||
parent: GeometryProjection,
|
|
||||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
|
||||||
): GeometryProjection {
|
|
||||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
|
||||||
return {
|
|
||||||
sourceClips: [],
|
|
||||||
edges: [
|
|
||||||
{
|
|
||||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
|
||||||
destinationQuad,
|
|
||||||
destinationClips: parent.sourceClips,
|
|
||||||
},
|
|
||||||
...parent.edges,
|
|
||||||
],
|
|
||||||
topViewport: parent.topViewport,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||||
|
import type { GeometryProjection, ViewportRect } from "../geometry";
|
||||||
|
import { projectRectToViewport } from "../geometry";
|
||||||
|
|
||||||
|
export interface CoordinateOwner {
|
||||||
|
target: CdpTarget;
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FrameViewportRect {
|
||||||
|
space: "frame-viewport-css";
|
||||||
|
owner: CoordinateOwner;
|
||||||
|
rect: ViewportRect;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SnapshotProjection {
|
||||||
|
source: CoordinateOwner;
|
||||||
|
geometry: GeometryProjection;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
|
||||||
|
export function snapshotViewportRect(
|
||||||
|
bounds: number[],
|
||||||
|
owner: CoordinateOwner,
|
||||||
|
scroll: { x: number; y: number },
|
||||||
|
): FrameViewportRect | null {
|
||||||
|
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
|
||||||
|
const [x, y, width, height] = bounds;
|
||||||
|
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
space: "frame-viewport-css",
|
||||||
|
owner,
|
||||||
|
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectSnapshotRect(
|
||||||
|
input: FrameViewportRect,
|
||||||
|
projection: SnapshotProjection,
|
||||||
|
): ViewportRect | null {
|
||||||
|
if (
|
||||||
|
input.owner.frameId !== projection.source.frameId ||
|
||||||
|
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const { x, y, width: w, height: h } = input.rect;
|
||||||
|
return projectRectToViewport({ x, y, w, h }, projection.geometry);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
|
||||||
|
export interface CssViewport {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
scrollX: number;
|
||||||
|
scrollY: number;
|
||||||
|
cssToDip: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
|
||||||
|
export function screenshotPageRect(
|
||||||
|
rect: ViewportRect,
|
||||||
|
viewport: CssViewport,
|
||||||
|
): { space: "page-dip"; rect: ViewportRect } | null {
|
||||||
|
const { cssToDip, scrollX, scrollY } = viewport;
|
||||||
|
if (
|
||||||
|
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
|
||||||
|
cssToDip <= 0 ||
|
||||||
|
rect.width <= 0 ||
|
||||||
|
rect.height <= 0
|
||||||
|
)
|
||||||
|
return null;
|
||||||
|
return {
|
||||||
|
space: "page-dip",
|
||||||
|
rect: {
|
||||||
|
x: (rect.x + scrollX) * cssToDip,
|
||||||
|
y: (rect.y + scrollY) * cssToDip,
|
||||||
|
width: rect.width * cssToDip,
|
||||||
|
height: rect.height * cssToDip,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,297 @@
|
|||||||
|
import {
|
||||||
|
type CdpFrame,
|
||||||
|
type CdpFrameGraph,
|
||||||
|
type CdpTarget,
|
||||||
|
cdpTargetKey,
|
||||||
|
} from "@/browser-driver/frame-graph";
|
||||||
|
import {
|
||||||
|
type GeometryProjection,
|
||||||
|
type Polygon,
|
||||||
|
type ProjectiveEdge,
|
||||||
|
parseCdpQuad,
|
||||||
|
type Quad,
|
||||||
|
type Size,
|
||||||
|
} from "../geometry";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||||
|
import type { CoordinateOwner, CssViewport, SnapshotProjection } from "./coordinate-types";
|
||||||
|
|
||||||
|
export interface LayoutMetrics {
|
||||||
|
cssVisualViewport?: { zoom?: number };
|
||||||
|
visualViewport?: { zoom?: number };
|
||||||
|
cssLayoutViewport?: {
|
||||||
|
clientWidth?: number;
|
||||||
|
clientHeight?: number;
|
||||||
|
pageX?: number;
|
||||||
|
pageY?: number;
|
||||||
|
};
|
||||||
|
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
|
||||||
|
export function cssViewport(metrics: LayoutMetrics): CssViewport {
|
||||||
|
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
|
||||||
|
return {
|
||||||
|
width: value?.clientWidth ?? 0,
|
||||||
|
height: value?.clientHeight ?? 0,
|
||||||
|
scrollX: value?.pageX ?? 0,
|
||||||
|
scrollY: value?.pageY ?? 0,
|
||||||
|
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One read-only measurement phase. Discard after scrolling or any later operation. */
|
||||||
|
export class GeometryContext {
|
||||||
|
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
|
||||||
|
private readonly owners = new Map<string, Promise<Quad | null>>();
|
||||||
|
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
|
||||||
|
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
|
||||||
|
private active = 0;
|
||||||
|
private readonly waiting: Array<() => void> = [];
|
||||||
|
private graphPromise?: Promise<CdpFrameGraph | null>;
|
||||||
|
private framesPromise?: Promise<Map<string, CdpFrame>>;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly cdp: CdpRunner,
|
||||||
|
private readonly tabId: number,
|
||||||
|
graph?: CdpFrameGraph,
|
||||||
|
private readonly signal?: AbortSignal,
|
||||||
|
) {
|
||||||
|
if (graph) this.graphPromise = Promise.resolve(graph);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async request<T>(
|
||||||
|
target: CdpTarget,
|
||||||
|
method: string,
|
||||||
|
params: object,
|
||||||
|
cleanup = false,
|
||||||
|
): Promise<T> {
|
||||||
|
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
|
||||||
|
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||||
|
else this.active++;
|
||||||
|
try {
|
||||||
|
if (!cleanup && this.signal?.aborted)
|
||||||
|
throw new DOMException("geometry aborted", "AbortError");
|
||||||
|
return await sendToCdpTarget<T>(this.cdp, target, method, params);
|
||||||
|
} finally {
|
||||||
|
const next = this.waiting.shift();
|
||||||
|
if (next) next();
|
||||||
|
else this.active--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
graph(): Promise<CdpFrameGraph | null> {
|
||||||
|
this.graphPromise ??=
|
||||||
|
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
|
||||||
|
return this.graphPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
private frames(): Promise<Map<string, CdpFrame>> {
|
||||||
|
this.framesPromise ??= this.graph().then(
|
||||||
|
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
|
||||||
|
);
|
||||||
|
return this.framesPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async frame(frameId: string): Promise<CdpFrame | undefined> {
|
||||||
|
return (await this.frames()).get(frameId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Topology only: safe to resolve before the caller scrolls frame owners. */
|
||||||
|
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
|
||||||
|
const frames = await this.frames();
|
||||||
|
const path: CdpFrame[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let current = frames.get(frameId);
|
||||||
|
if (!current) return null;
|
||||||
|
while (current.parentFrameId) {
|
||||||
|
if (seen.has(current.frameId)) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
path.push(current);
|
||||||
|
const parent = frames.get(current.parentFrameId);
|
||||||
|
if (!parent) return null;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
|
||||||
|
const key = cdpTargetKey(target);
|
||||||
|
let promise = this.metrics.get(key);
|
||||||
|
if (!promise) {
|
||||||
|
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
|
||||||
|
this.metrics.set(key, promise);
|
||||||
|
}
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async viewport(target: CdpTarget): Promise<Size | null> {
|
||||||
|
const { width, height } = cssViewport(await this.layoutMetrics(target));
|
||||||
|
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
|
||||||
|
? { width, height }
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
|
||||||
|
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||||
|
let promise = this.owners.get(key);
|
||||||
|
if (!promise) {
|
||||||
|
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
|
||||||
|
backendNodeId,
|
||||||
|
}).then((reply) => parseCdpQuad(reply.model?.content));
|
||||||
|
this.owners.set(key, promise);
|
||||||
|
}
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Same-target owner quads are already target-relative, including nested owners. */
|
||||||
|
async snapshotProjection(
|
||||||
|
source: CoordinateOwner,
|
||||||
|
ownerBackendNodeId: number,
|
||||||
|
ancestorClips: Polygon[],
|
||||||
|
viewport: Size,
|
||||||
|
): Promise<SnapshotProjection | null> {
|
||||||
|
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
||||||
|
let promise = this.snapshotOwners.get(key);
|
||||||
|
if (!promise) {
|
||||||
|
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||||
|
this.snapshotOwners.set(key, promise);
|
||||||
|
}
|
||||||
|
const owner = await promise;
|
||||||
|
return owner
|
||||||
|
? {
|
||||||
|
source,
|
||||||
|
geometry: {
|
||||||
|
sourceClips: [],
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
sourceViewport: owner.size,
|
||||||
|
destinationQuad: owner.quad,
|
||||||
|
destinationClips: ancestorClips,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
topViewport: viewport,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async snapshotOwner(
|
||||||
|
target: CdpTarget,
|
||||||
|
backendNodeId: number,
|
||||||
|
): Promise<{ quad: Quad; size: Size } | null> {
|
||||||
|
const quad = await this.ownerContent(target, backendNodeId);
|
||||||
|
if (!quad) return null;
|
||||||
|
const resolved = await this.request<{ object?: { objectId?: string } }>(
|
||||||
|
target,
|
||||||
|
"DOM.resolveNode",
|
||||||
|
{ backendNodeId },
|
||||||
|
);
|
||||||
|
const objectId = resolved.object?.objectId;
|
||||||
|
if (!objectId) return null;
|
||||||
|
try {
|
||||||
|
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
|
||||||
|
target,
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
{
|
||||||
|
objectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
if (!this.isConnected) return null;
|
||||||
|
const style = getComputedStyle(this);
|
||||||
|
return {
|
||||||
|
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
|
||||||
|
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
|
||||||
|
};
|
||||||
|
}`,
|
||||||
|
returnByValue: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const size = reply.result?.value;
|
||||||
|
return !reply.exceptionDetails &&
|
||||||
|
size &&
|
||||||
|
Number.isFinite(size.width) &&
|
||||||
|
Number.isFinite(size.height) &&
|
||||||
|
size.width > 0 &&
|
||||||
|
size.height > 0
|
||||||
|
? { quad, size }
|
||||||
|
: null;
|
||||||
|
} finally {
|
||||||
|
// Cleanup must still run if cancellation arrived after resolving the object.
|
||||||
|
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let current = frame;
|
||||||
|
while (current.parentFrameId) {
|
||||||
|
if (seen.has(current.frameId)) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
const parent = frames.get(current.parentFrameId);
|
||||||
|
if (!parent) return null;
|
||||||
|
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async clips(
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
frame: CdpFrame,
|
||||||
|
root: CdpFrame,
|
||||||
|
): Promise<Polygon[] | null> {
|
||||||
|
const clips: Polygon[] = [];
|
||||||
|
let current = frame;
|
||||||
|
const seen = new Set<string>();
|
||||||
|
while (current.frameId !== root.frameId) {
|
||||||
|
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
const parent = frames.get(current.parentFrameId);
|
||||||
|
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||||
|
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
|
||||||
|
if (!quad) return null;
|
||||||
|
clips.push(quad);
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return clips;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Input is already in the frame's CDP target viewport, not its local viewport. */
|
||||||
|
targetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||||
|
let promise = this.projections.get(frameId);
|
||||||
|
if (!promise) {
|
||||||
|
promise = this.buildTargetProjection(frameId);
|
||||||
|
this.projections.set(frameId, promise);
|
||||||
|
}
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||||
|
const frames = await this.frames();
|
||||||
|
const frame = frames.get(frameId);
|
||||||
|
if (!frame) return null;
|
||||||
|
let root = this.targetRoot(frames, frame);
|
||||||
|
if (!root) return null;
|
||||||
|
const sourceViewport = await this.viewport(root.target);
|
||||||
|
if (!sourceViewport) return null;
|
||||||
|
const sourceClips = await this.clips(frames, frame, root);
|
||||||
|
if (!sourceClips) return null;
|
||||||
|
const edges: ProjectiveEdge[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
while (root.parentFrameId) {
|
||||||
|
if (seen.has(root.frameId)) return null;
|
||||||
|
seen.add(root.frameId);
|
||||||
|
const parent = frames.get(root.parentFrameId);
|
||||||
|
if (!parent || root.ownerBackendNodeId === undefined) return null;
|
||||||
|
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
|
||||||
|
const source = await this.viewport(root.target);
|
||||||
|
const parentRoot = this.targetRoot(frames, parent);
|
||||||
|
if (!destinationQuad || !source || !parentRoot) return null;
|
||||||
|
const destinationClips = await this.clips(frames, parent, parentRoot);
|
||||||
|
if (!destinationClips) return null;
|
||||||
|
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||||
|
root = parentRoot;
|
||||||
|
}
|
||||||
|
const topViewport = await this.viewport(root.target);
|
||||||
|
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,6 +34,8 @@ import type {
|
|||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
import { resolveNodeGeometry } from "./frame-geometry";
|
import { resolveNodeGeometry } from "./frame-geometry";
|
||||||
|
import { screenshotPageRect } from "./geometry/coordinate-types";
|
||||||
|
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||||
import {
|
import {
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
enforceToolTargetScope,
|
enforceToolTargetScope,
|
||||||
@@ -197,15 +199,14 @@ async function captureElementScreenshot(
|
|||||||
if (isRpcError(geometry)) return geometry;
|
if (isRpcError(geometry)) return geometry;
|
||||||
if (signal?.aborted) return cancelled("screenshot");
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const rect = geometry.topBounds;
|
const rect = geometry.topBounds;
|
||||||
|
const clip = screenshotPageRect(rect, geometry.topViewport);
|
||||||
|
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||||
format: "png",
|
format: "png",
|
||||||
clip: {
|
clip: {
|
||||||
x: rect.x,
|
...clip.rect,
|
||||||
y: rect.y,
|
|
||||||
width: rect.width,
|
|
||||||
height: rect.height,
|
|
||||||
scale: 1,
|
scale: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -984,26 +985,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie
|
|||||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||||
}
|
}
|
||||||
|
|
||||||
interface LayoutMetricsViewportReply {
|
|
||||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fallbackCapturedViewModel(
|
async function fallbackCapturedViewModel(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
|
geometry: GeometryContext,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
throwIfAborted(signal, "observation");
|
throwIfAborted(signal, "observation");
|
||||||
let viewport = { width: 0, height: 0 };
|
let viewport = { width: 0, height: 0 };
|
||||||
try {
|
try {
|
||||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
const metrics = await geometry.layoutMetrics({ tabId });
|
||||||
throwIfAborted(signal, "observation");
|
throwIfAborted(signal, "observation");
|
||||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
const source = cssViewport(metrics);
|
||||||
viewport = {
|
viewport = { width: source.width, height: source.height };
|
||||||
width: source.clientWidth ?? 0,
|
|
||||||
height: source.clientHeight ?? 0,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isAbortError(error)) throw error;
|
if (isAbortError(error)) throw error;
|
||||||
}
|
}
|
||||||
@@ -1016,12 +1010,13 @@ async function captureForVom(
|
|||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
options: CaptureVomObservationOptions,
|
options: CaptureVomObservationOptions,
|
||||||
|
geometry: GeometryContext,
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
try {
|
try {
|
||||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isAbortError(error)) throw error;
|
if (isAbortError(error)) throw error;
|
||||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1096,9 +1091,16 @@ export async function captureVomObservation(
|
|||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
const captured = await captureForVom(cdp, tabId, options);
|
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
|
||||||
|
const captured = await captureForVom(cdp, tabId, options, geometry);
|
||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
const documents = await captureFrameData<CdpAxNode>(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
captured,
|
||||||
|
options.signal,
|
||||||
|
geometry,
|
||||||
|
);
|
||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
const normalizedDocuments =
|
const normalizedDocuments =
|
||||||
documents.length === 1 && captured.iframeNodes.size > 0
|
documents.length === 1 && captured.iframeNodes.size > 0
|
||||||
|
|||||||
@@ -754,7 +754,7 @@ describe("captureViewModel", () => {
|
|||||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
|
||||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||||
const i = (s: string) => S.indexOf(s);
|
const i = (s: string) => S.indexOf(s);
|
||||||
const snapshot = {
|
const snapshot = {
|
||||||
@@ -802,6 +802,12 @@ describe("captureViewModel", () => {
|
|||||||
w: 1000,
|
w: 1000,
|
||||||
h: 800,
|
h: 800,
|
||||||
});
|
});
|
||||||
|
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
|
||||||
|
x: 0,
|
||||||
|
y: 0,
|
||||||
|
w: 2000,
|
||||||
|
h: 1600,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||||
@@ -950,6 +956,12 @@ describe("captureViewModel", () => {
|
|||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (method === "DOM.getBoxModel")
|
||||||
|
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||||
|
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||||
|
if (method === "Runtime.callFunctionOn")
|
||||||
|
return { result: { value: { width: 800, height: 400 } } };
|
||||||
|
if (method === "Runtime.releaseObject") return {};
|
||||||
throw new Error(method);
|
throw new Error(method);
|
||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
@@ -979,7 +991,7 @@ describe("captureViewModel", () => {
|
|||||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
it("projects iframe documents independently of owner snapshot geometry", async () => {
|
||||||
const S = [
|
const S = [
|
||||||
"html",
|
"html",
|
||||||
"body",
|
"body",
|
||||||
@@ -1040,6 +1052,12 @@ describe("captureViewModel", () => {
|
|||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (method === "DOM.getBoxModel")
|
||||||
|
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||||
|
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||||
|
if (method === "Runtime.callFunctionOn")
|
||||||
|
return { result: { value: { width: 800, height: 400 } } };
|
||||||
|
if (method === "Runtime.releaseObject") return {};
|
||||||
throw new Error(method);
|
throw new Error(method);
|
||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
@@ -1048,7 +1066,7 @@ describe("captureViewModel", () => {
|
|||||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||||
|
|
||||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||||
expect(input?.rect).toBeNull();
|
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||||
@@ -1127,16 +1145,43 @@ describe("captureViewModel", () => {
|
|||||||
],
|
],
|
||||||
};
|
};
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_t: number, method: string) => {
|
send: vi.fn(
|
||||||
if (method === "DOMSnapshot.enable") return {};
|
async (
|
||||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
_t: number,
|
||||||
if (method === "Page.getLayoutMetrics") {
|
method: string,
|
||||||
return {
|
params?: { backendNodeId?: number; objectId?: string },
|
||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
) => {
|
||||||
};
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
}
|
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||||
throw new Error(method);
|
if (method === "Page.getLayoutMetrics") {
|
||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel")
|
||||||
|
return {
|
||||||
|
model: {
|
||||||
|
content:
|
||||||
|
params?.backendNodeId === 13
|
||||||
|
? [10, 20, 310, 20, 310, 220, 10, 220]
|
||||||
|
: [15, 26, 115, 26, 115, 106, 15, 106],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if (method === "DOM.resolveNode")
|
||||||
|
return { object: { objectId: String(params?.backendNodeId) } };
|
||||||
|
if (method === "Runtime.callFunctionOn")
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value:
|
||||||
|
params?.objectId === "13"
|
||||||
|
? { width: 300, height: 200 }
|
||||||
|
: { width: 100, height: 80 },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if (method === "Runtime.releaseObject") return {};
|
||||||
|
throw new Error(method);
|
||||||
|
},
|
||||||
|
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
|
|
||||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||||
@@ -1150,7 +1195,7 @@ describe("captureViewModel", () => {
|
|||||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
|
||||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||||
const i = (s: string) => S.indexOf(s);
|
const i = (s: string) => S.indexOf(s);
|
||||||
const snapshot = {
|
const snapshot = {
|
||||||
@@ -1192,7 +1237,7 @@ describe("captureViewModel", () => {
|
|||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
const { nodes } = await captureViewModel(cdp, 4);
|
const { nodes } = await captureViewModel(cdp, 4);
|
||||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
||||||
|
|||||||
@@ -5,9 +5,15 @@
|
|||||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||||
|
|
||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
import {
|
||||||
|
projectSnapshotRect,
|
||||||
|
type SnapshotProjection,
|
||||||
|
snapshotViewportRect,
|
||||||
|
} from "../geometry/coordinate-types";
|
||||||
|
import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||||
|
|
||||||
@@ -77,6 +83,8 @@ export interface CapturedViewModel {
|
|||||||
|
|
||||||
export interface CaptureViewModelOptions {
|
export interface CaptureViewModelOptions {
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
geometry?: GeometryContext;
|
||||||
|
target?: CdpTarget;
|
||||||
}
|
}
|
||||||
|
|
||||||
function captureAbortError(): Error {
|
function captureAbortError(): Error {
|
||||||
@@ -147,16 +155,6 @@ interface SnapshotReply {
|
|||||||
documents?: SnapshotDocument[];
|
documents?: SnapshotDocument[];
|
||||||
}
|
}
|
||||||
|
|
||||||
interface LayoutMetricsReply {
|
|
||||||
cssLayoutViewport?: {
|
|
||||||
clientWidth?: number;
|
|
||||||
clientHeight?: number;
|
|
||||||
pageX?: number;
|
|
||||||
pageY?: number;
|
|
||||||
};
|
|
||||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
|
||||||
}
|
|
||||||
|
|
||||||
function isFormControlTag(tag: string): boolean {
|
function isFormControlTag(tag: string): boolean {
|
||||||
return tag === "input" || tag === "textarea" || tag === "select";
|
return tag === "input" || tag === "textarea" || tag === "select";
|
||||||
}
|
}
|
||||||
@@ -353,7 +351,8 @@ interface ParseDocumentResult {
|
|||||||
interface FrameContext {
|
interface FrameContext {
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
ownerFrameBackendNodeId: number | null;
|
ownerFrameBackendNodeId: number | null;
|
||||||
projection: GeometryProjection | null;
|
projection: SnapshotProjection | null;
|
||||||
|
target: CdpTarget;
|
||||||
scrollX: number;
|
scrollX: number;
|
||||||
scrollY: number;
|
scrollY: number;
|
||||||
}
|
}
|
||||||
@@ -363,15 +362,6 @@ function str(strings: string[], idx: number | undefined): string {
|
|||||||
return strings[idx] ?? "";
|
return strings[idx] ?? "";
|
||||||
}
|
}
|
||||||
|
|
||||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
|
||||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
|
||||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
|
||||||
if (!layoutW || !cssW) return 1;
|
|
||||||
const dpr = layoutW / cssW;
|
|
||||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
|
||||||
return dpr >= 1 ? dpr : 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||||
const out = new Map<number, number>();
|
const out = new Map<number, number>();
|
||||||
if (!sparse?.index || !sparse.value) return out;
|
if (!sparse?.index || !sparse.value) return out;
|
||||||
@@ -797,13 +787,11 @@ export async function collectOverlayExcludedBackendIds(
|
|||||||
*
|
*
|
||||||
* @param doc - the raw DOMSnapshot document object
|
* @param doc - the raw DOMSnapshot document object
|
||||||
* @param strings - the shared string table for the whole snapshot
|
* @param strings - the shared string table for the whole snapshot
|
||||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
|
||||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||||
*/
|
*/
|
||||||
function parseDocumentNodes(
|
function parseDocumentNodes(
|
||||||
doc: SnapshotDocument,
|
doc: SnapshotDocument,
|
||||||
strings: string[],
|
strings: string[],
|
||||||
dpr: number,
|
|
||||||
context: FrameContext,
|
context: FrameContext,
|
||||||
): ParseDocumentResult {
|
): ParseDocumentResult {
|
||||||
const dn = doc.nodes;
|
const dn = doc.nodes;
|
||||||
@@ -898,15 +886,17 @@ function parseDocumentNodes(
|
|||||||
if (li !== undefined) {
|
if (li !== undefined) {
|
||||||
const b = dl?.bounds?.[li];
|
const b = dl?.bounds?.[li];
|
||||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||||
localRect = {
|
const input = snapshotViewportRect(
|
||||||
x: b[0] / dpr - context.scrollX,
|
b,
|
||||||
y: b[1] / dpr - context.scrollY,
|
{ target: context.target, frameId: context.frameId },
|
||||||
w: b[2] / dpr,
|
{ x: context.scrollX, y: context.scrollY },
|
||||||
h: b[3] / dpr,
|
);
|
||||||
};
|
if (input) {
|
||||||
const bounds = context.projection
|
const { x, y, width: w, height: h } = input.rect;
|
||||||
? projectRectToViewport(localRect, context.projection)
|
localRect = { x, y, w, h };
|
||||||
: null;
|
}
|
||||||
|
const bounds =
|
||||||
|
input && context.projection ? projectSnapshotRect(input, context.projection) : null;
|
||||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||||
}
|
}
|
||||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||||
@@ -980,15 +970,14 @@ interface ParseFrameDocumentsResult {
|
|||||||
excludedBackendNodeIds: Set<number>;
|
excludedBackendNodeIds: Set<number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseChildFrameDocuments(
|
async function parseChildFrameDocuments(
|
||||||
documents: SnapshotDocument[],
|
documents: SnapshotDocument[],
|
||||||
strings: string[],
|
strings: string[],
|
||||||
dpr: number,
|
|
||||||
parentDocIndex: number,
|
parentDocIndex: number,
|
||||||
parentNodes: CapturedNode[],
|
|
||||||
parentContext: FrameContext,
|
parentContext: FrameContext,
|
||||||
|
geometry: GeometryContext,
|
||||||
visited = new Set<number>(),
|
visited = new Set<number>(),
|
||||||
): ParseFrameDocumentsResult {
|
): Promise<ParseFrameDocumentsResult> {
|
||||||
const iframeNodes: CapturedIframeNodes = new Map();
|
const iframeNodes: CapturedIframeNodes = new Map();
|
||||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||||
const frameParentIds = new Map<string, string>();
|
const frameParentIds = new Map<string, string>();
|
||||||
@@ -1000,7 +989,6 @@ function parseChildFrameDocuments(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
|
||||||
|
|
||||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||||
if (visited.has(childDocIndex)) continue;
|
if (visited.has(childDocIndex)) continue;
|
||||||
@@ -1008,14 +996,31 @@ function parseChildFrameDocuments(
|
|||||||
if (!childDoc) continue;
|
if (!childDoc) continue;
|
||||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||||
if (iframeBackendId === undefined) continue;
|
if (iframeBackendId === undefined) continue;
|
||||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
let projection: SnapshotProjection | null = null;
|
||||||
const projection =
|
const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
|
||||||
parentContext.projection && iframeNode?.localRect
|
const parentProjection = parentContext.projection?.geometry;
|
||||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
if (parentProjection) {
|
||||||
: null;
|
// Each owner quad is target-relative; do not apply the parent's transform twice.
|
||||||
|
const edge = parentProjection.edges[0];
|
||||||
|
const clips = edge
|
||||||
|
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
|
||||||
|
: parentProjection.sourceClips;
|
||||||
|
try {
|
||||||
|
projection = await geometry.snapshotProjection(
|
||||||
|
source,
|
||||||
|
iframeBackendId,
|
||||||
|
clips,
|
||||||
|
parentProjection.topViewport,
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
if (isAbortError(error)) throw error;
|
||||||
|
console.debug("[bsk capture] frame owner geometry unavailable", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const childContext: FrameContext = {
|
const childContext: FrameContext = {
|
||||||
frameId: snapshotFrameId(childDoc, strings),
|
frameId: source.frameId,
|
||||||
|
target: source.target,
|
||||||
ownerFrameBackendNodeId: iframeBackendId,
|
ownerFrameBackendNodeId: iframeBackendId,
|
||||||
projection,
|
projection,
|
||||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||||
@@ -1023,7 +1028,7 @@ function parseChildFrameDocuments(
|
|||||||
};
|
};
|
||||||
const nextVisited = new Set(visited);
|
const nextVisited = new Set(visited);
|
||||||
nextVisited.add(childDocIndex);
|
nextVisited.add(childDocIndex);
|
||||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
const parsed = parseDocumentNodes(childDoc, strings, childContext);
|
||||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||||
if (childContext.frameId) {
|
if (childContext.frameId) {
|
||||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||||
@@ -1031,13 +1036,12 @@ function parseChildFrameDocuments(
|
|||||||
}
|
}
|
||||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||||
|
|
||||||
const nested = parseChildFrameDocuments(
|
const nested = await parseChildFrameDocuments(
|
||||||
documents,
|
documents,
|
||||||
strings,
|
strings,
|
||||||
dpr,
|
|
||||||
childDocIndex,
|
childDocIndex,
|
||||||
parsed.nodes,
|
|
||||||
childContext,
|
childContext,
|
||||||
|
geometry,
|
||||||
nextVisited,
|
nextVisited,
|
||||||
);
|
);
|
||||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||||
@@ -1061,22 +1065,20 @@ export async function captureViewModel(
|
|||||||
options: CaptureViewModelOptions = {},
|
options: CaptureViewModelOptions = {},
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
let metrics: LayoutMetricsReply = {};
|
const target = options.target ?? { tabId };
|
||||||
|
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
|
||||||
|
let metrics: LayoutMetrics = {};
|
||||||
try {
|
try {
|
||||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
metrics = await geometry.layoutMetrics(target);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isAbortError(error)) throw error;
|
if (isAbortError(error)) throw error;
|
||||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||||
}
|
}
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const dpr = devicePixelRatio(metrics);
|
const measured = cssViewport(metrics);
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
const viewport: Viewport = { width: measured.width, height: measured.height };
|
||||||
const viewport: Viewport = {
|
const scrollX = measured.scrollX;
|
||||||
width: vpSrc.clientWidth ?? 0,
|
const scrollY = measured.scrollY;
|
||||||
height: vpSrc.clientHeight ?? 0,
|
|
||||||
};
|
|
||||||
const scrollX = vpSrc.pageX ?? 0;
|
|
||||||
const scrollY = vpSrc.pageY ?? 0;
|
|
||||||
|
|
||||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
@@ -1102,19 +1104,19 @@ export async function captureViewModel(
|
|||||||
const topContext: FrameContext = {
|
const topContext: FrameContext = {
|
||||||
frameId: snapshotFrameId(doc0, strings),
|
frameId: snapshotFrameId(doc0, strings),
|
||||||
ownerFrameBackendNodeId: null,
|
ownerFrameBackendNodeId: null,
|
||||||
|
target,
|
||||||
projection: {
|
projection: {
|
||||||
sourceClips: [],
|
source: { target, frameId: snapshotFrameId(doc0, strings) },
|
||||||
edges: [],
|
geometry: { sourceClips: [], edges: [], topViewport: viewport },
|
||||||
topViewport: viewport,
|
|
||||||
},
|
},
|
||||||
scrollX,
|
scrollX,
|
||||||
scrollY,
|
scrollY,
|
||||||
};
|
};
|
||||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
const mainParsed = parseDocumentNodes(doc0, strings, topContext);
|
||||||
const nodes = mainParsed.nodes;
|
const nodes = mainParsed.nodes;
|
||||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||||
|
|
||||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
const frameParsed = await parseChildFrameDocuments(documents, strings, 0, topContext, geometry);
|
||||||
const iframeNodes = frameParsed.iframeNodes;
|
const iframeNodes = frameParsed.iframeNodes;
|
||||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||||
excludedBackendNodeIds.add(id);
|
excludedBackendNodeIds.add(id);
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||||
import { resolveFrameProjection } from "../frame-geometry";
|
|
||||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
|
import { GeometryContext } from "../geometry/frame-context";
|
||||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||||
import {
|
import {
|
||||||
@@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean {
|
|||||||
return err instanceof Error && err.name === "AbortError";
|
return err instanceof Error && err.name === "AbortError";
|
||||||
}
|
}
|
||||||
|
|
||||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
|
||||||
if (!cdp.getFrameGraph) return null;
|
|
||||||
try {
|
|
||||||
return await cdp.getFrameGraph(tabId);
|
|
||||||
} catch (err) {
|
|
||||||
console.debug("[bsk observation] frame graph capture failed", err);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function transformFrameNodes(
|
function transformFrameNodes(
|
||||||
nodes: CapturedNode[],
|
nodes: CapturedNode[],
|
||||||
projection: GeometryProjection | null,
|
projection: GeometryProjection | null,
|
||||||
@@ -51,6 +41,7 @@ async function captureMissingFrameDocuments(
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
graph: CdpFrameGraph,
|
graph: CdpFrameGraph,
|
||||||
captured: CapturedViewModel,
|
captured: CapturedViewModel,
|
||||||
|
geometry: GeometryContext,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||||
@@ -62,10 +53,12 @@ async function captureMissingFrameDocuments(
|
|||||||
try {
|
try {
|
||||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||||
signal,
|
signal,
|
||||||
|
geometry,
|
||||||
|
target: frame.target,
|
||||||
});
|
});
|
||||||
let projection: GeometryProjection | null = null;
|
let projection: GeometryProjection | null = null;
|
||||||
try {
|
try {
|
||||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
projection = await geometry.targetProjection(frame.frameId);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (isAbortError(err)) throw err;
|
if (isAbortError(err)) throw err;
|
||||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||||
@@ -171,10 +164,11 @@ export async function captureFrameData<T extends FrameAxNode>(
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
captured: CapturedViewModel,
|
captured: CapturedViewModel,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
|
geometry = new GeometryContext(cdp, tabId, undefined, signal),
|
||||||
): Promise<CapturedFrameDocument<T>[]> {
|
): Promise<CapturedFrameDocument<T>[]> {
|
||||||
const graph = await discoverFrameGraph(cdp, tabId);
|
const graph = await geometry.graph();
|
||||||
const frames = graph?.frames ?? [];
|
const frames = graph?.frames ?? [];
|
||||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||||
return buildFrameDocuments(graph, batches, captured);
|
return buildFrameDocuments(graph, batches, captured);
|
||||||
|
|||||||
Reference in New Issue
Block a user