diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index 9116f6a..b535ea0 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; +import { resolveNodeGeometry } from "../frame-geometry"; import { clipPolygon, polygonArea, @@ -10,6 +10,7 @@ import { rectPolygon, regionBounds, } from "../geometry"; +import { GeometryContext } from "../geometry/frame-context"; import type { CdpRunner } from "../shared"; describe("frame geometry projection", () => { @@ -82,28 +83,24 @@ describe("frame geometry projection", () => { throw new Error(`unexpected child command ${method}`); }) as CdpRunner["sendToTarget"], }; - const projection = await resolveFrameProjection( - cdp, - { - rootFrameId: "main", - frames: [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "same-process-parent", - parentFrameId: "main", - ownerBackendNodeId: 10, - target: { tabId: 4 }, - }, - { - frameId: "oopif", - parentFrameId: "same-process-parent", - ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "oopif-session" }, - }, - ], - }, - "oopif", - ); + const projection = await new GeometryContext(cdp, 4, { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "same-process-parent", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4 }, + }, + { + frameId: "oopif", + parentFrameId: "same-process-parent", + ownerBackendNodeId: 20, + target: { tabId: 4, sessionId: "oopif-session" }, + }, + ], + }).targetProjection("oopif"); expect(projection?.edges).toEqual([ { diff --git a/apps/extension/src/tools/__tests__/geometry-context.test.ts b/apps/extension/src/tools/__tests__/geometry-context.test.ts new file mode 100644 index 0000000..5094495 --- /dev/null +++ b/apps/extension/src/tools/__tests__/geometry-context.test.ts @@ -0,0 +1,284 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { resolveNodeGeometry } from "../frame-geometry"; +import { rectPolygon } from "../geometry"; +import { + projectSnapshotRect, + screenshotPageRect, + snapshotViewportRect, +} from "../geometry/coordinate-types"; +import { GeometryContext } from "../geometry/frame-context"; +import type { CdpRunner } from "../shared"; + +const target = { tabId: 4 }; +const graph: CdpFrameGraph = { + rootFrameId: "main", + frames: [ + { frameId: "main", target }, + { frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target }, + ], +}; + +function driver() { + const send = vi.fn(async (_tab: number, method: string): Promise => { + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } }; + if (method === "DOM.getBoxModel") + return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } }; + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 300, height: 200 } } }; + if (method === "Runtime.releaseObject") return {}; + throw new Error(method); + }); + return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send }; +} + +describe("measurement geometry context", () => { + it("shares in-flight frame measurements, but never shares them with the next context", async () => { + const cdp = driver(); + const context = new GeometryContext(cdp, 4); + const results = await Promise.all( + Array.from({ length: 20 }, () => context.targetProjection("child")), + ); + expect(results.every((value) => value === results[0])).toBe(true); + expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([ + "Page.getLayoutMetrics", + "DOM.getBoxModel", + ]); + await new GeometryContext(cdp, 4).targetProjection("child"); + expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2); + expect(cdp.calls.mock.calls).toHaveLength(4); + }); + + it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => { + const cdp = driver(); + const context = new GeometryContext(cdp, 4); + const source = { target, frameId: "child" }; + const projection = await context.snapshotProjection(source, 10, [], { + width: 1000, + height: 1000, + }); + const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 }); + expect(projectSnapshotRect(input!, projection!)).toEqual({ + x: 73.75, + y: 641.25, + width: 150, + height: 50, + }); + const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 }); + expect(projectSnapshotRect(scrolled!, projection!)?.y).toBe(628.75); + await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 }); + expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength( + 1, + ); + expect( + cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"), + ).toHaveLength(1); + }); + + it("rejects owner mismatches and invalid numeric bounds", () => { + const input = snapshotViewportRect( + [30, 50, 120, 40], + { target, frameId: "main" }, + { x: 10, y: 20 }, + )!; + const projection = { + source: { target, frameId: "main" }, + geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } }, + }; + expect(projectSnapshotRect(input, projection)).toEqual({ + x: 20, + y: 30, + width: 120, + height: 40, + }); + expect( + projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }), + ).toBeNull(); + expect( + projectSnapshotRect(input, { + ...projection, + source: { target: { tabId: 5 }, frameId: "main" }, + }), + ).toBeNull(); + expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull(); + expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull(); + }); + + it("does not turn a border quad or a failed owner read into a content projection", async () => { + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + model: { border: [0, 0, 100, 0, 100, 100, 0, 100] }, + })) as CdpRunner["send"], + }; + expect( + await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], { + width: 100, + height: 100, + }), + ).toBeNull(); + }); + + it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => { + const cdp = driver(); + const projection = await new GeometryContext(cdp, 4).snapshotProjection( + { target, frameId: "nested" }, + 10, + [rectPolygon({ x: 60, y: 620, w: 100, h: 100 })], + { width: 1000, height: 1000 }, + ); + const input = snapshotViewportRect( + [0, 0, 300, 200], + { target, frameId: "nested" }, + { x: 0, y: 0 }, + ); + expect(projectSnapshotRect(input!, projection!)).toEqual({ + x: 60, + y: 620, + width: 100, + height: 100, + }); + }); + + it("rejects malformed frame ancestry without looping", async () => { + const cdp = driver(); + const cycle: CdpFrameGraph = { + rootFrameId: "a", + frames: [ + { frameId: "a", parentFrameId: "b", target }, + { frameId: "b", parentFrameId: "a", target }, + ], + }; + expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull(); + expect(cdp.calls).not.toHaveBeenCalled(); + const orphan: CdpFrameGraph = { + rootFrameId: "a", + frames: [{ frameId: "a", parentFrameId: "missing", target }], + }; + expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull(); + }); + + it("rejects a live node whose frame belongs to another target before any DOM input", async () => { + const cdp = driver(); + expect( + await resolveNodeGeometry( + cdp, + 4, + { target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 }, + { scrollIntoView: true }, + ), + ).toMatchObject({ code: "cdp_failed" }); + expect(cdp.calls).not.toHaveBeenCalled(); + }); + + it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => { + let active = 0; + let peak = 0; + const release: Array<() => void> = []; + const send = vi.fn(async () => { + active++; + peak = Math.max(peak, active); + await new Promise((resolve) => release.push(resolve)); + active--; + return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } }; + }); + const controller = new AbortController(); + const context = new GeometryContext( + { send: send as CdpRunner["send"] }, + 4, + undefined, + controller.signal, + ); + const tasks = Array.from({ length: 8 }, (_, index) => + context.viewport({ tabId: 4, sessionId: String(index) }), + ); + const settled = Promise.allSettled(tasks); + expect(send).toHaveBeenCalledTimes(4); + controller.abort(); + for (const done of release) done(); + const results = await settled; + expect(peak).toBe(4); + expect(send).toHaveBeenCalledTimes(4); + expect(results.filter((item) => item.status === "rejected")).toHaveLength(4); + }); + + it("releases an owner object even when cancellation arrives before its viewport read", async () => { + const controller = new AbortController(); + const cdp = driver(); + const original = cdp.calls.getMockImplementation()!; + cdp.calls.mockImplementation(async (tab, method) => { + const result = await original(tab, method); + if (method === "DOM.resolveNode") controller.abort(); + return result; + }); + const context = new GeometryContext(cdp, 4, undefined, controller.signal); + await expect( + context.snapshotProjection({ target, frameId: "child" }, 10, [], { + width: 1000, + height: 1000, + }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([ + "DOM.getBoxModel", + "DOM.resolveNode", + "Runtime.releaseObject", + ]); + }); + + it("reads live owner geometry only after scrolling", async () => { + const cdp = driver(); + const original = cdp.calls.getMockImplementation()!; + let scrolled = false; + cdp.calls.mockImplementation(async (tab, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") { + scrolled = true; + return {}; + } + if (method === "DOM.getContentQuads") { + expect(scrolled).toBe(true); + return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] }; + } + if (method === "DOM.getBoxModel") expect(scrolled).toBe(true); + return original(tab, method); + }); + expect( + await resolveNodeGeometry( + cdp, + 4, + { target, frameId: "child", backendNodeId: 20 }, + { scrollIntoView: true }, + ), + ).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } }); + expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength( + 1, + ); + }); + + it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => { + const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 }; + const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport); + expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } }); + expect( + screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }), + ).toBeNull(); + expect( + screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }), + ).toBeNull(); + }); + + it("retries a rejected measurement only in a new operation", async () => { + const cdp = driver(); + cdp.calls.mockRejectedValueOnce(new Error("unavailable")); + const context = new GeometryContext(cdp, 4); + await expect(context.viewport(target)).rejects.toThrow("unavailable"); + await expect(context.viewport(target)).rejects.toThrow("unavailable"); + expect(cdp.calls).toHaveBeenCalledTimes(1); + expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({ + width: 1000, + height: 1000, + }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 655e30c..798efe0 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -266,7 +266,15 @@ describe("handleScreenshot", () => { expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined(); }); - it("captures a clipped PNG when ref is given", async () => { + it.each([ + { zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } }, + { zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } }, + { zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } }, + ])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({ + zoom, + scroll, + expected, + }) => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e5", 999, { tabId: 7 }); @@ -274,6 +282,10 @@ describe("handleScreenshot", () => { "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "Page.captureScreenshot": () => ({ data: TINY_PNG }), + "Page.getLayoutMetrics": () => ({ + cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll }, + cssVisualViewport: { zoom }, + }), }); const captureVisibleTab = vi.fn(); const res = await handleScreenshot( @@ -297,7 +309,8 @@ describe("handleScreenshot", () => { clip?: { x: number; y: number; width: number; height: number }; } )?.clip; - expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); + expect(clip).toMatchObject(expected); + expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1); }); it("returns not_found for unknown ref", async () => { diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index 0c788c9..0fb997c 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -1,24 +1,21 @@ -import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; import type { RpcError } from "@/transport/types"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { clipPolygon, - type GeometryProjection, type Point, type Polygon, - type ProjectiveEdge, - parseCdpQuad, polygonArea, polygonCentroid, projectRegionToViewport, - type Quad, type Region, rectPolygon, regionBounds, - type Size, type ViewportRect, } from "./geometry"; -import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; +import type { CssViewport } from "./geometry/coordinate-types"; +import { cssViewport, GeometryContext } from "./geometry/frame-context"; +import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared"; export interface NodeAddress { target: CdpTarget; @@ -27,6 +24,8 @@ export interface NodeAddress { } export interface ResolvedNodeGeometry { + /** The same top-level measurement used by projection, for screenshot coordinate adaptation. */ + topViewport: CssViewport; topVisibleRegions: Region; topBounds: ViewportRect; /** Point in the top-level tab viewport, used by root-target input events. */ @@ -39,145 +38,16 @@ function geometryError(message: string): RpcError { return { code: "cdp_failed", message }; } -function frameMap(graph: CdpFrameGraph): Map { - return new Map(graph.frames.map((frame) => [frame.frameId, frame])); -} - -function targetRootFrame(byId: Map, frame: CdpFrame): CdpFrame | null { - const seen = new Set(); - let current = frame; - while (current.parentFrameId) { - if (seen.has(current.frameId)) return null; - seen.add(current.frameId); - const parent = byId.get(current.parentFrameId); - if (!parent || parent.target.sessionId !== current.target.sessionId) break; - current = parent; - } - return current; -} - -function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null { - const byId = frameMap(graph); - const path: CdpFrame[] = []; - const seen = new Set(); - let current = byId.get(frameId); - if (!current) return null; - while (current.parentFrameId) { - if (seen.has(current.frameId)) return null; - seen.add(current.frameId); - path.push(current); - const parent = byId.get(current.parentFrameId); - if (!parent) return null; - current = parent; - } - return path; -} - -async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise { - const metrics = await sendToCdpTarget<{ - cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; - layoutViewport?: { clientWidth?: number; clientHeight?: number }; - }>(cdp, target, "Page.getLayoutMetrics", {}); - const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; - const width = viewport.clientWidth ?? 0; - const height = viewport.clientHeight ?? 0; - return width > 0 && height > 0 ? { width, height } : null; -} - -async function ownerContentQuad( - cdp: CdpRunner, - parent: CdpFrame, - ownerBackendNodeId: number, -): Promise { - const result = await sendToCdpTarget<{ model?: { content?: number[] } }>( - cdp, - parent.target, - "DOM.getBoxModel", - { backendNodeId: ownerBackendNodeId }, - ); - return parseCdpQuad(result.model?.content); -} - -async function sameTargetFrameClips( - cdp: CdpRunner, - byId: Map, - frame: CdpFrame, - targetRoot: CdpFrame, -): Promise { - const clips: Polygon[] = []; - let current = frame; - const seen = new Set(); - while (current.frameId !== targetRoot.frameId) { - if (seen.has(current.frameId) || !current.parentFrameId) return null; - seen.add(current.frameId); - const parent = byId.get(current.parentFrameId); - if (!parent || current.ownerBackendNodeId === undefined) return null; - const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId); - if (!clip) return null; - clips.push(clip); - current = parent; - } - return clips; -} - -export async function resolveFrameProjection( - cdp: CdpRunner, - graph: CdpFrameGraph, - frameId: string, -): Promise { - const byId = frameMap(graph); - const frame = byId.get(frameId); - if (!frame) return null; - let targetRoot = targetRootFrame(byId, frame); - if (!targetRoot) return null; - const sourceViewport = await targetViewport(cdp, targetRoot.target); - if (!sourceViewport) return null; - const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot); - if (!sourceClips) return null; - - const edges: ProjectiveEdge[] = []; - while (targetRoot.parentFrameId) { - const parent = byId.get(targetRoot.parentFrameId); - if (!parent || targetRoot.ownerBackendNodeId === undefined) return null; - const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId); - if (!destinationQuad) return null; - const source = - edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - if (!source) return null; - const parentTargetRoot = targetRootFrame(byId, parent); - if (!parentTargetRoot) return null; - const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot); - if (!destinationClips) return null; - edges.push({ sourceViewport: source, destinationQuad, destinationClips }); - targetRoot = parentTargetRoot; - } - - const topViewport = - edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - return topViewport ? { sourceClips, edges, topViewport } : null; -} - -async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise { - if (!cdp.getFrameGraph) return null; - try { - return await cdp.getFrameGraph(tabId); - } catch (error) { - console.debug("[bsk frame-geometry] frame graph resolution failed", error); - return null; - } -} - async function scrollFrameOwners( cdp: CdpRunner, tabId: number, - graph: CdpFrameGraph, + context: GeometryContext, frameId: string, ): Promise { - const byId = frameMap(graph); - const ancestry = frameAncestry(graph, frameId); + const ancestry = await context.ancestry(frameId); if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); for (const child of [...ancestry].reverse()) { - const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; + const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined; if (!parent || child.ownerBackendNodeId === undefined) { return geometryError(`could not resolve frame owner for ${child.frameId}`); } @@ -191,17 +61,20 @@ async function scrollFrameOwners( return null; } -async function scrollElementWithFrameGraph( +async function scrollElementWithContext( cdp: CdpRunner, tabId: number, target: CdpTarget, backendNodeId: number, frameId: string | undefined, - graph: CdpFrameGraph | null, + context: GeometryContext, ): Promise { if (frameId) { - if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); - const error = await scrollFrameOwners(cdp, tabId, graph, frameId); + const frame = await context.frame(frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) { + return geometryError("node frame does not belong to its target"); + } + const error = await scrollFrameOwners(cdp, tabId, context, frameId); if (error) return error; } else if (target.sessionId) { return geometryError("an OOPIF node address requires frameId"); @@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView( backendNodeId: number, frameId?: string, ): Promise { - const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; - return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); + if (target.tabId !== tabId) return geometryError("node target belongs to another tab"); + return scrollElementWithContext( + cdp, + tabId, + target, + backendNodeId, + frameId, + new GeometryContext(cdp, tabId), + ); } function largestRegion(regions: Region): Polygon | null { @@ -240,23 +120,32 @@ export async function resolveNodeGeometry( if (address.target.sessionId && !address.frameId) { return geometryError("an OOPIF node address requires frameId"); } - const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; + if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab"); + const context = new GeometryContext(cdp, tabId); + const graph = address.frameId ? await context.graph() : null; + if (address.frameId && graph) { + const frame = await context.frame(address.frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) { + return geometryError("node frame does not belong to its target"); + } + } if (address.frameId && !graph) { return geometryError(`could not resolve frame graph for ${address.frameId}`); } if (options.scrollIntoView) { - const scrollError = await scrollElementWithFrameGraph( + const scrollError = await scrollElementWithContext( cdp, tabId, address.target, address.backendNodeId, address.frameId, - graph, + context, ); if (scrollError) return scrollError; } + // Only topology was read above. Begin live measurements after scrolling. const localRegion = await nodeContentRegion( cdpRunnerForTarget(cdp, address.target), tabId, @@ -267,7 +156,7 @@ export async function resolveNodeGeometry( let topVisibleRegions: Region; let targetActionPoint: Point | null = null; if (address.frameId && graph) { - const projection = await resolveFrameProjection(cdp, graph, address.frameId); + const projection = await context.targetProjection(address.frameId); if (!projection) return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); @@ -290,7 +179,7 @@ export async function resolveNodeGeometry( targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; } } else { - const viewport = await targetViewport(cdp, address.target); + const viewport = await context.viewport(address.target); if (!viewport) return geometryError("could not resolve top viewport geometry"); topVisibleRegions = localRegion .map((polygon) => @@ -309,7 +198,13 @@ export async function resolveNodeGeometry( return { code: "permission_denied", message: "element not visible in its target" }; } targetActionPoint ??= actionPoint; - return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; + return { + topVisibleRegions, + topBounds, + actionPoint, + targetActionPoint, + topViewport: cssViewport(await context.layoutMetrics({ tabId })), + }; } catch (error) { return geometryError(error instanceof Error ? error.message : String(error)); } diff --git a/apps/extension/src/tools/geometry.ts b/apps/extension/src/tools/geometry.ts index fea8782..247ca02 100644 --- a/apps/extension/src/tools/geometry.ts +++ b/apps/extension/src/tools/geometry.ts @@ -245,22 +245,3 @@ export function projectRectToViewport( if (!rect) return null; return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); } - -export function childFrameProjection( - parent: GeometryProjection, - ownerRectInParent: { x: number; y: number; w: number; h: number }, -): GeometryProjection { - const destinationQuad = rectPolygon(ownerRectInParent); - return { - sourceClips: [], - edges: [ - { - sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h }, - destinationQuad, - destinationClips: parent.sourceClips, - }, - ...parent.edges, - ], - topViewport: parent.topViewport, - }; -} diff --git a/apps/extension/src/tools/geometry/coordinate-types.ts b/apps/extension/src/tools/geometry/coordinate-types.ts new file mode 100644 index 0000000..09a2cec --- /dev/null +++ b/apps/extension/src/tools/geometry/coordinate-types.ts @@ -0,0 +1,84 @@ +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import type { GeometryProjection, ViewportRect } from "../geometry"; +import { projectRectToViewport } from "../geometry"; + +export interface CoordinateOwner { + target: CdpTarget; + frameId?: string; +} + +export interface FrameViewportRect { + space: "frame-viewport-css"; + owner: CoordinateOwner; + rect: ViewportRect; +} + +export interface SnapshotProjection { + source: CoordinateOwner; + geometry: GeometryProjection; +} + +/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */ +export function snapshotViewportRect( + bounds: number[], + owner: CoordinateOwner, + scroll: { x: number; y: number }, +): FrameViewportRect | null { + if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null; + const [x, y, width, height] = bounds; + if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) { + return null; + } + return { + space: "frame-viewport-css", + owner, + rect: { x: x - scroll.x, y: y - scroll.y, width, height }, + }; +} + +export function projectSnapshotRect( + input: FrameViewportRect, + projection: SnapshotProjection, +): ViewportRect | null { + if ( + input.owner.frameId !== projection.source.frameId || + cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target) + ) { + return null; + } + const { x, y, width: w, height: h } = input.rect; + return projectRectToViewport({ x, y, w, h }, projection.geometry); +} + +/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */ +export interface CssViewport { + width: number; + height: number; + scrollX: number; + scrollY: number; + cssToDip: number; +} + +/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */ +export function screenshotPageRect( + rect: ViewportRect, + viewport: CssViewport, +): { space: "page-dip"; rect: ViewportRect } | null { + const { cssToDip, scrollX, scrollY } = viewport; + if ( + ![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) || + cssToDip <= 0 || + rect.width <= 0 || + rect.height <= 0 + ) + return null; + return { + space: "page-dip", + rect: { + x: (rect.x + scrollX) * cssToDip, + y: (rect.y + scrollY) * cssToDip, + width: rect.width * cssToDip, + height: rect.height * cssToDip, + }, + }; +} diff --git a/apps/extension/src/tools/geometry/frame-context.ts b/apps/extension/src/tools/geometry/frame-context.ts new file mode 100644 index 0000000..3a81699 --- /dev/null +++ b/apps/extension/src/tools/geometry/frame-context.ts @@ -0,0 +1,297 @@ +import { + type CdpFrame, + type CdpFrameGraph, + type CdpTarget, + cdpTargetKey, +} from "@/browser-driver/frame-graph"; +import { + type GeometryProjection, + type Polygon, + type ProjectiveEdge, + parseCdpQuad, + type Quad, + type Size, +} from "../geometry"; +import { type CdpRunner, sendToCdpTarget } from "../shared"; +import type { CoordinateOwner, CssViewport, SnapshotProjection } from "./coordinate-types"; + +export interface LayoutMetrics { + cssVisualViewport?: { zoom?: number }; + visualViewport?: { zoom?: number }; + cssLayoutViewport?: { + clientWidth?: number; + clientHeight?: number; + pageX?: number; + pageY?: number; + }; + layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number }; +} + +/** The legacy field is a compatibility fallback, never a source of raster DPR. */ +export function cssViewport(metrics: LayoutMetrics): CssViewport { + const value = metrics.cssLayoutViewport ?? metrics.layoutViewport; + return { + width: value?.clientWidth ?? 0, + height: value?.clientHeight ?? 0, + scrollX: value?.pageX ?? 0, + scrollY: value?.pageY ?? 0, + cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1, + }; +} + +/** One read-only measurement phase. Discard after scrolling or any later operation. */ +export class GeometryContext { + private readonly metrics = new Map>(); + private readonly owners = new Map>(); + private readonly projections = new Map>(); + private readonly snapshotOwners = new Map>(); + private active = 0; + private readonly waiting: Array<() => void> = []; + private graphPromise?: Promise; + private framesPromise?: Promise>; + + constructor( + private readonly cdp: CdpRunner, + private readonly tabId: number, + graph?: CdpFrameGraph, + private readonly signal?: AbortSignal, + ) { + if (graph) this.graphPromise = Promise.resolve(graph); + } + + private async request( + target: CdpTarget, + method: string, + params: object, + cleanup = false, + ): Promise { + if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError"); + if (this.active >= 4) await new Promise((resolve) => this.waiting.push(resolve)); + else this.active++; + try { + if (!cleanup && this.signal?.aborted) + throw new DOMException("geometry aborted", "AbortError"); + return await sendToCdpTarget(this.cdp, target, method, params); + } finally { + const next = this.waiting.shift(); + if (next) next(); + else this.active--; + } + } + + graph(): Promise { + this.graphPromise ??= + this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null); + return this.graphPromise; + } + + private frames(): Promise> { + this.framesPromise ??= this.graph().then( + (graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []), + ); + return this.framesPromise; + } + + async frame(frameId: string): Promise { + return (await this.frames()).get(frameId); + } + + /** Topology only: safe to resolve before the caller scrolls frame owners. */ + async ancestry(frameId: string): Promise { + const frames = await this.frames(); + const path: CdpFrame[] = []; + const seen = new Set(); + let current = frames.get(frameId); + if (!current) return null; + while (current.parentFrameId) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + path.push(current); + const parent = frames.get(current.parentFrameId); + if (!parent) return null; + current = parent; + } + return path; + } + + layoutMetrics(target: CdpTarget): Promise { + const key = cdpTargetKey(target); + let promise = this.metrics.get(key); + if (!promise) { + promise = this.request(target, "Page.getLayoutMetrics", {}); + this.metrics.set(key, promise); + } + return promise; + } + + async viewport(target: CdpTarget): Promise { + const { width, height } = cssViewport(await this.layoutMetrics(target)); + return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0 + ? { width, height } + : null; + } + + ownerContent(target: CdpTarget, backendNodeId: number): Promise { + const key = `${cdpTargetKey(target)}:${backendNodeId}`; + let promise = this.owners.get(key); + if (!promise) { + promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", { + backendNodeId, + }).then((reply) => parseCdpQuad(reply.model?.content)); + this.owners.set(key, promise); + } + return promise; + } + + /** Same-target owner quads are already target-relative, including nested owners. */ + async snapshotProjection( + source: CoordinateOwner, + ownerBackendNodeId: number, + ancestorClips: Polygon[], + viewport: Size, + ): Promise { + const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`; + let promise = this.snapshotOwners.get(key); + if (!promise) { + promise = this.snapshotOwner(source.target, ownerBackendNodeId); + this.snapshotOwners.set(key, promise); + } + const owner = await promise; + return owner + ? { + source, + geometry: { + sourceClips: [], + edges: [ + { + sourceViewport: owner.size, + destinationQuad: owner.quad, + destinationClips: ancestorClips, + }, + ], + topViewport: viewport, + }, + } + : null; + } + + private async snapshotOwner( + target: CdpTarget, + backendNodeId: number, + ): Promise<{ quad: Quad; size: Size } | null> { + const quad = await this.ownerContent(target, backendNodeId); + if (!quad) return null; + const resolved = await this.request<{ object?: { objectId?: string } }>( + target, + "DOM.resolveNode", + { backendNodeId }, + ); + const objectId = resolved.object?.objectId; + if (!objectId) return null; + try { + const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>( + target, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: `function() { + if (!this.isConnected) return null; + const style = getComputedStyle(this); + return { + width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight), + height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom) + }; + }`, + returnByValue: true, + }, + ); + const size = reply.result?.value; + return !reply.exceptionDetails && + size && + Number.isFinite(size.width) && + Number.isFinite(size.height) && + size.width > 0 && + size.height > 0 + ? { quad, size } + : null; + } finally { + // Cleanup must still run if cancellation arrived after resolving the object. + await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {}); + } + } + + private targetRoot(frames: Map, frame: CdpFrame): CdpFrame | null { + const seen = new Set(); + let current = frame; + while (current.parentFrameId) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + const parent = frames.get(current.parentFrameId); + if (!parent) return null; + if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break; + current = parent; + } + return current; + } + + private async clips( + frames: Map, + frame: CdpFrame, + root: CdpFrame, + ): Promise { + const clips: Polygon[] = []; + let current = frame; + const seen = new Set(); + while (current.frameId !== root.frameId) { + if (seen.has(current.frameId) || !current.parentFrameId) return null; + seen.add(current.frameId); + const parent = frames.get(current.parentFrameId); + if (!parent || current.ownerBackendNodeId === undefined) return null; + const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId); + if (!quad) return null; + clips.push(quad); + current = parent; + } + return clips; + } + + /** Input is already in the frame's CDP target viewport, not its local viewport. */ + targetProjection(frameId: string): Promise { + let promise = this.projections.get(frameId); + if (!promise) { + promise = this.buildTargetProjection(frameId); + this.projections.set(frameId, promise); + } + return promise; + } + + private async buildTargetProjection(frameId: string): Promise { + const frames = await this.frames(); + const frame = frames.get(frameId); + if (!frame) return null; + let root = this.targetRoot(frames, frame); + if (!root) return null; + const sourceViewport = await this.viewport(root.target); + if (!sourceViewport) return null; + const sourceClips = await this.clips(frames, frame, root); + if (!sourceClips) return null; + const edges: ProjectiveEdge[] = []; + const seen = new Set(); + while (root.parentFrameId) { + if (seen.has(root.frameId)) return null; + seen.add(root.frameId); + const parent = frames.get(root.parentFrameId); + if (!parent || root.ownerBackendNodeId === undefined) return null; + const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId); + const source = await this.viewport(root.target); + const parentRoot = this.targetRoot(frames, parent); + if (!destinationQuad || !source || !parentRoot) return null; + const destinationClips = await this.clips(frames, parent, parentRoot); + if (!destinationClips) return null; + edges.push({ sourceViewport: source, destinationQuad, destinationClips }); + root = parentRoot; + } + const topViewport = await this.viewport(root.target); + return topViewport ? { sourceClips, edges, topViewport } : null; + } +} diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 9c944ab..e8726a0 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -34,6 +34,8 @@ import type { import { attachDialogs, markDialogCursor } from "./dialogs"; import { rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; +import { screenshotPageRect } from "./geometry/coordinate-types"; +import { cssViewport, GeometryContext } from "./geometry/frame-context"; import { type ChromeTabsApi, enforceToolTargetScope, @@ -197,15 +199,14 @@ async function captureElementScreenshot( if (isRpcError(geometry)) return geometry; if (signal?.aborted) return cancelled("screenshot"); const rect = geometry.topBounds; + const clip = screenshotPageRect(rect, geometry.topViewport); + if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" }; try { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { format: "png", clip: { - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height, + ...clip.rect, scale: 1, }, }); @@ -984,26 +985,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; } -interface LayoutMetricsViewportReply { - cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; - layoutViewport?: { clientWidth?: number; clientHeight?: number }; -} - async function fallbackCapturedViewModel( cdp: CdpRunner, tabId: number, + geometry: GeometryContext, signal?: AbortSignal, ): Promise { throwIfAborted(signal, "observation"); let viewport = { width: 0, height: 0 }; try { - const metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); + const metrics = await geometry.layoutMetrics({ tabId }); throwIfAborted(signal, "observation"); - const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; - viewport = { - width: source.clientWidth ?? 0, - height: source.clientHeight ?? 0, - }; + const source = cssViewport(metrics); + viewport = { width: source.width, height: source.height }; } catch (error) { if (isAbortError(error)) throw error; } @@ -1016,12 +1010,13 @@ async function captureForVom( cdp: CdpRunner, tabId: number, options: CaptureVomObservationOptions, + geometry: GeometryContext, ): Promise { try { - return await captureViewModel(cdp, tabId, { signal: options.signal }); + return await captureViewModel(cdp, tabId, { signal: options.signal, geometry }); } catch (error) { if (isAbortError(error)) throw error; - return fallbackCapturedViewModel(cdp, tabId, options.signal); + return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal); } } @@ -1096,9 +1091,16 @@ export async function captureVomObservation( throwIfAborted(options.signal, "observation"); await cdp.ensureAttachedToUrl?.(tabId, url); throwIfAborted(options.signal, "observation"); - const captured = await captureForVom(cdp, tabId, options); + const geometry = new GeometryContext(cdp, tabId, undefined, options.signal); + const captured = await captureForVom(cdp, tabId, options, geometry); throwIfAborted(options.signal, "observation"); - const documents = await captureFrameData(cdp, tabId, captured, options.signal); + const documents = await captureFrameData( + cdp, + tabId, + captured, + options.signal, + geometry, + ); throwIfAborted(options.signal, "observation"); const normalizedDocuments = documents.length === 1 && captured.iframeNodes.size > 0 diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 97b4658..3c6c1b4 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -754,7 +754,7 @@ describe("captureViewModel", () => { expect(div?.rect).toMatchObject({ y: 0, h: 600 }); }); - it("normalizes device-pixel bounds by devicePixelRatio", async () => { + it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => { const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { @@ -802,6 +802,12 @@ describe("captureViewModel", () => { w: 1000, h: 800, }); + expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({ + x: 0, + y: 0, + w: 2000, + h: 1600, + }); }); it("extracts textContent from #text child nodes via nodeValue", async () => { @@ -950,6 +956,12 @@ describe("captureViewModel", () => { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } + if (method === "DOM.getBoxModel") + return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } }; + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 800, height: 400 } } }; + if (method === "Runtime.releaseObject") return {}; throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; @@ -979,7 +991,7 @@ describe("captureViewModel", () => { expect(frameParentIds?.get("child-frame")).toBe("main-frame"); }); - it("captures iframe documents without requiring owner-frame geometry", async () => { + it("projects iframe documents independently of owner snapshot geometry", async () => { const S = [ "html", "body", @@ -1040,6 +1052,12 @@ describe("captureViewModel", () => { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } + if (method === "DOM.getBoxModel") + return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } }; + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 800, height: 400 } } }; + if (method === "Runtime.releaseObject") return {}; throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; @@ -1048,7 +1066,7 @@ describe("captureViewModel", () => { const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); - expect(input?.rect).toBeNull(); + expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 }); }); it("recursively normalizes nested iframe sub-documents", async () => { @@ -1127,16 +1145,43 @@ describe("captureViewModel", () => { ], }; const cdp = { - send: vi.fn(async (_t: number, method: string) => { - if (method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return snapshot; - if (method === "Page.getLayoutMetrics") { - return { - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, - }; - } - throw new Error(method); - }) as unknown as (tabId: number, method: string, params?: object) => Promise, + send: vi.fn( + async ( + _t: number, + method: string, + params?: { backendNodeId?: number; objectId?: string }, + ) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "DOM.getBoxModel") + return { + model: { + content: + params?.backendNodeId === 13 + ? [10, 20, 310, 20, 310, 220, 10, 220] + : [15, 26, 115, 26, 115, 106, 15, 106], + }, + }; + if (method === "DOM.resolveNode") + return { object: { objectId: String(params?.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") + return { + result: { + value: + params?.objectId === "13" + ? { width: 300, height: 200 } + : { width: 100, height: 80 }, + }, + }; + if (method === "Runtime.releaseObject") return {}; + throw new Error(method); + }, + ) as unknown as (tabId: number, method: string, params?: object) => Promise, }; const { iframeNodes } = await captureViewModel(cdp, 4); @@ -1150,7 +1195,7 @@ describe("captureViewModel", () => { expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); }); - it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { + it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => { const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { @@ -1192,7 +1237,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; const { nodes } = await captureViewModel(cdp, 4); - expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); + expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100); }); it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => { diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index b0f794f..5271f7a 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -5,9 +5,15 @@ // STYLE_COL so capture and visibility policy share one explicit contract. import type { Rect, Viewport } from "@browser-skill/vom"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; -import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; +import { + projectSnapshotRect, + type SnapshotProjection, + snapshotViewportRect, +} from "../geometry/coordinate-types"; +import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context"; import type { CdpRunner } from "../shared"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; @@ -77,6 +83,8 @@ export interface CapturedViewModel { export interface CaptureViewModelOptions { signal?: AbortSignal; + geometry?: GeometryContext; + target?: CdpTarget; } function captureAbortError(): Error { @@ -147,16 +155,6 @@ interface SnapshotReply { documents?: SnapshotDocument[]; } -interface LayoutMetricsReply { - cssLayoutViewport?: { - clientWidth?: number; - clientHeight?: number; - pageX?: number; - pageY?: number; - }; - layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number }; -} - function isFormControlTag(tag: string): boolean { return tag === "input" || tag === "textarea" || tag === "select"; } @@ -353,7 +351,8 @@ interface ParseDocumentResult { interface FrameContext { frameId?: string; ownerFrameBackendNodeId: number | null; - projection: GeometryProjection | null; + projection: SnapshotProjection | null; + target: CdpTarget; scrollX: number; scrollY: number; } @@ -363,15 +362,6 @@ function str(strings: string[], idx: number | undefined): string { return strings[idx] ?? ""; } -function devicePixelRatio(metrics: LayoutMetricsReply): number { - const layoutW = metrics.layoutViewport?.clientWidth ?? 0; - const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0; - if (!layoutW || !cssW) return 1; - const dpr = layoutW / cssW; - if (!Number.isFinite(dpr) || dpr <= 0) return 1; - return dpr >= 1 ? dpr : 1; -} - function sparseIndexMap(sparse: SparseArray | undefined): Map { const out = new Map(); if (!sparse?.index || !sparse.value) return out; @@ -797,13 +787,11 @@ export async function collectOverlayExcludedBackendIds( * * @param doc - the raw DOMSnapshot document object * @param strings - the shared string table for the whole snapshot - * @param dpr - device-pixel-ratio from Page.getLayoutMetrics * @param context - frame coordinate context; output rects are top viewport-relative */ function parseDocumentNodes( doc: SnapshotDocument, strings: string[], - dpr: number, context: FrameContext, ): ParseDocumentResult { const dn = doc.nodes; @@ -898,15 +886,17 @@ function parseDocumentNodes( if (li !== undefined) { const b = dl?.bounds?.[li]; if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) { - localRect = { - x: b[0] / dpr - context.scrollX, - y: b[1] / dpr - context.scrollY, - w: b[2] / dpr, - h: b[3] / dpr, - }; - const bounds = context.projection - ? projectRectToViewport(localRect, context.projection) - : null; + const input = snapshotViewportRect( + b, + { target: context.target, frameId: context.frameId }, + { x: context.scrollX, y: context.scrollY }, + ); + if (input) { + const { x, y, width: w, height: h } = input.rect; + localRect = { x, y, w, h }; + } + const bounds = + input && context.projection ? projectSnapshotRect(input, context.projection) : null; rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; } paintOrder = dl?.paintOrders?.[li] ?? 0; @@ -980,15 +970,14 @@ interface ParseFrameDocumentsResult { excludedBackendNodeIds: Set; } -function parseChildFrameDocuments( +async function parseChildFrameDocuments( documents: SnapshotDocument[], strings: string[], - dpr: number, parentDocIndex: number, - parentNodes: CapturedNode[], parentContext: FrameContext, + geometry: GeometryContext, visited = new Set(), -): ParseFrameDocumentsResult { +): Promise { const iframeNodes: CapturedIframeNodes = new Map(); const frameOwnerBackendNodeIds = new Map(); const frameParentIds = new Map(); @@ -1000,7 +989,6 @@ function parseChildFrameDocuments( } const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; - const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node])); for (const [nodeArrayIdx, childDocIndex] of cdi) { if (visited.has(childDocIndex)) continue; @@ -1008,14 +996,31 @@ function parseChildFrameDocuments( if (!childDoc) continue; const iframeBackendId = parentBackendIds[nodeArrayIdx]; if (iframeBackendId === undefined) continue; - const iframeNode = parentNodeByBackendId.get(iframeBackendId); - const projection = - parentContext.projection && iframeNode?.localRect - ? childFrameProjection(parentContext.projection, iframeNode.localRect) - : null; + let projection: SnapshotProjection | null = null; + const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) }; + const parentProjection = parentContext.projection?.geometry; + if (parentProjection) { + // Each owner quad is target-relative; do not apply the parent's transform twice. + const edge = parentProjection.edges[0]; + const clips = edge + ? [edge.destinationQuad, ...(edge.destinationClips ?? [])] + : parentProjection.sourceClips; + try { + projection = await geometry.snapshotProjection( + source, + iframeBackendId, + clips, + parentProjection.topViewport, + ); + } catch (error) { + if (isAbortError(error)) throw error; + console.debug("[bsk capture] frame owner geometry unavailable", error); + } + } const childContext: FrameContext = { - frameId: snapshotFrameId(childDoc, strings), + frameId: source.frameId, + target: source.target, ownerFrameBackendNodeId: iframeBackendId, projection, scrollX: childDoc.scrollOffsetX ?? 0, @@ -1023,7 +1028,7 @@ function parseChildFrameDocuments( }; const nextVisited = new Set(visited); nextVisited.add(childDocIndex); - const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext); + const parsed = parseDocumentNodes(childDoc, strings, childContext); iframeNodes.set(iframeBackendId, parsed.nodes); if (childContext.frameId) { frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId); @@ -1031,13 +1036,12 @@ function parseChildFrameDocuments( } for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); - const nested = parseChildFrameDocuments( + const nested = await parseChildFrameDocuments( documents, strings, - dpr, childDocIndex, - parsed.nodes, childContext, + geometry, nextVisited, ); for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) { @@ -1061,22 +1065,20 @@ export async function captureViewModel( options: CaptureViewModelOptions = {}, ): Promise { throwIfAborted(options.signal); - let metrics: LayoutMetricsReply = {}; + const target = options.target ?? { tabId }; + const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal); + let metrics: LayoutMetrics = {}; try { - metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); + metrics = await geometry.layoutMetrics(target); } catch (error) { if (isAbortError(error)) throw error; console.debug("[bsk capture] layout metrics unavailable", error); } throwIfAborted(options.signal); - const dpr = devicePixelRatio(metrics); - const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; - const viewport: Viewport = { - width: vpSrc.clientWidth ?? 0, - height: vpSrc.clientHeight ?? 0, - }; - const scrollX = vpSrc.pageX ?? 0; - const scrollY = vpSrc.pageY ?? 0; + const measured = cssViewport(metrics); + const viewport: Viewport = { width: measured.width, height: measured.height }; + const scrollX = measured.scrollX; + const scrollY = measured.scrollY; await cdp.send(tabId, "DOMSnapshot.enable", {}); throwIfAborted(options.signal); @@ -1102,19 +1104,19 @@ export async function captureViewModel( const topContext: FrameContext = { frameId: snapshotFrameId(doc0, strings), ownerFrameBackendNodeId: null, + target, projection: { - sourceClips: [], - edges: [], - topViewport: viewport, + source: { target, frameId: snapshotFrameId(doc0, strings) }, + geometry: { sourceClips: [], edges: [], topViewport: viewport }, }, scrollX, scrollY, }; - const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext); + const mainParsed = parseDocumentNodes(doc0, strings, topContext); const nodes = mainParsed.nodes; const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds); - const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext); + const frameParsed = await parseChildFrameDocuments(documents, strings, 0, topContext, geometry); const iframeNodes = frameParsed.iframeNodes; for (const id of frameParsed.excludedBackendNodeIds) { excludedBackendNodeIds.add(id); diff --git a/apps/extension/src/tools/vom/frame-capture.ts b/apps/extension/src/tools/vom/frame-capture.ts index d643a7b..45beeb7 100644 --- a/apps/extension/src/tools/vom/frame-capture.ts +++ b/apps/extension/src/tools/vom/frame-capture.ts @@ -1,6 +1,6 @@ import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; -import { resolveFrameProjection } from "../frame-geometry"; import { type GeometryProjection, projectRectToViewport } from "../geometry"; +import { GeometryContext } from "../geometry/frame-context"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { @@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean { return err instanceof Error && err.name === "AbortError"; } -async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise { - if (!cdp.getFrameGraph) return null; - try { - return await cdp.getFrameGraph(tabId); - } catch (err) { - console.debug("[bsk observation] frame graph capture failed", err); - return null; - } -} - function transformFrameNodes( nodes: CapturedNode[], projection: GeometryProjection | null, @@ -51,6 +41,7 @@ async function captureMissingFrameDocuments( tabId: number, graph: CdpFrameGraph, captured: CapturedViewModel, + geometry: GeometryContext, signal?: AbortSignal, ): Promise { if (!captured.frameNodes) captured.frameNodes = new Map(); @@ -62,10 +53,12 @@ async function captureMissingFrameDocuments( try { const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { signal, + geometry, + target: frame.target, }); let projection: GeometryProjection | null = null; try { - projection = await resolveFrameProjection(cdp, graph, frame.frameId); + projection = await geometry.targetProjection(frame.frameId); } catch (err) { if (isAbortError(err)) throw err; console.debug("[bsk observation] child frame geometry projection failed", { @@ -171,10 +164,11 @@ export async function captureFrameData( tabId: number, captured: CapturedViewModel, signal?: AbortSignal, + geometry = new GeometryContext(cdp, tabId, undefined, signal), ): Promise[]> { - const graph = await discoverFrameGraph(cdp, tabId); + const graph = await geometry.graph(); const frames = graph?.frames ?? []; - if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal); + if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal); throwIfAborted(signal); const batches = await captureAxTrees(cdp, tabId, frames, captured, signal); return buildFrameDocuments(graph, batches, captured);