diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index 9116f6a..b535ea0 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; +import { resolveNodeGeometry } from "../frame-geometry"; import { clipPolygon, polygonArea, @@ -10,6 +10,7 @@ import { rectPolygon, regionBounds, } from "../geometry"; +import { GeometryContext } from "../geometry/frame-context"; import type { CdpRunner } from "../shared"; describe("frame geometry projection", () => { @@ -82,28 +83,24 @@ describe("frame geometry projection", () => { throw new Error(`unexpected child command ${method}`); }) as CdpRunner["sendToTarget"], }; - const projection = await resolveFrameProjection( - cdp, - { - rootFrameId: "main", - frames: [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "same-process-parent", - parentFrameId: "main", - ownerBackendNodeId: 10, - target: { tabId: 4 }, - }, - { - frameId: "oopif", - parentFrameId: "same-process-parent", - ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "oopif-session" }, - }, - ], - }, - "oopif", - ); + const projection = await new GeometryContext(cdp, 4, { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "same-process-parent", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4 }, + }, + { + frameId: "oopif", + parentFrameId: "same-process-parent", + ownerBackendNodeId: 20, + target: { tabId: 4, sessionId: "oopif-session" }, + }, + ], + }).targetProjection("oopif"); expect(projection?.edges).toEqual([ { diff --git a/apps/extension/src/tools/__tests__/geometry-context.test.ts b/apps/extension/src/tools/__tests__/geometry-context.test.ts new file mode 100644 index 0000000..34bac76 --- /dev/null +++ b/apps/extension/src/tools/__tests__/geometry-context.test.ts @@ -0,0 +1,464 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { resolveNodeGeometry } from "../frame-geometry"; +import { rectPolygon } from "../geometry"; +import { + projectSnapshotRect, + screenshotPageRect, + snapshotCoordinates, + snapshotViewportRect, +} from "../geometry/coordinate-types"; +import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context"; +import type { CdpRunner } from "../shared"; + +const target = { tabId: 4 }; +const graph: CdpFrameGraph = { + rootFrameId: "main", + frames: [ + { frameId: "main", target }, + { frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target }, + ], +}; + +function driver() { + const send = vi.fn(async (_tab: number, method: string): Promise => { + if (method === "Page.getLayoutMetrics") + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 }, + }; + if (method === "DOM.getBoxModel") + return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } }; + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 300, height: 200 } } }; + if (method === "Runtime.releaseObject") return {}; + throw new Error(method); + }); + return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send }; +} + +describe("measurement geometry context", () => { + it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => { + const measured = snapshotLayoutScale({ + visualViewport: { clientWidth: 997.25, zoom: 1.1 }, + cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 }, + // Rounded layout dimensions must not supply the scale. + layoutViewport: { clientWidth: 997 }, + cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) }, + }); + expect(measured).toBeCloseTo(scale, 12); + const coordinates = snapshotCoordinates( + { scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale }, + measured, + { x: 999, y: 999 }, + ); + expect( + snapshotViewportRect( + [160 * scale, 320 * scale, 120 * scale, 40 * scale], + { target }, + coordinates, + )?.rect, + ).toEqual({ x: 80, y: 120, width: 120, height: 40 }); + }); + + it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => { + const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 }); + expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({ + x: 60, + y: 300, + width: 120, + height: 40, + }); + expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull(); + expect( + snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }), + ).toBeNull(); + }); + + it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => { + expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull(); + for (const raw of [0, -1, NaN, Infinity]) { + expect( + snapshotLayoutScale({ + visualViewport: { clientWidth: raw }, + cssVisualViewport: { clientWidth: 800 }, + }), + ).toBeNull(); + } + expect( + snapshotLayoutScale({ + visualViewport: { clientWidth: 0, clientHeight: 600 }, + cssVisualViewport: { clientWidth: 0, clientHeight: 300 }, + }), + ).toBe(2); + expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull(); + expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull(); + }); + + it("shares in-flight frame measurements, but never shares them with the next context", async () => { + const cdp = driver(); + const context = new GeometryContext(cdp, 4); + const results = await Promise.all( + Array.from({ length: 20 }, () => context.targetProjection("child")), + ); + expect(results.every((value) => value === results[0])).toBe(true); + expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([ + "Page.getLayoutMetrics", + "DOM.getBoxModel", + ]); + await new GeometryContext(cdp, 4).targetProjection("child"); + expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2); + expect(cdp.calls.mock.calls).toHaveLength(4); + }); + + it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => { + const cdp = driver(); + const context = new GeometryContext(cdp, 4); + const source = { target, frameId: "child" }; + const projection = await context.snapshotProjection(source, 10, [], { + width: 1000, + height: 1000, + }); + if (projection.status !== "available") throw new Error("expected available projection"); + const input = snapshotViewportRect([17, 23, 120, 40], source, { + layoutUnitsPerCssPixel: 1, + scrollCss: { x: 0, y: 0 }, + }); + expect(projectSnapshotRect(input!, projection.projection)).toEqual({ + x: 73.75, + y: 641.25, + width: 150, + height: 50, + }); + const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { + layoutUnitsPerCssPixel: 1, + scrollCss: { x: 0, y: 10 }, + }); + expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75); + await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 }); + expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength( + 1, + ); + expect( + cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"), + ).toHaveLength(1); + }); + + it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => { + const cdp = driver(); + const original = cdp.send; + const calls: string[] = []; + cdp.send = async (tabId, method, params) => { + calls.push(method); + if (method === "Runtime.evaluate") + return { + result: { + deepSerializedValue: { + type: "array", + value: [ + { + type: "array", + value: [ + { type: "node", value: { backendNodeId: 10 } }, + { type: "string", value: '{"width":300,"height":200}' }, + ], + }, + { + type: "array", + value: [ + { type: "node", value: { backendNodeId: 11 } }, + { type: "string", value: '{"width":0,"height":200}' }, + ], + }, + ], + }, + }, + } as never; + if (method === "Runtime.releaseObjectGroup") return {} as never; + return original(tabId, method, params); + }; + const context = new GeometryContext(cdp, 4); + context.registerSnapshotOwners(target, new Set([10, 11])); + const values = await Promise.all( + [10, 11].map((id) => + context.snapshotProjection({ target, frameId: String(id) }, id, [], { + width: 1000, + height: 1000, + }), + ), + ); + expect(values.every((value) => value.status === "available")).toBe(true); + expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1); + expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1); + await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], { + width: 1000, + height: 1000, + }); + expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2); + }); + + it("waits for batch object cleanup and propagates cancellation", async () => { + const cdp = driver(); + const original = cdp.send; + const controller = new AbortController(); + let release!: () => void; + let cleanup!: () => void; + const started = new Promise((resolve) => { + cleanup = resolve; + }); + cdp.send = async (tabId, method, params) => { + if (method === "Runtime.evaluate") return {} as never; + if (method === "Runtime.releaseObjectGroup") { + cleanup(); + await new Promise((resolve) => { + release = resolve; + }); + return {} as never; + } + return original(tabId, method, params); + }; + const context = new GeometryContext(cdp, 4, undefined, controller.signal); + context.registerSnapshotOwners(target, new Set([10, 11])); + let settled = false; + const operation = context + .snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 }) + .finally(() => { + settled = true; + }); + const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" }); + await started; + controller.abort(); + expect(settled).toBe(false); + release(); + await rejected; + expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false); + }); + + it("rejects owner mismatches and invalid numeric bounds", () => { + const input = snapshotViewportRect( + [30, 50, 120, 40], + { target, frameId: "main" }, + { layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } }, + )!; + const projection = { + source: { target, frameId: "main" }, + geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } }, + }; + expect(projectSnapshotRect(input, projection)).toEqual({ + x: 20, + y: 30, + width: 120, + height: 40, + }); + expect( + projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }), + ).toBeNull(); + expect( + projectSnapshotRect(input, { + ...projection, + source: { target: { tabId: 5 }, frameId: "main" }, + }), + ).toBeNull(); + expect( + snapshotViewportRect( + [0, 0, NaN, 40], + { target }, + { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } }, + ), + ).toBeNull(); + expect( + snapshotViewportRect( + [0, 0, 10, 40], + { target }, + { layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } }, + ), + ).toBeNull(); + }); + + it("does not turn a border quad or a failed owner read into a content projection", async () => { + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + model: { border: [0, 0, 100, 0, 100, 100, 0, 100] }, + })) as CdpRunner["send"], + }; + expect( + await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], { + width: 100, + height: 100, + }), + ).toEqual({ + status: "unavailable", + source: { target, frameId: "child" }, + ownerBackendNodeId: 10, + }); + }); + + it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => { + const cdp = driver(); + const projection = await new GeometryContext(cdp, 4).snapshotProjection( + { target, frameId: "nested" }, + 10, + [rectPolygon({ x: 60, y: 620, w: 100, h: 100 })], + { width: 1000, height: 1000 }, + ); + if (projection.status !== "available") throw new Error("expected available projection"); + const input = snapshotViewportRect( + [0, 0, 300, 200], + { target, frameId: "nested" }, + { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } }, + ); + expect(projectSnapshotRect(input!, projection.projection)).toEqual({ + x: 60, + y: 620, + width: 100, + height: 100, + }); + }); + + it("rejects malformed frame ancestry without looping", async () => { + const cdp = driver(); + const cycle: CdpFrameGraph = { + rootFrameId: "a", + frames: [ + { frameId: "a", parentFrameId: "b", target }, + { frameId: "b", parentFrameId: "a", target }, + ], + }; + expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull(); + expect(cdp.calls).not.toHaveBeenCalled(); + const orphan: CdpFrameGraph = { + rootFrameId: "a", + frames: [{ frameId: "a", parentFrameId: "missing", target }], + }; + expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull(); + }); + + it("rejects a live node whose frame belongs to another target before any DOM input", async () => { + const cdp = driver(); + expect( + await resolveNodeGeometry( + cdp, + 4, + { target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 }, + { scrollIntoView: true }, + ), + ).toMatchObject({ code: "cdp_failed" }); + expect(cdp.calls).not.toHaveBeenCalled(); + }); + + it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => { + let active = 0; + let peak = 0; + const release: Array<() => void> = []; + const send = vi.fn(async () => { + active++; + peak = Math.max(peak, active); + await new Promise((resolve) => release.push(resolve)); + active--; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 100, clientHeight: 100 }, + }; + }); + const controller = new AbortController(); + const context = new GeometryContext( + { send: send as CdpRunner["send"] }, + 4, + undefined, + controller.signal, + ); + const tasks = Array.from({ length: 8 }, (_, index) => + context.viewport({ tabId: 4, sessionId: String(index) }), + ); + const settled = Promise.allSettled(tasks); + expect(send).toHaveBeenCalledTimes(4); + controller.abort(); + for (const done of release) done(); + const results = await settled; + expect(peak).toBe(4); + expect(send).toHaveBeenCalledTimes(4); + expect(results.filter((item) => item.status === "rejected")).toHaveLength(4); + }); + + it("releases an owner object even when cancellation arrives before its viewport read", async () => { + const controller = new AbortController(); + const cdp = driver(); + const original = cdp.calls.getMockImplementation()!; + cdp.calls.mockImplementation(async (tab, method) => { + const result = await original(tab, method); + if (method === "DOM.resolveNode") controller.abort(); + return result; + }); + const context = new GeometryContext(cdp, 4, undefined, controller.signal); + await expect( + context.snapshotProjection({ target, frameId: "child" }, 10, [], { + width: 1000, + height: 1000, + }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([ + "DOM.getBoxModel", + "DOM.resolveNode", + "Runtime.releaseObject", + ]); + }); + + it("reads live owner geometry only after scrolling", async () => { + const cdp = driver(); + const original = cdp.calls.getMockImplementation()!; + let scrolled = false; + cdp.calls.mockImplementation(async (tab, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") { + scrolled = true; + return {}; + } + if (method === "DOM.getContentQuads") { + expect(scrolled).toBe(true); + return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] }; + } + if (method === "DOM.getBoxModel") expect(scrolled).toBe(true); + return original(tab, method); + }); + expect( + await resolveNodeGeometry( + cdp, + 4, + { target, frameId: "child", backendNodeId: 20 }, + { scrollIntoView: true }, + ), + ).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } }); + expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength( + 1, + ); + }); + + it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => { + const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 }; + const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport); + expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } }); + expect( + screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }), + ).toBeNull(); + expect( + screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }), + ).toBeNull(); + }); + + it("retries a rejected measurement only in a new operation", async () => { + const cdp = driver(); + cdp.calls.mockRejectedValueOnce(new Error("unavailable")); + const context = new GeometryContext(cdp, 4); + await expect(context.viewport(target)).rejects.toThrow("unavailable"); + await expect(context.viewport(target)).rejects.toThrow("unavailable"); + expect(cdp.calls).toHaveBeenCalledTimes(1); + expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({ + width: 1000, + height: 1000, + }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 655e30c..76d989c 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -65,7 +65,11 @@ function makeFakeCdp(handlers: Record unknown>) { sent.push({ method, params }); const handler = handlers[method]; if (!handler && method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 }, + }; } if (!handler) throw new Error(`unexpected CDP call ${method}`); return handler(params); @@ -266,7 +270,15 @@ describe("handleScreenshot", () => { expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined(); }); - it("captures a clipped PNG when ref is given", async () => { + it.each([ + { zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } }, + { zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } }, + { zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } }, + ])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({ + zoom, + scroll, + expected, + }) => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e5", 999, { tabId: 7 }); @@ -274,6 +286,10 @@ describe("handleScreenshot", () => { "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "Page.captureScreenshot": () => ({ data: TINY_PNG }), + "Page.getLayoutMetrics": () => ({ + cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll }, + cssVisualViewport: { zoom }, + }), }); const captureVisibleTab = vi.fn(); const res = await handleScreenshot( @@ -297,7 +313,8 @@ describe("handleScreenshot", () => { clip?: { x: number; y: number; width: number; height: number }; } )?.clip; - expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); + expect(clip).toMatchObject(expected); + expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1); }); it("returns not_found for unknown ref", async () => { @@ -2481,7 +2498,11 @@ describe("handleSnapshot", () => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes }; if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") throw new Error("snapshot unsupported"); @@ -2568,7 +2589,11 @@ describe("handleSnapshot", () => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { @@ -2659,7 +2684,11 @@ describe("handleSnapshot", () => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { @@ -2753,7 +2782,11 @@ describe("handleSnapshot", () => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { @@ -2918,7 +2951,11 @@ describe("handleSnapshot", () => { }; } if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, + }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { @@ -3131,10 +3168,14 @@ describe("handleSnapshot", () => { } const VP_METRICS = { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; - function makeFrameAwareDeps() { + function makeFrameAwareDeps( + ownerGeometry: "unavailable" | "available" | "clipped" = "unavailable", + ) { const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"]; const i = (value: string) => strings.indexOf(value); const styles = [i("static"), i("auto"), i("auto")]; @@ -3142,6 +3183,8 @@ describe("handleSnapshot", () => { strings, documents: [ { + scrollOffsetX: 0, + scrollOffsetY: 0, frameId: "main", nodes: { parentIndex: [-1, 0, 1], @@ -3162,6 +3205,8 @@ describe("handleSnapshot", () => { }, }, { + scrollOffsetX: 0, + scrollOffsetY: 0, frameId: "child", nodes: { parentIndex: [-1, 0, 1], @@ -3214,6 +3259,16 @@ describe("handleSnapshot", () => { ]; const send = vi.fn(async (_tabId: number, method: string) => { if (method === "Page.getLayoutMetrics") return VP_METRICS; + if (ownerGeometry !== "unavailable") { + if (method === "DOM.getBoxModel") { + const x = ownerGeometry === "clipped" ? 10000 : 100; + return { model: { content: [x, 100, x + 400, 100, x + 400, 400, x, 400] } }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 400, height: 300 } } }; + if (method === "Runtime.releaseObject") return {}; + } if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; @@ -3454,6 +3509,43 @@ describe("handleSnapshot", () => { expect(result.text).toContain("•••"); }); + it.each([ + undefined, + 100, + 0, + ])("reports unavailable iframe geometry within the observation budget %s", async (maxTokens) => { + const result = await captureVomObservation(makeFrameAwareDeps().cdp, 4, "https://example.com", { + maxTokens, + }); + expect(result.text.match(/@warning/g)).toHaveLength(1); + expect(result.text).toContain("geometry incomplete"); + expect(result.text.startsWith("@vom 1\n")).toBe(true); + if (maxTokens === undefined) { + expect(result.truncated).toBe(false); + expect(result.text).toContain("Frame action"); + expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({ + rect: null, + localRect: { x: 20, y: 30, w: 120, h: 40 }, + }); + } + if (maxTokens === 100) expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens); + }); + + it.each([ + "available", + "clipped", + ] as const)("does not report successful %s geometry as an observation failure", async (ownerGeometry) => { + const result = await captureVomObservation( + makeFrameAwareDeps(ownerGeometry).cdp, + 4, + "https://example.com", + ); + expect(result.text).not.toContain("@warning"); + const node = result.matchNodes.find((item) => item.backendNodeId === 22); + if (ownerGeometry === "clipped") expect(node?.rect).toBeNull(); + else expect(node?.rect).toEqual({ x: 120, y: 130, w: 120, h: 40 }); + }); + it("keeps frames and rendered refs on the same frame identity", async () => { const deps = makeFrameAwareDeps(); diff --git a/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts b/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts new file mode 100644 index 0000000..5ef89d9 --- /dev/null +++ b/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts @@ -0,0 +1,237 @@ +// @vitest-environment node +import { describe, expect, it } from "vitest"; +import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { GeometryContext } from "../geometry/frame-context"; +import type { CdpRunner } from "../shared"; +import { captureViewModel } from "../vom/capture"; +import { captureFrameData } from "../vom/frame-capture"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] }; +type Rect = { x: number; y: number; w: number; h: number }; +type Oracle = { probe: Rect; owners: Record }; + +// The independent oracle uses DOM border boxes and this fixture's axis-aligned +// iframe transforms. No production snapshot conversion/projection builds expectations. +const oracleExpression = `(() => { + const box = document.querySelector('#probe').getBoundingClientRect(); + const owners = {}; + for (const frame of document.querySelectorAll('iframe')) { + const rect = frame.getBoundingClientRect(); + const style = getComputedStyle(frame); + const scale = rect.width / frame.offsetWidth; + owners[frame.name] = { + x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale, + y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale, + scale, + }; + } + return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners }; +})()`; + +describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => { + it.each([ + { deviceScale: 1, zoom: 1 }, + { deviceScale: 0.8, zoom: 1 }, + { deviceScale: 1, zoom: 1.25 }, + { deviceScale: 2, zoom: 1 }, + { deviceScale: 2, zoom: 0.8 }, + ])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => { + const evalRoot = new URL("../../../../../evals/browser/", import.meta.url); + const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href); + const { withChrome } = await import( + new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href + ); + const server = createEvalServer(); + const { baseUrl } = await server.start(); + try { + await withChrome( + { executable: process.env.BSK_GEOMETRY_CHROME, ...configuration }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId: rootSession } = await send<{ sessionId: string }>( + "Target.attachToTarget", + { targetId, flatten: true }, + ); + await send( + "Page.navigate", + { url: `${baseUrl}/snapshot-coordinates?run=coordinates` }, + rootSession, + ); + await expect + .poll( + () => + server + .snapshot("coordinates") + .events.some( + (event: { type: string; data: { root?: boolean } }) => + event.type === "geometry.ready" && event.data.root, + ), + { timeout: 10_000 }, + ) + .toBe(true); + + const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>( + "Target.getTargets", + ); + const sessions = [rootSession]; + for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) { + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId: target.targetId, + flatten: true, + }); + sessions.push(sessionId); + } + expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF. + const frames: CdpFrame[] = []; + const names = new Map(); + const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; + for (const sessionId of sessions) { + const { frameTree } = await send<{ frameTree: Tree }>( + "Page.getFrameTree", + {}, + sessionId, + ); + const target: CdpTarget = { + tabId: 1, + ...(sessionId === rootSession ? {} : { sessionId }), + }; + const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { + frames.push({ frameId: tree.frame.id, parentFrameId, target }); + names.set(tree.frame.id, tree.frame.name ?? ""); + for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); + }; + visit(frameTree); + } + expect(frames).toHaveLength(5); + for (const frame of frames) { + if (!frame.parentFrameId) continue; + const parent = frames.find((item) => item.frameId === frame.parentFrameId)!; + const owner = await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: frame.frameId }, + sessionFor(parent.target), + ); + frame.ownerBackendNodeId = owner.backendNodeId; + } + const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; + const calls: string[] = []; + const cdp: CdpRunner = { + send: (tabId, method, params) => { + calls.push(`${tabId}:${method}`); + return send(method, params, rootSession); + }, + sendToTarget: (target, method, params) => { + calls.push(`${target.sessionId ?? target.tabId}:${method}`); + return send(method, params, sessionFor(target)); + }, + getFrameGraph: async () => graph, + }; + const oracles = new Map(); + for (const frame of frames) { + const session = sessionFor(frame.target); + const { executionContextId } = await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { frameId: frame.frameId, worldName: "coordinate-oracle" }, + session, + ); + await expect + .poll( + async () => { + const ready = await send<{ result: { value: boolean } }>( + "Runtime.evaluate", + { + expression: 'document.documentElement.dataset.geometryReady === "true"', + contextId: executionContextId, + returnByValue: true, + }, + session, + ); + return ready.result.value; + }, + { timeout: 5_000 }, + ) + .toBe(true); + // Establish scroll after cross-process target creation/layout settles. + const scroll = frame.parentFrameId + ? names.get(frame.frameId) === "nested" + ? [10, 20] + : [40, 100] + : [80, 240]; + await send( + "Runtime.evaluate", + { + expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`, + contextId: executionContextId, + awaitPromise: true, + }, + session, + ); + const reply = await send<{ result: { value: Oracle } }>( + "Runtime.evaluate", + { expression: oracleExpression, contextId: executionContextId, returnByValue: true }, + session, + ); + oracles.set(frame.frameId, reply.result.value); + } + const geometry = new GeometryContext(cdp, 1, graph); + const captured = await captureViewModel(cdp, 1, { geometry }); + await captureFrameData(cdp, 1, captured, undefined, geometry); + expect(captured.frameGeometryIssues).toEqual([]); + for (const frame of frames) { + const node = captured.frameNodes + ?.get(frame.frameId) + ?.find((node) => node.attrs.id === "probe"); + expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined(); + const local = oracles.get(frame.frameId)!.probe; + const top = { ...local }; + let current = frame; + while (current.parentFrameId) { + const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!]; + top.x = owner.x + top.x * owner.scale; + top.y = owner.y + top.y * owner.scale; + top.w *= owner.scale; + top.h *= owner.scale; + current = frames.find((item) => item.frameId === current.parentFrameId)!; + } + expect( + node!.rect, + JSON.stringify({ + frame: names.get(frame.frameId), + local, + top, + viewport: captured.viewport, + }), + ).not.toBeNull(); + for (const key of ["x", "y", "w", "h"] as const) { + expect( + Math.abs(node!.localRect![key] - local[key]), + `${names.get(frame.frameId)} local ${key}`, + ).toBeLessThan(2); + expect( + Math.abs(node!.rect![key] - top[key]), + `${names.get(frame.frameId)} top ${key}`, + ).toBeLessThan(2); + } + } + expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength( + sessions.length, + ); + const metrics = await send<{ + cssVisualViewport: { zoom: number }; + visualViewport: { clientWidth: number }; + }>("Page.getLayoutMetrics", {}, rootSession); + expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4); + }, + ); + } finally { + await server.stop(); + } + }, 30_000); +}); diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index 0c788c9..0fb997c 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -1,24 +1,21 @@ -import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; import type { RpcError } from "@/transport/types"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { clipPolygon, - type GeometryProjection, type Point, type Polygon, - type ProjectiveEdge, - parseCdpQuad, polygonArea, polygonCentroid, projectRegionToViewport, - type Quad, type Region, rectPolygon, regionBounds, - type Size, type ViewportRect, } from "./geometry"; -import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; +import type { CssViewport } from "./geometry/coordinate-types"; +import { cssViewport, GeometryContext } from "./geometry/frame-context"; +import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared"; export interface NodeAddress { target: CdpTarget; @@ -27,6 +24,8 @@ export interface NodeAddress { } export interface ResolvedNodeGeometry { + /** The same top-level measurement used by projection, for screenshot coordinate adaptation. */ + topViewport: CssViewport; topVisibleRegions: Region; topBounds: ViewportRect; /** Point in the top-level tab viewport, used by root-target input events. */ @@ -39,145 +38,16 @@ function geometryError(message: string): RpcError { return { code: "cdp_failed", message }; } -function frameMap(graph: CdpFrameGraph): Map { - return new Map(graph.frames.map((frame) => [frame.frameId, frame])); -} - -function targetRootFrame(byId: Map, frame: CdpFrame): CdpFrame | null { - const seen = new Set(); - let current = frame; - while (current.parentFrameId) { - if (seen.has(current.frameId)) return null; - seen.add(current.frameId); - const parent = byId.get(current.parentFrameId); - if (!parent || parent.target.sessionId !== current.target.sessionId) break; - current = parent; - } - return current; -} - -function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null { - const byId = frameMap(graph); - const path: CdpFrame[] = []; - const seen = new Set(); - let current = byId.get(frameId); - if (!current) return null; - while (current.parentFrameId) { - if (seen.has(current.frameId)) return null; - seen.add(current.frameId); - path.push(current); - const parent = byId.get(current.parentFrameId); - if (!parent) return null; - current = parent; - } - return path; -} - -async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise { - const metrics = await sendToCdpTarget<{ - cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; - layoutViewport?: { clientWidth?: number; clientHeight?: number }; - }>(cdp, target, "Page.getLayoutMetrics", {}); - const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; - const width = viewport.clientWidth ?? 0; - const height = viewport.clientHeight ?? 0; - return width > 0 && height > 0 ? { width, height } : null; -} - -async function ownerContentQuad( - cdp: CdpRunner, - parent: CdpFrame, - ownerBackendNodeId: number, -): Promise { - const result = await sendToCdpTarget<{ model?: { content?: number[] } }>( - cdp, - parent.target, - "DOM.getBoxModel", - { backendNodeId: ownerBackendNodeId }, - ); - return parseCdpQuad(result.model?.content); -} - -async function sameTargetFrameClips( - cdp: CdpRunner, - byId: Map, - frame: CdpFrame, - targetRoot: CdpFrame, -): Promise { - const clips: Polygon[] = []; - let current = frame; - const seen = new Set(); - while (current.frameId !== targetRoot.frameId) { - if (seen.has(current.frameId) || !current.parentFrameId) return null; - seen.add(current.frameId); - const parent = byId.get(current.parentFrameId); - if (!parent || current.ownerBackendNodeId === undefined) return null; - const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId); - if (!clip) return null; - clips.push(clip); - current = parent; - } - return clips; -} - -export async function resolveFrameProjection( - cdp: CdpRunner, - graph: CdpFrameGraph, - frameId: string, -): Promise { - const byId = frameMap(graph); - const frame = byId.get(frameId); - if (!frame) return null; - let targetRoot = targetRootFrame(byId, frame); - if (!targetRoot) return null; - const sourceViewport = await targetViewport(cdp, targetRoot.target); - if (!sourceViewport) return null; - const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot); - if (!sourceClips) return null; - - const edges: ProjectiveEdge[] = []; - while (targetRoot.parentFrameId) { - const parent = byId.get(targetRoot.parentFrameId); - if (!parent || targetRoot.ownerBackendNodeId === undefined) return null; - const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId); - if (!destinationQuad) return null; - const source = - edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - if (!source) return null; - const parentTargetRoot = targetRootFrame(byId, parent); - if (!parentTargetRoot) return null; - const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot); - if (!destinationClips) return null; - edges.push({ sourceViewport: source, destinationQuad, destinationClips }); - targetRoot = parentTargetRoot; - } - - const topViewport = - edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - return topViewport ? { sourceClips, edges, topViewport } : null; -} - -async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise { - if (!cdp.getFrameGraph) return null; - try { - return await cdp.getFrameGraph(tabId); - } catch (error) { - console.debug("[bsk frame-geometry] frame graph resolution failed", error); - return null; - } -} - async function scrollFrameOwners( cdp: CdpRunner, tabId: number, - graph: CdpFrameGraph, + context: GeometryContext, frameId: string, ): Promise { - const byId = frameMap(graph); - const ancestry = frameAncestry(graph, frameId); + const ancestry = await context.ancestry(frameId); if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); for (const child of [...ancestry].reverse()) { - const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; + const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined; if (!parent || child.ownerBackendNodeId === undefined) { return geometryError(`could not resolve frame owner for ${child.frameId}`); } @@ -191,17 +61,20 @@ async function scrollFrameOwners( return null; } -async function scrollElementWithFrameGraph( +async function scrollElementWithContext( cdp: CdpRunner, tabId: number, target: CdpTarget, backendNodeId: number, frameId: string | undefined, - graph: CdpFrameGraph | null, + context: GeometryContext, ): Promise { if (frameId) { - if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); - const error = await scrollFrameOwners(cdp, tabId, graph, frameId); + const frame = await context.frame(frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) { + return geometryError("node frame does not belong to its target"); + } + const error = await scrollFrameOwners(cdp, tabId, context, frameId); if (error) return error; } else if (target.sessionId) { return geometryError("an OOPIF node address requires frameId"); @@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView( backendNodeId: number, frameId?: string, ): Promise { - const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; - return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); + if (target.tabId !== tabId) return geometryError("node target belongs to another tab"); + return scrollElementWithContext( + cdp, + tabId, + target, + backendNodeId, + frameId, + new GeometryContext(cdp, tabId), + ); } function largestRegion(regions: Region): Polygon | null { @@ -240,23 +120,32 @@ export async function resolveNodeGeometry( if (address.target.sessionId && !address.frameId) { return geometryError("an OOPIF node address requires frameId"); } - const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; + if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab"); + const context = new GeometryContext(cdp, tabId); + const graph = address.frameId ? await context.graph() : null; + if (address.frameId && graph) { + const frame = await context.frame(address.frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) { + return geometryError("node frame does not belong to its target"); + } + } if (address.frameId && !graph) { return geometryError(`could not resolve frame graph for ${address.frameId}`); } if (options.scrollIntoView) { - const scrollError = await scrollElementWithFrameGraph( + const scrollError = await scrollElementWithContext( cdp, tabId, address.target, address.backendNodeId, address.frameId, - graph, + context, ); if (scrollError) return scrollError; } + // Only topology was read above. Begin live measurements after scrolling. const localRegion = await nodeContentRegion( cdpRunnerForTarget(cdp, address.target), tabId, @@ -267,7 +156,7 @@ export async function resolveNodeGeometry( let topVisibleRegions: Region; let targetActionPoint: Point | null = null; if (address.frameId && graph) { - const projection = await resolveFrameProjection(cdp, graph, address.frameId); + const projection = await context.targetProjection(address.frameId); if (!projection) return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); @@ -290,7 +179,7 @@ export async function resolveNodeGeometry( targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; } } else { - const viewport = await targetViewport(cdp, address.target); + const viewport = await context.viewport(address.target); if (!viewport) return geometryError("could not resolve top viewport geometry"); topVisibleRegions = localRegion .map((polygon) => @@ -309,7 +198,13 @@ export async function resolveNodeGeometry( return { code: "permission_denied", message: "element not visible in its target" }; } targetActionPoint ??= actionPoint; - return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; + return { + topVisibleRegions, + topBounds, + actionPoint, + targetActionPoint, + topViewport: cssViewport(await context.layoutMetrics({ tabId })), + }; } catch (error) { return geometryError(error instanceof Error ? error.message : String(error)); } diff --git a/apps/extension/src/tools/geometry.ts b/apps/extension/src/tools/geometry.ts index fea8782..247ca02 100644 --- a/apps/extension/src/tools/geometry.ts +++ b/apps/extension/src/tools/geometry.ts @@ -245,22 +245,3 @@ export function projectRectToViewport( if (!rect) return null; return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); } - -export function childFrameProjection( - parent: GeometryProjection, - ownerRectInParent: { x: number; y: number; w: number; h: number }, -): GeometryProjection { - const destinationQuad = rectPolygon(ownerRectInParent); - return { - sourceClips: [], - edges: [ - { - sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h }, - destinationQuad, - destinationClips: parent.sourceClips, - }, - ...parent.edges, - ], - topViewport: parent.topViewport, - }; -} diff --git a/apps/extension/src/tools/geometry/coordinate-types.ts b/apps/extension/src/tools/geometry/coordinate-types.ts new file mode 100644 index 0000000..5b2c48e --- /dev/null +++ b/apps/extension/src/tools/geometry/coordinate-types.ts @@ -0,0 +1,157 @@ +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import type { GeometryProjection, ViewportRect } from "../geometry"; +import { projectRectToViewport } from "../geometry"; + +export interface CoordinateOwner { + target: CdpTarget; + frameId?: string; +} + +export interface FrameViewportRect { + space: "frame-viewport-css"; + owner: CoordinateOwner; + rect: ViewportRect; +} + +export interface SnapshotProjection { + source: CoordinateOwner; + geometry: GeometryProjection; +} + +/** A failed owner read is different from a successfully projected, clipped-out rect. */ +export interface UnavailableFrameProjection { + status: "unavailable"; + source: CoordinateOwner; + ownerBackendNodeId: number; +} + +export interface UnavailableSnapshotCoordinates { + status: "unavailable"; + source: CoordinateOwner; + reason: "snapshot-coordinates-unavailable"; +} + +export type SnapshotProjectionResult = + | { status: "available"; projection: SnapshotProjection } + | UnavailableFrameProjection; + +/** Descendants inherit the failed boundary without attempting another owner read. */ +export type FrameProjectionIssue = + | UnavailableFrameProjection + | UnavailableSnapshotCoordinates + | { + status: "blocked"; + source: CoordinateOwner; + cause: UnavailableFrameProjection | UnavailableSnapshotCoordinates; + }; + +export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue; + +export interface SnapshotCoordinates { + layoutUnitsPerCssPixel: number; + scrollCss: { x: number; y: number }; +} + +/** Snapshot scroll offsets share the raw layout units of its bounds. Only a + * target's root document may fall back to that target's CSS layout viewport. */ +export function snapshotCoordinates( + document: { scrollOffsetX?: number; scrollOffsetY?: number }, + layoutUnitsPerCssPixel: number | null, + rootScrollCss?: { x?: number; y?: number }, +): SnapshotCoordinates | null { + if ( + layoutUnitsPerCssPixel === null || + !Number.isFinite(layoutUnitsPerCssPixel) || + layoutUnitsPerCssPixel <= 0 + ) + return null; + const x = + document.scrollOffsetX === undefined + ? rootScrollCss?.x + : document.scrollOffsetX / layoutUnitsPerCssPixel; + const y = + document.scrollOffsetY === undefined + ? rootScrollCss?.y + : document.scrollOffsetY / layoutUnitsPerCssPixel; + if (x === undefined || y === undefined || !Number.isFinite(x) || !Number.isFinite(y)) return null; + return { layoutUnitsPerCssPixel, scrollCss: { x, y } }; +} + +/** Convert raw document-relative Blink layout units exactly once, before any + * CSS viewport clipping or frame projection. This scale is not screenshot zoom. */ +export function snapshotViewportRect( + bounds: number[], + owner: CoordinateOwner, + coordinates: SnapshotCoordinates | null, +): FrameViewportRect | null { + if (!coordinates || bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null; + const { layoutUnitsPerCssPixel: scale, scrollCss: scroll } = coordinates; + const [x, y, width, height] = bounds; + if ( + !Number.isFinite(scale) || + scale <= 0 || + width <= 0 || + height <= 0 || + !Number.isFinite(scroll.x) || + !Number.isFinite(scroll.y) + ) { + return null; + } + return { + space: "frame-viewport-css", + owner, + rect: { + x: x / scale - scroll.x, + y: y / scale - scroll.y, + width: width / scale, + height: height / scale, + }, + }; +} + +export function projectSnapshotRect( + input: FrameViewportRect, + projection: SnapshotProjection, +): ViewportRect | null { + if ( + input.owner.frameId !== projection.source.frameId || + cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target) + ) { + return null; + } + const { x, y, width: w, height: h } = input.rect; + return projectRectToViewport({ x, y, w, h }, projection.geometry); +} + +/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */ +export interface CssViewport { + width: number; + height: number; + scrollX: number; + scrollY: number; + cssToDip: number; +} + +/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */ +export function screenshotPageRect( + rect: ViewportRect, + viewport: CssViewport, +): { space: "page-dip"; rect: ViewportRect } | null { + const { cssToDip, scrollX, scrollY } = viewport; + if ( + ![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) || + cssToDip <= 0 || + rect.width <= 0 || + rect.height <= 0 + ) + return null; + return { + space: "page-dip", + rect: { + x: (rect.x + scrollX) * cssToDip, + y: (rect.y + scrollY) * cssToDip, + width: rect.width * cssToDip, + height: rect.height * cssToDip, + }, + }; +} diff --git a/apps/extension/src/tools/geometry/frame-context.ts b/apps/extension/src/tools/geometry/frame-context.ts new file mode 100644 index 0000000..c728096 --- /dev/null +++ b/apps/extension/src/tools/geometry/frame-context.ts @@ -0,0 +1,356 @@ +import { + type CdpFrame, + type CdpFrameGraph, + type CdpTarget, + cdpTargetKey, +} from "@/browser-driver/frame-graph"; +import { + type GeometryProjection, + type Polygon, + type ProjectiveEdge, + parseCdpQuad, + type Quad, + type Size, +} from "../geometry"; +import { type CdpRunner, sendToCdpTarget } from "../shared"; +import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types"; + +import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes"; + +export interface LayoutMetrics { + cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number }; + visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number }; + cssLayoutViewport?: { + clientWidth?: number; + clientHeight?: number; + pageX?: number; + pageY?: number; + }; + layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number }; +} + +/** Chromium emits these paired floating-point dimensions before/after dividing + * by LayoutZoomFactor. Integer layout viewport dimensions lose precision at + * fractional zoom. Neither visualViewport.scale nor .zoom is this conversion. + * See InspectorPageAgent::getLayoutMetrics in Chromium. */ +export function snapshotLayoutScale(metrics: LayoutMetrics): number | null { + for (const dimension of ["clientWidth", "clientHeight"] as const) { + const raw = metrics.visualViewport?.[dimension]; + const css = metrics.cssVisualViewport?.[dimension]; + if ( + raw !== undefined && + css !== undefined && + Number.isFinite(raw) && + Number.isFinite(css) && + raw > 0 && + css > 0 + ) { + const scale = raw / css; + if (Number.isFinite(scale) && scale > 0) return scale; + } + } + return null; +} + +/** The legacy field is a compatibility fallback, never a source of raster DPR. */ +export function cssViewport(metrics: LayoutMetrics): CssViewport { + const value = metrics.cssLayoutViewport ?? metrics.layoutViewport; + return { + width: value?.clientWidth ?? 0, + height: value?.clientHeight ?? 0, + scrollX: value?.pageX ?? 0, + scrollY: value?.pageY ?? 0, + cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1, + }; +} + +/** One read-only measurement phase. Discard after scrolling or any later operation. */ +export class GeometryContext { + private readonly metrics = new Map>(); + private readonly owners = new Map>(); + private readonly projections = new Map>(); + private readonly snapshotOwners = new Map>(); + private readonly sizeBatches = new Map Promise>>(); + private active = 0; + private readonly waiting: Array<() => void> = []; + private graphPromise?: Promise; + private framesPromise?: Promise>; + + constructor( + private readonly cdp: CdpRunner, + private readonly tabId: number, + graph?: CdpFrameGraph, + private readonly signal?: AbortSignal, + ) { + if (graph) this.graphPromise = Promise.resolve(graph); + } + + private async request( + target: CdpTarget, + method: string, + params: object, + cleanup = false, + ): Promise { + if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError"); + if (this.active >= 4) await new Promise((resolve) => this.waiting.push(resolve)); + else this.active++; + try { + if (!cleanup && this.signal?.aborted) + throw new DOMException("geometry aborted", "AbortError"); + return await sendToCdpTarget(this.cdp, target, method, params); + } finally { + const next = this.waiting.shift(); + if (next) next(); + else this.active--; + } + } + + graph(): Promise { + this.graphPromise ??= + this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null); + return this.graphPromise; + } + + private frames(): Promise> { + this.framesPromise ??= this.graph().then( + (graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []), + ); + return this.framesPromise; + } + + async frame(frameId: string): Promise { + return (await this.frames()).get(frameId); + } + + /** Topology only: safe to resolve before the caller scrolls frame owners. */ + async ancestry(frameId: string): Promise { + const frames = await this.frames(); + const path: CdpFrame[] = []; + const seen = new Set(); + let current = frames.get(frameId); + if (!current) return null; + while (current.parentFrameId) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + path.push(current); + const parent = frames.get(current.parentFrameId); + if (!parent) return null; + current = parent; + } + return path; + } + + layoutMetrics(target: CdpTarget): Promise { + const key = cdpTargetKey(target); + let promise = this.metrics.get(key); + if (!promise) { + promise = this.request(target, "Page.getLayoutMetrics", {}); + this.metrics.set(key, promise); + } + return promise; + } + + async viewport(target: CdpTarget): Promise { + const { width, height } = cssViewport(await this.layoutMetrics(target)); + return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0 + ? { width, height } + : null; + } + + ownerContent(target: CdpTarget, backendNodeId: number): Promise { + const key = `${cdpTargetKey(target)}:${backendNodeId}`; + let promise = this.owners.get(key); + if (!promise) { + promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", { + backendNodeId, + }).then((reply) => parseCdpQuad(reply.model?.content)); + this.owners.set(key, promise); + } + return promise; + } + + /** Same-target owner quads are already target-relative, including nested owners. */ + async snapshotProjection( + source: CoordinateOwner, + ownerBackendNodeId: number, + ancestorClips: Polygon[], + viewport: Size, + ): Promise { + const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`; + let promise = this.snapshotOwners.get(key); + if (!promise) { + promise = this.snapshotOwner(source.target, ownerBackendNodeId); + this.snapshotOwners.set(key, promise); + } + let owner: { quad: Quad; size: Size } | null; + try { + owner = await promise; + } catch (error) { + if (error && typeof error === "object" && "name" in error && error.name === "AbortError") + throw error; + console.debug("[bsk geometry] frame owner unavailable", error); + owner = null; + } + return owner + ? { + status: "available", + projection: { + source, + geometry: { + sourceClips: [], + edges: [ + { + sourceViewport: owner.size, + destinationQuad: owner.quad, + destinationClips: ancestorClips, + }, + ], + topViewport: viewport, + }, + }, + } + : { status: "unavailable", source, ownerBackendNodeId }; + } + + /** Register one target's snapshot owners; the batch starts only when a + * projection actually needs a size. Single owners keep the cheaper old path. */ + registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet): void { + const key = cdpTargetKey(target); + if (owners.size < 2 || this.sizeBatches.has(key)) return; + let pending: Promise> | undefined; + this.sizeBatches.set(key, () => { + pending ??= readSnapshotOwnerSizes( + (method, params, cleanup) => this.request(target, method, params, cleanup), + owners, + ).catch((error) => { + if (error && typeof error === "object" && "name" in error && error.name === "AbortError") + throw error; + return new Map(); + }); + return pending; + }); + } + + private async snapshotOwner( + target: CdpTarget, + backendNodeId: number, + ): Promise<{ quad: Quad; size: Size } | null> { + const quad = await this.ownerContent(target, backendNodeId); + if (!quad) return null; + const batchSize = (await this.sizeBatches.get(cdpTargetKey(target))?.())?.get(backendNodeId); + if (this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError"); + if (batchSize) return { quad, size: batchSize }; + const resolved = await this.request<{ object?: { objectId?: string } }>( + target, + "DOM.resolveNode", + { backendNodeId }, + ); + const objectId = resolved.object?.objectId; + if (!objectId) return null; + try { + const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>( + target, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: `function() { + if (!this.isConnected) return null; + const style = getComputedStyle(this); + return { + width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight), + height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom) + }; + }`, + returnByValue: true, + }, + ); + const size = reply.result?.value; + return !reply.exceptionDetails && + size && + Number.isFinite(size.width) && + Number.isFinite(size.height) && + size.width > 0 && + size.height > 0 + ? { quad, size } + : null; + } finally { + // Cleanup must still run if cancellation arrived after resolving the object. + await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {}); + } + } + + private targetRoot(frames: Map, frame: CdpFrame): CdpFrame | null { + const seen = new Set(); + let current = frame; + while (current.parentFrameId) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + const parent = frames.get(current.parentFrameId); + if (!parent) return null; + if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break; + current = parent; + } + return current; + } + + private async clips( + frames: Map, + frame: CdpFrame, + root: CdpFrame, + ): Promise { + const clips: Polygon[] = []; + let current = frame; + const seen = new Set(); + while (current.frameId !== root.frameId) { + if (seen.has(current.frameId) || !current.parentFrameId) return null; + seen.add(current.frameId); + const parent = frames.get(current.parentFrameId); + if (!parent || current.ownerBackendNodeId === undefined) return null; + const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId); + if (!quad) return null; + clips.push(quad); + current = parent; + } + return clips; + } + + /** Input is already in the frame's CDP target viewport, not its local viewport. */ + targetProjection(frameId: string): Promise { + let promise = this.projections.get(frameId); + if (!promise) { + promise = this.buildTargetProjection(frameId); + this.projections.set(frameId, promise); + } + return promise; + } + + private async buildTargetProjection(frameId: string): Promise { + const frames = await this.frames(); + const frame = frames.get(frameId); + if (!frame) return null; + let root = this.targetRoot(frames, frame); + if (!root) return null; + const sourceViewport = await this.viewport(root.target); + if (!sourceViewport) return null; + const sourceClips = await this.clips(frames, frame, root); + if (!sourceClips) return null; + const edges: ProjectiveEdge[] = []; + const seen = new Set(); + while (root.parentFrameId) { + if (seen.has(root.frameId)) return null; + seen.add(root.frameId); + const parent = frames.get(root.parentFrameId); + if (!parent || root.ownerBackendNodeId === undefined) return null; + const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId); + const source = await this.viewport(root.target); + const parentRoot = this.targetRoot(frames, parent); + if (!destinationQuad || !source || !parentRoot) return null; + const destinationClips = await this.clips(frames, parent, parentRoot); + if (!destinationClips) return null; + edges.push({ sourceViewport: source, destinationQuad, destinationClips }); + root = parentRoot; + } + const topViewport = await this.viewport(root.target); + return topViewport ? { sourceClips, edges, topViewport } : null; + } +} diff --git a/apps/extension/src/tools/geometry/snapshot-owner-sizes.ts b/apps/extension/src/tools/geometry/snapshot-owner-sizes.ts new file mode 100644 index 0000000..0de2c55 --- /dev/null +++ b/apps/extension/src/tools/geometry/snapshot-owner-sizes.ts @@ -0,0 +1,74 @@ +import type { Size } from "../geometry"; + +interface SerializedValue { + type: string; + value?: unknown; +} + +type Send = (method: string, params: object, cleanup?: boolean) => Promise; + +/** Batch only accessible frame owners. Missing/shadow/cross-origin owners keep + * the existing node-scoped read; no page-wide style columns are requested. */ +export async function readSnapshotOwnerSizes( + send: Send, + owners: ReadonlySet, +): Promise> { + const sizes = new Map(); + const objectGroup = `bsk-frame-sizes-${crypto.randomUUID()}`; + try { + const reply = await send<{ result?: { deepSerializedValue?: SerializedValue } }>( + "Runtime.evaluate", + { + expression: `(() => { + const result = []; + const documents = [document]; + for (let i = 0; i < documents.length && result.length < ${owners.size}; i++) { + for (const owner of documents[i].querySelectorAll('iframe,frame')) { + if (result.length >= ${owners.size}) break; + if (!owner.isConnected) continue; + const style = getComputedStyle(owner); + result.push([owner, JSON.stringify({ + width: owner.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight), + height: owner.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom) + })]); + try { if (owner.contentDocument) documents.push(owner.contentDocument); } catch {} + } + } + return result; + })()`, + objectGroup, + serializationOptions: { + serialization: "deep", + maxDepth: 3, + additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" }, + }, + }, + ); + const result = reply.result?.deepSerializedValue; + if (result?.type !== "array" || !Array.isArray(result.value)) return sizes; + for (const entry of result.value as SerializedValue[]) { + if (entry.type !== "array" || !Array.isArray(entry.value)) continue; + const [node, data] = entry.value as SerializedValue[]; + if (node?.type !== "node" || data?.type !== "string" || typeof data.value !== "string") + continue; + const id = (node.value as { backendNodeId?: number } | undefined)?.backendNodeId; + if (id === undefined || !owners.has(id)) continue; + try { + const size = JSON.parse(data.value) as Size; + if ( + size && + Number.isFinite(size.width) && + Number.isFinite(size.height) && + size.width > 0 && + size.height > 0 + ) + sizes.set(id, size); + } catch { + /* An invalid entry uses the existing individual read. */ + } + } + } finally { + await send("Runtime.releaseObjectGroup", { objectGroup }, true).catch(() => {}); + } + return sizes; +} diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 9c944ab..6f3cae2 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -34,6 +34,8 @@ import type { import { attachDialogs, markDialogCursor } from "./dialogs"; import { rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; +import { screenshotPageRect } from "./geometry/coordinate-types"; +import { cssViewport, GeometryContext } from "./geometry/frame-context"; import { type ChromeTabsApi, enforceToolTargetScope, @@ -197,15 +199,14 @@ async function captureElementScreenshot( if (isRpcError(geometry)) return geometry; if (signal?.aborted) return cancelled("screenshot"); const rect = geometry.topBounds; + const clip = screenshotPageRect(rect, geometry.topViewport); + if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" }; try { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { format: "png", clip: { - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height, + ...clip.rect, scale: 1, }, }); @@ -984,26 +985,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; } -interface LayoutMetricsViewportReply { - cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; - layoutViewport?: { clientWidth?: number; clientHeight?: number }; -} - async function fallbackCapturedViewModel( cdp: CdpRunner, tabId: number, + geometry: GeometryContext, signal?: AbortSignal, ): Promise { throwIfAborted(signal, "observation"); let viewport = { width: 0, height: 0 }; try { - const metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); + const metrics = await geometry.layoutMetrics({ tabId }); throwIfAborted(signal, "observation"); - const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; - viewport = { - width: source.clientWidth ?? 0, - height: source.clientHeight ?? 0, - }; + const source = cssViewport(metrics); + viewport = { width: source.width, height: source.height }; } catch (error) { if (isAbortError(error)) throw error; } @@ -1016,12 +1010,13 @@ async function captureForVom( cdp: CdpRunner, tabId: number, options: CaptureVomObservationOptions, + geometry: GeometryContext, ): Promise { try { - return await captureViewModel(cdp, tabId, { signal: options.signal }); + return await captureViewModel(cdp, tabId, { signal: options.signal, geometry }); } catch (error) { if (isAbortError(error)) throw error; - return fallbackCapturedViewModel(cdp, tabId, options.signal); + return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal); } } @@ -1096,9 +1091,16 @@ export async function captureVomObservation( throwIfAborted(options.signal, "observation"); await cdp.ensureAttachedToUrl?.(tabId, url); throwIfAborted(options.signal, "observation"); - const captured = await captureForVom(cdp, tabId, options); + const geometry = new GeometryContext(cdp, tabId, undefined, options.signal); + const captured = await captureForVom(cdp, tabId, options, geometry); throwIfAborted(options.signal, "observation"); - const documents = await captureFrameData(cdp, tabId, captured, options.signal); + const documents = await captureFrameData( + cdp, + tabId, + captured, + options.signal, + geometry, + ); throwIfAborted(options.signal, "observation"); const normalizedDocuments = documents.length === 1 && captured.iframeNodes.size > 0 @@ -1131,12 +1133,21 @@ export async function captureVomObservation( captured, hoverProbes.surfaceProbes, ); + // Reserve space using the renderer's character-based token estimate. Like VOM + // headers, this integrity notice remains visible even under a tiny token budget. + const geometryNotice = captured.frameGeometryIssues?.length + ? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n" + : ""; const rendered = renderVom(decoratedScene, { maxDepth: options.maxDepth, - maxTokens: options.maxTokens, + maxTokens: + !geometryNotice || options.maxTokens === undefined + ? options.maxTokens + : Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)), redactValues: options.redactValues, activeRegionPolicy: options.activeRegionPolicy, }); + if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`; throwIfAborted(options.signal, "observation"); return projectRecordSafeObservation({ rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root", diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 97b4658..b70ca2d 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; +import type { CdpRunner } from "../../shared"; import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; // Minimal but format-accurate captureSnapshot reply: a body with one @@ -210,7 +211,11 @@ function makeCdp(snapshot: unknown) { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; } if (method === "Runtime.evaluate") { return { @@ -298,6 +303,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -361,6 +368,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -409,6 +418,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -742,6 +753,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 }, }; } @@ -754,7 +767,46 @@ describe("captureViewModel", () => { expect(div?.rect).toMatchObject({ y: 0, h: 600 }); }); - it("normalizes device-pixel bounds by devicePixelRatio", async () => { + it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => { + const snapshot = { + strings: ["html", "button"], + documents: [ + { + frameId: "root", + scrollOffsetX: 40, + scrollOffsetY: 200, + nodes: { + parentIndex: [-1, 0], + nodeName: [0, 1], + backendNodeId: [10, 11], + attributes: [[], []], + }, + layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] }, + }, + ], + }; + const cdp: CdpRunner = { + send: vi.fn(async (_tab: number, method: string) => { + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") + return { + visualViewport: { clientWidth: 2000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 }, + }; + return {}; + }) as CdpRunner["send"], + }; + const captured = await captureViewModel(cdp, 4); + expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({ + localRect: { x: 80, y: 300, w: 120, h: 40 }, + rect: { x: 80, y: 300, w: 120, h: 40 }, + rendered: true, + }); + expect(captured.frameGeometryIssues).toEqual([]); + }); + + it("normalizes raw snapshot bounds before viewport clipping", async () => { const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { @@ -788,6 +840,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 2000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, layoutViewport: { clientWidth: 2000, clientHeight: 1600 }, }; @@ -802,6 +856,12 @@ describe("captureViewModel", () => { w: 1000, h: 800, }); + expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({ + x: 0, + y: 0, + w: 1000, + h: 800, + }); }); it("extracts textContent from #text child nodes via nodeValue", async () => { @@ -857,6 +917,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -947,9 +1009,17 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } + if (method === "DOM.getBoxModel") + return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } }; + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 800, height: 400 } } }; + if (method === "Runtime.releaseObject") return {}; throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; @@ -979,7 +1049,7 @@ describe("captureViewModel", () => { expect(frameParentIds?.get("child-frame")).toBe("main-frame"); }); - it("captures iframe documents without requiring owner-frame geometry", async () => { + it("projects iframe documents independently of owner snapshot geometry", async () => { const S = [ "html", "body", @@ -1037,9 +1107,17 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } + if (method === "DOM.getBoxModel") + return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } }; + if (method === "DOM.resolveNode") return { object: { objectId: "owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 800, height: 400 } } }; + if (method === "Runtime.releaseObject") return {}; throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; @@ -1048,7 +1126,7 @@ describe("captureViewModel", () => { const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); - expect(input?.rect).toBeNull(); + expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 }); }); it("recursively normalizes nested iframe sub-documents", async () => { @@ -1097,6 +1175,8 @@ describe("captureViewModel", () => { attributes: [[], [], []], contentDocumentIndex: { index: [2], value: [2] }, }, + scrollOffsetX: 0, + scrollOffsetY: 0, layout: { nodeIndex: [1, 2], styles: [ @@ -1117,6 +1197,8 @@ describe("captureViewModel", () => { backendNodeId: [30, 31], attributes: [[], [i("type"), i("text")]], }, + scrollOffsetX: 0, + scrollOffsetY: 0, layout: { nodeIndex: [1], styles: [[i("static"), i("auto")]], @@ -1127,16 +1209,45 @@ describe("captureViewModel", () => { ], }; const cdp = { - send: vi.fn(async (_t: number, method: string) => { - if (method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return snapshot; - if (method === "Page.getLayoutMetrics") { - return { - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, - }; - } - throw new Error(method); - }) as unknown as (tabId: number, method: string, params?: object) => Promise, + send: vi.fn( + async ( + _t: number, + method: string, + params?: { backendNodeId?: number; objectId?: string }, + ) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") { + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "DOM.getBoxModel") + return { + model: { + content: + params?.backendNodeId === 13 + ? [10, 20, 310, 20, 310, 220, 10, 220] + : [15, 26, 115, 26, 115, 106, 15, 106], + }, + }; + if (method === "DOM.resolveNode") + return { object: { objectId: String(params?.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") + return { + result: { + value: + params?.objectId === "13" + ? { width: 300, height: 200 } + : { width: 100, height: 80 }, + }, + }; + if (method === "Runtime.releaseObject") return {}; + throw new Error(method); + }, + ) as unknown as (tabId: number, method: string, params?: object) => Promise, }; const { iframeNodes } = await captureViewModel(cdp, 4); @@ -1150,7 +1261,7 @@ describe("captureViewModel", () => { expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); }); - it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { + it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => { const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { @@ -1184,6 +1295,8 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { + visualViewport: { clientWidth: 2000 }, + cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 }, layoutViewport: { clientWidth: 2000, clientHeight: 1600 }, }; @@ -1217,3 +1330,200 @@ describe("captureViewModel", () => { expect(excluded).toEqual(new Set([200, 201, 202])); }); }); + +// Six sibling frames and one nested frame exercise scheduling through the real capture entry. +function siblingCaptureFixture( + beforeReply: (method: string, params: Record) => Promise = async () => {}, +) { + const document = (id: number, owners: number[], childIndexes: number[]) => ({ + scrollOffsetX: 0, + scrollOffsetY: 0, + frameId: `frame-${id}`, + nodes: { + parentIndex: [-1, ...owners.map(() => 0)], + nodeName: [0, ...owners.map(() => 1)], + backendNodeId: [1000 + id, ...owners], + attributes: [[], ...owners.map(() => [])], + contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes }, + }, + layout: { + nodeIndex: [0, ...owners.map((_, i) => i + 1)], + bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])], + }, + }); + const snapshot = { + strings: ["body", "iframe"], + documents: [ + document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]), + document(1, [200], [7]), + ...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])), + ], + }; + let active = 0; + let peak = 0; + const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => { + const args = params as Record; + active++; + peak = Math.max(peak, active); + try { + await beforeReply(method, args); + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, + }; + if (method === "DOM.getBoxModel") + return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } }; + if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 200, height: 100 } } }; + return {}; + } finally { + active--; + } + }); + return { + cdp: { send: send as CdpRunner["send"] }, + send, + peak: () => peak, + active: () => active, + measured: () => + send.mock.calls + .filter(([, method]) => method === "DOM.getBoxModel") + .map(([, , params]) => (params as { backendNodeId: number }).backendNodeId), + }; +} + +describe("sibling frame measurement scheduling", () => { + it("batches same-target owner sizes without changing projected results", async () => { + const fixture = siblingCaptureFixture(); + const original = fixture.cdp.send; + const methods: string[] = []; + fixture.cdp.send = async (tabId, method, params) => { + methods.push(method); + if (method === "Runtime.evaluate") + return { + result: { + deepSerializedValue: { + type: "array", + value: [100, 101, 102, 103, 104, 105, 200].map((backendNodeId) => ({ + type: "array", + value: [ + { type: "node", value: { backendNodeId } }, + { type: "string", value: JSON.stringify({ width: 200, height: 100 }) }, + ], + })), + }, + }, + } as never; + return original(tabId, method, params); + }; + const captured = await captureViewModel(fixture.cdp, 4); + expect(captured.frameGeometryIssues).toEqual([]); + expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]); + expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + expect(methods.filter((method) => method === "DOM.getBoxModel")).toHaveLength(7); + expect(methods.filter((method) => method === "Runtime.evaluate")).toHaveLength(1); + expect(methods.filter((method) => method === "Runtime.releaseObjectGroup")).toHaveLength(1); + expect(methods).not.toContain("DOM.resolveNode"); + expect(methods).not.toContain("Runtime.callFunctionOn"); + }); + + it("bounds concurrent reads, fills free slots and preserves depth-first output despite reordered replies", async () => { + const pending = new Map void>(); + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200) + await new Promise((resolve) => pending.set(Number(params.backendNodeId), resolve)); + }); + const capture = captureViewModel(fixture.cdp, 4); + await vi.waitFor(() => expect(pending.size).toBe(4)); + expect(fixture.measured()).toEqual([100, 101, 102, 103]); + pending.get(103)!(); + await vi.waitFor(() => expect(pending.has(104)).toBe(true)); + pending.get(104)!(); + await vi.waitFor(() => expect(pending.has(105)).toBe(true)); + expect(fixture.measured()).not.toContain(200); + for (const id of [105, 102, 101, 100]) pending.get(id)!(); + const captured = await capture; + expect(captured.frameGeometryIssues).toEqual([]); + expect(fixture.peak()).toBe(4); + expect(fixture.active()).toBe(0); + expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]); + expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]); + expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + for (const method of [ + "DOM.getBoxModel", + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Runtime.releaseObject", + ]) + expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7); + }); + + it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => { + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.resolveNode" && params.backendNodeId === 100) + throw new Error("owner replaced"); + }); + const captured = await captureViewModel(fixture.cdp, 4); + expect(captured.frameGeometryIssues).toEqual([ + { + status: "unavailable", + source: { target: { tabId: 4 }, frameId: "frame-1" }, + ownerBackendNodeId: 100, + }, + { + status: "blocked", + source: { target: { tabId: 4 }, frameId: "frame-7" }, + cause: captured.frameGeometryIssues?.[0], + }, + ]); + expect(fixture.measured()).not.toContain(200); + expect(captured.iframeNodes.get(200)?.[0]).toMatchObject({ + tag: "body", + rect: null, + localRect: { x: 0, y: 0, w: 200, h: 100 }, + }); + expect(captured.iframeNodes.get(101)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]); + }); + + it("stops scheduling on cancellation and waits for resolved objects to be released", async () => { + const controller = new AbortController(); + const resolutions = new Map void>(); + const releases = new Map void>(); + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.resolveNode") + await new Promise((resolve) => + resolutions.set(String(params.backendNodeId), resolve), + ); + if (method === "Runtime.releaseObject") + await new Promise((resolve) => releases.set(String(params.objectId), resolve)); + }); + let settled = false; + const capture = captureViewModel(fixture.cdp, 4, { signal: controller.signal }); + const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" }); + void capture.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.waitFor(() => expect(resolutions.size).toBe(4)); + controller.abort(); + for (const resolve of resolutions.values()) resolve(); + await vi.waitFor(() => expect(releases.size).toBe(4)); + expect(settled).toBe(false); + expect(fixture.measured()).toEqual([100, 101, 102, 103]); + for (const release of releases.values()) release(); + await rejected; + expect(fixture.active()).toBe(0); + expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe( + false, + ); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts b/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts index 6bbd224..1e7d5b5 100644 --- a/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts @@ -25,6 +25,8 @@ function childSnapshot(frameId: string, backendNodeId: number) { strings, documents: [ { + scrollOffsetX: 0, + scrollOffsetY: 0, frameId, nodes: { parentIndex: [-1, 0], @@ -50,6 +52,72 @@ function childSnapshot(frameId: string, backendNodeId: number) { } describe("captureFrameData", () => { + it("retains nested owner failures when merging a captured OOPIF", async () => { + const owner = ownerNode(10, 50); + const captured: CapturedViewModel = { + nodes: [owner], + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + frameNodes: new Map([["main", [owner]]]), + rootFrameId: "main", + excludedBackendNodeIds: new Set(), + }; + const child = childSnapshot("child", 101); + const nested = childSnapshot("nested", 201); + const document = child.documents[0]; + const snapshot = { + strings: child.strings, + documents: [ + { + ...document, + nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } }, + }, + nested.documents[0], + ], + }; + const reply = async (_target: unknown, method: string) => { + if (method === "Page.getLayoutMetrics") + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 300, clientHeight: 200 }, + }; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "DOM.getBoxModel") + return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } }; + if (method === "DOM.resolveNode") throw new Error("nested owner replaced"); + return {}; + }; + const cdp: CdpRunner = { + send: vi.fn(reply) as CdpRunner["send"], + sendToTarget: vi.fn(reply) as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + await captureFrameData(cdp, 4, captured); + expect(captured.frameGeometryIssues).toEqual([ + { + status: "unavailable", + source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" }, + ownerBackendNodeId: 101, + }, + ]); + expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({ + rect: null, + localRect: { x: 10, y: 20, w: 100, h: 40 }, + }); + }); + it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { const leftOwner = ownerNode(10, 50); const rightOwner = ownerNode(20, 500); @@ -64,7 +132,11 @@ describe("captureFrameData", () => { }; const sendToTarget = vi.fn(async (target, method) => { if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 }, + }; } if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { @@ -85,7 +157,11 @@ describe("captureFrameData", () => { return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } }; } if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, + }; } throw new Error(`unexpected root ${method}`); }) as CdpRunner["send"], @@ -149,7 +225,11 @@ describe("captureFrameData", () => { if (method === "Accessibility.getFullAXTree") return { nodes: [] }; if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable"); if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, + }; } throw new Error(`unexpected root ${method}`); }) as CdpRunner["send"], @@ -192,7 +272,12 @@ describe("captureFrameData", () => { expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }), ]); expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual( - expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }), + expect.objectContaining({ rect: null, localRect: null, rendered: true }), ); + expect(captured.frameGeometryIssues).toContainEqual({ + status: "unavailable", + source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "child" }, + reason: "snapshot-coordinates-unavailable", + }); }); }); diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index b0f794f..0c894d9 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -5,9 +5,22 @@ // STYLE_COL so capture and visibility policy share one explicit contract. import type { Rect, Viewport } from "@browser-skill/vom"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; -import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; +import { + type FrameProjectionIssue, + type FrameProjectionState, + projectSnapshotRect, + type SnapshotCoordinates, + snapshotCoordinates, + snapshotViewportRect, +} from "../geometry/coordinate-types"; +import { + GeometryContext, + type LayoutMetrics, + snapshotLayoutScale, +} from "../geometry/frame-context"; import type { CdpRunner } from "../shared"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; @@ -71,12 +84,16 @@ export interface CapturedViewModel { /** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */ frameParentIds?: Map; rootFrameId?: string; + /** Failed boundaries and descendants blocked by them; clipping is not a failure. */ + frameGeometryIssues?: FrameProjectionIssue[]; /** Backend node ids belonging to the agent overlay host + its shadow subtree. */ excludedBackendNodeIds: Set; } export interface CaptureViewModelOptions { signal?: AbortSignal; + geometry?: GeometryContext; + target?: CdpTarget; } function captureAbortError(): Error { @@ -147,16 +164,6 @@ interface SnapshotReply { documents?: SnapshotDocument[]; } -interface LayoutMetricsReply { - cssLayoutViewport?: { - clientWidth?: number; - clientHeight?: number; - pageX?: number; - pageY?: number; - }; - layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number }; -} - function isFormControlTag(tag: string): boolean { return tag === "input" || tag === "textarea" || tag === "select"; } @@ -353,9 +360,10 @@ interface ParseDocumentResult { interface FrameContext { frameId?: string; ownerFrameBackendNodeId: number | null; - projection: GeometryProjection | null; - scrollX: number; - scrollY: number; + projection: FrameProjectionState; + target: CdpTarget; + coordinates: SnapshotCoordinates | null; + layoutUnitsPerCssPixel: number | null; } function str(strings: string[], idx: number | undefined): string { @@ -363,15 +371,6 @@ function str(strings: string[], idx: number | undefined): string { return strings[idx] ?? ""; } -function devicePixelRatio(metrics: LayoutMetricsReply): number { - const layoutW = metrics.layoutViewport?.clientWidth ?? 0; - const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0; - if (!layoutW || !cssW) return 1; - const dpr = layoutW / cssW; - if (!Number.isFinite(dpr) || dpr <= 0) return 1; - return dpr >= 1 ? dpr : 1; -} - function sparseIndexMap(sparse: SparseArray | undefined): Map { const out = new Map(); if (!sparse?.index || !sparse.value) return out; @@ -797,13 +796,11 @@ export async function collectOverlayExcludedBackendIds( * * @param doc - the raw DOMSnapshot document object * @param strings - the shared string table for the whole snapshot - * @param dpr - device-pixel-ratio from Page.getLayoutMetrics * @param context - frame coordinate context; output rects are top viewport-relative */ function parseDocumentNodes( doc: SnapshotDocument, strings: string[], - dpr: number, context: FrameContext, ): ParseDocumentResult { const dn = doc.nodes; @@ -898,15 +895,19 @@ function parseDocumentNodes( if (li !== undefined) { const b = dl?.bounds?.[li]; if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) { - localRect = { - x: b[0] / dpr - context.scrollX, - y: b[1] / dpr - context.scrollY, - w: b[2] / dpr, - h: b[3] / dpr, - }; - const bounds = context.projection - ? projectRectToViewport(localRect, context.projection) - : null; + const input = snapshotViewportRect( + b, + { target: context.target, frameId: context.frameId }, + context.coordinates, + ); + if (input) { + const { x, y, width: w, height: h } = input.rect; + localRect = { x, y, w, h }; + } + const bounds = + input && context.projection.status === "available" + ? projectSnapshotRect(input, context.projection.projection) + : null; rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; } paintOrder = dl?.paintOrders?.[li] ?? 0; @@ -917,7 +918,11 @@ function parseDocumentNodes( const visibility = str(strings, styleRow[visibilityCol]) || "visible"; const opacity = str(strings, styleRow[opacityCol]) || "1"; rendered = - localRect !== null && + !!b && + b.length >= 4 && + b.slice(0, 4).every(Number.isFinite) && + b[2] > 0 && + b[3] > 0 && visibility !== "hidden" && visibility !== "collapse" && (Number.parseFloat(opacity) || 0) > 0; @@ -974,21 +979,23 @@ function parseDocumentNodes( } interface ParseFrameDocumentsResult { + frameGeometryIssues: FrameProjectionIssue[]; iframeNodes: CapturedIframeNodes; frameOwnerBackendNodeIds: Map; frameParentIds: Map; excludedBackendNodeIds: Set; } -function parseChildFrameDocuments( +async function parseChildFrameDocuments( documents: SnapshotDocument[], strings: string[], - dpr: number, parentDocIndex: number, - parentNodes: CapturedNode[], parentContext: FrameContext, + geometry: GeometryContext, + signal?: AbortSignal, visited = new Set(), -): ParseFrameDocumentsResult { +): Promise { + const frameGeometryIssues: FrameProjectionIssue[] = []; const iframeNodes: CapturedIframeNodes = new Map(); const frameOwnerBackendNodeIds = new Map(); const frameParentIds = new Map(); @@ -996,34 +1003,100 @@ function parseChildFrameDocuments( const parentDoc = documents[parentDocIndex]; const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex); if (cdi.size === 0) { - return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds }; + return { + iframeNodes, + frameOwnerBackendNodeIds, + frameParentIds, + excludedBackendNodeIds, + frameGeometryIssues, + }; } const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; - const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node])); - for (const [nodeArrayIdx, childDocIndex] of cdi) { - if (visited.has(childDocIndex)) continue; + const children = [...cdi].flatMap(([nodeArrayIdx, childDocIndex]) => { const childDoc = documents[childDocIndex]; - if (!childDoc) continue; const iframeBackendId = parentBackendIds[nodeArrayIdx]; - if (iframeBackendId === undefined) continue; - const iframeNode = parentNodeByBackendId.get(iframeBackendId); - const projection = - parentContext.projection && iframeNode?.localRect - ? childFrameProjection(parentContext.projection, iframeNode.localRect) - : null; + if (visited.has(childDocIndex) || !childDoc || iframeBackendId === undefined) return []; + const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) }; + const coordinates = snapshotCoordinates(childDoc, parentContext.layoutUnitsPerCssPixel); + const parent = parentContext.projection; + const projection: FrameProjectionState = + parent.status === "available" + ? coordinates + ? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId } + : { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" } + : { status: "blocked", source, cause: parent.status === "blocked" ? parent.cause : parent }; + return [ + { + childDocIndex, + childDoc, + iframeBackendId, + source, + coordinates, + projection: projection as FrameProjectionState, + }, + ]; + }); + const parentProjection = + parentContext.projection.status === "available" + ? parentContext.projection.projection.geometry + : null; + if (parentProjection) { + // Measure only direct siblings. Recursion starts after these workers finish, + // so descendants never hold or recursively acquire a measurement slot. + const edge = parentProjection.edges[0]; + const clips = edge + ? [edge.destinationQuad, ...(edge.destinationClips ?? [])] + : parentProjection.sourceClips; + let cursor = 0; + let failure: unknown; + await Promise.all( + Array.from({ length: Math.min(4, children.length) }, async () => { + while (cursor < children.length && !signal?.aborted && !failure) { + const child = children[cursor++]; + if (!child.coordinates) continue; + try { + // Owner quads are target-relative; do not apply the parent transform twice. + child.projection = await geometry.snapshotProjection( + child.source, + child.iframeBackendId, + clips, + parentProjection.topViewport, + ); + } catch (error) { + failure ??= error; + } + } + }), + ); + // Join all active reads, including object cleanup, before propagating abort. + if (failure) throw failure; + } + throwIfAborted(signal); + // Completion order must not change document traversal or result insertion order. + for (const { + childDocIndex, + childDoc, + iframeBackendId, + source, + projection, + coordinates, + } of children) { + throwIfAborted(signal); + if (projection.status !== "available") frameGeometryIssues.push(projection); const childContext: FrameContext = { - frameId: snapshotFrameId(childDoc, strings), + frameId: source.frameId, + target: source.target, ownerFrameBackendNodeId: iframeBackendId, projection, - scrollX: childDoc.scrollOffsetX ?? 0, - scrollY: childDoc.scrollOffsetY ?? 0, + coordinates, + layoutUnitsPerCssPixel: parentContext.layoutUnitsPerCssPixel, }; const nextVisited = new Set(visited); nextVisited.add(childDocIndex); - const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext); + const parsed = parseDocumentNodes(childDoc, strings, childContext); iframeNodes.set(iframeBackendId, parsed.nodes); if (childContext.frameId) { frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId); @@ -1031,15 +1104,16 @@ function parseChildFrameDocuments( } for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); - const nested = parseChildFrameDocuments( + const nested = await parseChildFrameDocuments( documents, strings, - dpr, childDocIndex, - parsed.nodes, childContext, + geometry, + signal, nextVisited, ); + frameGeometryIssues.push(...nested.frameGeometryIssues); for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) { iframeNodes.set(nestedFrameId, nestedNodes); } @@ -1052,7 +1126,13 @@ function parseChildFrameDocuments( for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id); } - return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds }; + return { + iframeNodes, + frameOwnerBackendNodeIds, + frameParentIds, + excludedBackendNodeIds, + frameGeometryIssues, + }; } export async function captureViewModel( @@ -1061,22 +1141,21 @@ export async function captureViewModel( options: CaptureViewModelOptions = {}, ): Promise { throwIfAborted(options.signal); - let metrics: LayoutMetricsReply = {}; + const target = options.target ?? { tabId }; + const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal); + let metrics: LayoutMetrics = {}; try { - metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); + metrics = await geometry.layoutMetrics(target); } catch (error) { if (isAbortError(error)) throw error; console.debug("[bsk capture] layout metrics unavailable", error); } throwIfAborted(options.signal); - const dpr = devicePixelRatio(metrics); - const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const viewport: Viewport = { - width: vpSrc.clientWidth ?? 0, - height: vpSrc.clientHeight ?? 0, + width: metrics.cssLayoutViewport?.clientWidth ?? 0, + height: metrics.cssLayoutViewport?.clientHeight ?? 0, }; - const scrollX = vpSrc.pageX ?? 0; - const scrollY = vpSrc.pageY ?? 0; + const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics); await cdp.send(tabId, "DOMSnapshot.enable", {}); throwIfAborted(options.signal); @@ -1099,22 +1178,49 @@ export async function captureViewModel( }; } + const source = { target, frameId: snapshotFrameId(doc0, strings) }; + const coordinates = snapshotCoordinates(doc0, layoutUnitsPerCssPixel, { + x: metrics.cssLayoutViewport?.pageX, + y: metrics.cssLayoutViewport?.pageY, + }); const topContext: FrameContext = { frameId: snapshotFrameId(doc0, strings), ownerFrameBackendNodeId: null, - projection: { - sourceClips: [], - edges: [], - topViewport: viewport, - }, - scrollX, - scrollY, + target, + projection: + coordinates && + [viewport.width, viewport.height].every((size) => Number.isFinite(size) && size > 0) + ? { + status: "available", + projection: { + source, + geometry: { sourceClips: [], edges: [], topViewport: viewport }, + }, + } + : { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }, + coordinates, + layoutUnitsPerCssPixel, }; - const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext); + const mainParsed = parseDocumentNodes(doc0, strings, topContext); const nodes = mainParsed.nodes; const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds); - const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext); + const ownerIds = new Set(); + for (const document of documents) { + for (const index of document.nodes?.contentDocumentIndex?.index ?? []) { + const id = document.nodes?.backendNodeId?.[index]; + if (id !== undefined) ownerIds.add(id); + } + } + geometry.registerSnapshotOwners(target, ownerIds); + const frameParsed = await parseChildFrameDocuments( + documents, + strings, + 0, + topContext, + geometry, + options.signal, + ); const iframeNodes = frameParsed.iframeNodes; for (const id of frameParsed.excludedBackendNodeIds) { excludedBackendNodeIds.add(id); @@ -1140,6 +1246,10 @@ export async function captureViewModel( viewport, iframeNodes, frameNodes, + frameGeometryIssues: [ + ...(topContext.projection.status === "available" ? [] : [topContext.projection]), + ...frameParsed.frameGeometryIssues, + ], frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds, frameParentIds: frameParsed.frameParentIds, ...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}), diff --git a/apps/extension/src/tools/vom/frame-capture.ts b/apps/extension/src/tools/vom/frame-capture.ts index d643a7b..374945a 100644 --- a/apps/extension/src/tools/vom/frame-capture.ts +++ b/apps/extension/src/tools/vom/frame-capture.ts @@ -1,6 +1,6 @@ import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; -import { resolveFrameProjection } from "../frame-geometry"; import { type GeometryProjection, projectRectToViewport } from "../geometry"; +import { GeometryContext } from "../geometry/frame-context"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { @@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean { return err instanceof Error && err.name === "AbortError"; } -async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise { - if (!cdp.getFrameGraph) return null; - try { - return await cdp.getFrameGraph(tabId); - } catch (err) { - console.debug("[bsk observation] frame graph capture failed", err); - return null; - } -} - function transformFrameNodes( nodes: CapturedNode[], projection: GeometryProjection | null, @@ -51,6 +41,7 @@ async function captureMissingFrameDocuments( tabId: number, graph: CdpFrameGraph, captured: CapturedViewModel, + geometry: GeometryContext, signal?: AbortSignal, ): Promise { if (!captured.frameNodes) captured.frameNodes = new Map(); @@ -62,10 +53,15 @@ async function captureMissingFrameDocuments( try { const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { signal, + geometry, + target: frame.target, }); + if (child.frameGeometryIssues?.length) { + (captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues); + } let projection: GeometryProjection | null = null; try { - projection = await resolveFrameProjection(cdp, graph, frame.frameId); + projection = await geometry.targetProjection(frame.frameId); } catch (err) { if (isAbortError(err)) throw err; console.debug("[bsk observation] child frame geometry projection failed", { @@ -171,10 +167,11 @@ export async function captureFrameData( tabId: number, captured: CapturedViewModel, signal?: AbortSignal, + geometry = new GeometryContext(cdp, tabId, undefined, signal), ): Promise[]> { - const graph = await discoverFrameGraph(cdp, tabId); + const graph = await geometry.graph(); const frames = graph?.frames ?? []; - if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal); + if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal); throwIfAborted(signal); const batches = await captureAxTrees(cdp, tabId, frames, captured, signal); return buildFrameDocuments(graph, batches, captured); diff --git a/evals/browser/cases/regression/snapshot-coordinates/README.md b/evals/browser/cases/regression/snapshot-coordinates/README.md new file mode 100644 index 0000000..4e7f950 --- /dev/null +++ b/evals/browser/cases/regression/snapshot-coordinates/README.md @@ -0,0 +1,44 @@ +# DOMSnapshot layout-unit regression (#195) + +Source: https://github.com/Tencent/BrowserSkill/pull/195 + +DOMSnapshot bounds and document scroll offsets retain Blink layout units. Treating them as CSS +pixels changes both positions and sizes under device scaling/browser zoom, before iframe +projection or viewport clipping even starts. + +The fixture contains a scrolled root, a scrolled same-process iframe, an OOPIF served from the +other loopback hostname, and a nested iframe in each. Owners have borders, padding and CSS +transforms. The geometry test compares the production capture's local and top-level rectangles +against independent DOM border boxes, with at most 2 CSS pixels of layout rounding. It also +checks that each target reads layout metrics only once. No credentials or personal tabs are used. + +Run the geometric assertions with Node 22+ and a local Chrome executable: + +```sh +BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \ + src/tools/__tests__/snapshot-coordinates.browser.test.ts +``` + +The test launches and cleans up its own headless Chrome profiles. It explicitly verifies the +requested browser zoom and OOPIF topology, so unsupported setups fail rather than silently +testing a different configuration. It is skipped in ordinary unit runs unless the executable +is specified. No browser download or new package dependency is required. + +The corpus smoke case checks fixture readiness and observable semantics through the real CLI: + +```sh +BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./target/debug/bsk +``` + +Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes. +Use the geometric test above for the regression's numeric assertions. + +## Separate existing boundary + +The OOPIF root uses non-occupying scrollbars in the unit-conversion regression. Root and +same-process frames retain normal scrollbars. Append `&classic-scrollbars` to the fixture URL +to reproduce the separate existing OOPIF projection issue: `targetProjection()` maps the CSS +layout viewport (which excludes occupying scrollbars) onto the entire owner content quad. +This inflates the projected coordinates when scrollbars occupy space. Main already has that +mapping; fixing it also changes shared live geometry and is outside this snapshot-unit fix. +The test does not relax its numeric tolerance to absorb that error. diff --git a/evals/browser/cases/regression/snapshot-coordinates/chrome.mjs b/evals/browser/cases/regression/snapshot-coordinates/chrome.mjs new file mode 100644 index 0000000..93c7bb3 --- /dev/null +++ b/evals/browser/cases/regression/snapshot-coordinates/chrome.mjs @@ -0,0 +1,99 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// This regression owns its browser/profile. It never attaches to a user's Chrome. +export async function withChrome({ executable, deviceScale, zoom }, run) { + const profile = await mkdtemp(join(tmpdir(), "bsk-snapshot-coordinates-")); + let chrome; + let socket; + const pending = new Map(); + try { + await mkdir(join(profile, "Default")); + await writeFile( + join(profile, "Default", "Preferences"), + JSON.stringify({ + partition: { default_zoom_level: { x: Math.log(zoom) / Math.log(1.2) } }, + }), + ); + chrome = spawn( + executable, + [ + "--headless=new", + "--no-first-run", + "--no-default-browser-check", + "--disable-extensions", + "--site-per-process", + "--remote-debugging-port=0", + "--window-size=1600,1200", + `--force-device-scale-factor=${deviceScale}`, + `--user-data-dir=${profile}`, + "about:blank", + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + const endpoint = await new Promise((resolve, reject) => { + let output = ""; + const timeout = setTimeout( + () => reject(new Error(`Chrome startup timed out: ${output}`)), + 15_000, + ); + const fail = (error) => { + clearTimeout(timeout); + reject(error); + }; + chrome.once("error", fail); + chrome.once("exit", (code) => fail(new Error(`Chrome exited (${code}): ${output}`))); + chrome.stderr.on("data", (chunk) => { + output += chunk; + const match = output.match(/DevTools listening on (ws:\/\/\S+)/); + if (match) { + clearTimeout(timeout); + resolve(match[1]); + } + }); + }); + socket = new WebSocket(endpoint); + await new Promise((resolve, reject) => { + socket.addEventListener("open", resolve, { once: true }); + socket.addEventListener("error", reject, { once: true }); + }); + socket.addEventListener("message", ({ data }) => { + const reply = JSON.parse(data); + const request = pending.get(reply.id); + if (!request) return; + clearTimeout(request.timeout); + pending.delete(reply.id); + if (reply.error) request.reject(new Error(JSON.stringify(reply.error))); + else request.resolve(reply.result); + }); + let sequence = 0; + const send = (method, params = {}, sessionId) => + new Promise((resolve, reject) => { + const id = ++sequence; + const timeout = setTimeout(() => { + pending.delete(id); + reject(new Error(`CDP timed out: ${method}`)); + }, 10_000); + pending.set(id, { resolve, reject, timeout }); + socket.send(JSON.stringify({ id, method, params, sessionId })); + }); + await run(send); + } finally { + for (const request of pending.values()) { + clearTimeout(request.timeout); + request.reject(new Error("regression browser closed")); + } + socket?.close(); + if (chrome && chrome.exitCode === null && chrome.signalCode === null) { + const exited = once(chrome, "exit"); + const timeout = setTimeout(() => chrome.kill("SIGKILL"), 5_000); + chrome.kill(); + await exited; + clearTimeout(timeout); + } + await rm(profile, { recursive: true, force: true }); + } +} diff --git a/evals/browser/cases/regression/snapshot-coordinates/snapshot-coordinates.case.json b/evals/browser/cases/regression/snapshot-coordinates/snapshot-coordinates.case.json new file mode 100644 index 0000000..cb45b05 --- /dev/null +++ b/evals/browser/cases/regression/snapshot-coordinates/snapshot-coordinates.case.json @@ -0,0 +1,33 @@ +{ + "$schema": "../../../schemas/case.schema.json", + "schemaVersion": 1, + "id": "snapshot-coordinates", + "title": "Scrolled and nested frame coordinate fixture", + "suite": "regression", + "tags": ["geometry", "iframe", "scroll"], + "fixture": { "startPath": "/snapshot-coordinates" }, + "prompts": { + "en": "Open {url}, observe the page and its frames, report the GEOMETRY marker, then close the session.", + "zh-CN": "打开 {url}，观察页面及其 iframe，报告 GEOMETRY 标记，然后关闭会话。" + }, + "coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"], + "assertions": { + "site": [ + { + "label": "root fixture scrolled and settled", + "type": "geometry.ready", + "where": { "data.root": true }, + "minCount": 1 + } + ], + "response": [{ "label": "observed the fixture marker", "includes": "GEOMETRY-195" }], + "adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }] + }, + "smoke": { + "steps": [ + { "action": "navigate" }, + { "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } }, + { "action": "observe" } + ] + } +} diff --git a/evals/browser/cases/regression/snapshot-coordinates/snapshot-coordinates.fixture.mjs b/evals/browser/cases/regression/snapshot-coordinates/snapshot-coordinates.fixture.mjs new file mode 100644 index 0000000..d4f2312 --- /dev/null +++ b/evals/browser/cases/regression/snapshot-coordinates/snapshot-coordinates.fixture.mjs @@ -0,0 +1,52 @@ +import { page, withRun } from "../../../lib/fixtures.mjs"; + +export default { + id: "snapshot-coordinates", + routes: ["/snapshot-coordinates", "/snapshot-coordinates/frame", "/snapshot-coordinates/nested"], + render({ pathname, runId, query }) { + const root = pathname === "/snapshot-coordinates"; + const nested = pathname.endsWith("/nested"); + return page({ + title: "Snapshot coordinate regression", + body: ` + + + ${root ? `` : ""} + ${!root && !nested ? `` : ""} + `, + script: ` + history.scrollRestoration = "manual"; + const cross = document.querySelector("#cross"); + if (cross) { + const url = new URL(${JSON.stringify(withRun("/snapshot-coordinates/frame", runId))}, location.href); + url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1"; + // Isolate unit conversion from the pre-existing OOPIF scrollbar-width + // projection issue. ?classic-scrollbars retains that separate reproduction. + if (!new URLSearchParams(location.search).has("classic-scrollbars")) url.searchParams.set("scrollbars", "none"); + cross.src = url.href; + } + window.addEventListener("load", () => { + scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"}); + requestAnimationFrame(() => requestAnimationFrame(() => { + window.geometryReady = true; + document.documentElement.dataset.geometryReady = "true"; + browserEval.send("geometry.ready", { root: ${root} }); + })); + }); + `, + }); + }, +}; diff --git a/evals/browser/tests/eval.test.mjs b/evals/browser/tests/eval.test.mjs index 2bc73de..c0ab22c 100644 --- a/evals/browser/tests/eval.test.mjs +++ b/evals/browser/tests/eval.test.mjs @@ -41,17 +41,22 @@ test("case manifests are discovered, ordered, and grouped into suites", () => { "diagnostics", "mobile-emulation", "generated-form", + "snapshot-coordinates", ], ); - assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 }); + assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { + core: 6, + matrix: 1, + regression: 1, + }); }); test("repository validation links every case to a fixture and valid workflow evidence", () => { assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []); const summary = repositorySummary(cases, fixtureRegistry); - assert.equal(summary.cases, 7); - assert.equal(summary.fixtureModules, 8); - assert.equal(summary.fixtureRoutes, 12); + assert.equal(summary.cases, 8); + assert.equal(summary.fixtureModules, 9); + assert.equal(summary.fixtureRoutes, 15); }); test("manifest validation rejects unknown operations and incomplete workflow steps", () => {