diff --git a/apps/extension/src/tools/__tests__/element-geometry.test.ts b/apps/extension/src/tools/__tests__/element-geometry.test.ts index 8e4b398..b5fcfbe 100644 --- a/apps/extension/src/tools/__tests__/element-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/element-geometry.test.ts @@ -1,25 +1,28 @@ import { describe, expect, it } from "vitest"; -import { nodeBoundingRect, quadBoundingRect } from "../element-geometry"; +import { nodeContentRegion } from "../element-geometry"; import type { CdpRunner } from "../shared"; -describe("quadBoundingRect", () => { - it("computes axis-aligned bounds from an 8-double quad", () => { - expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({ - x: 10, - y: 20, - width: 100, - height: 40, - }); +describe("nodeContentRegion", () => { + it("rejects invalid content quads before using the box model fallback", async () => { + const send = async (_tabId: number, method: string) => { + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] }; + if (method === "DOM.getBoxModel") { + return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } }; + } + throw new Error(`unexpected CDP call ${method}`); + }; + + await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([ + [ + { x: 5, y: 10 }, + { x: 25, y: 10 }, + { x: 25, y: 30 }, + { x: 5, y: 30 }, + ], + ]); }); - it("returns null for degenerate quads", () => { - expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull(); - expect(quadBoundingRect([1, 2, 3])).toBeNull(); - }); -}); - -describe("nodeBoundingRect", () => { - it("unions every visible content fragment", async () => { + it("preserves every visible content fragment", async () => { const send = async () => ({ quads: [ [10, 20, 60, 20, 60, 40, 10, 40], @@ -30,12 +33,20 @@ describe("nodeBoundingRect", () => { send: send as CdpRunner["send"], }; - await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({ - x: 10, - y: 20, - width: 80, - height: 60, - }); + await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ + [ + { x: 10, y: 20 }, + { x: 60, y: 20 }, + { x: 60, y: 40 }, + { x: 10, y: 40 }, + ], + [ + { x: 15, y: 50 }, + { x: 90, y: 50 }, + { x: 90, y: 80 }, + { x: 15, y: 80 }, + ], + ]); }); it("falls back to visible descendant bounds for zero-size containers", async () => { @@ -57,12 +68,14 @@ describe("nodeBoundingRect", () => { }; const cdp = { send: send as CdpRunner["send"] }; - await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({ - x: 242, - y: 468, - width: 769, - height: 180, - }); + await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ + [ + { x: 242, y: 468 }, + { x: 1011, y: 468 }, + { x: 1011, y: 648 }, + { x: 242, y: 648 }, + ], + ]); expect(calls.map((c) => c.method)).toEqual([ "DOM.getContentQuads", "DOM.getBoxModel", diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index c0eac24..f4d13d2 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -1,15 +1,27 @@ import { describe, expect, it, vi } from "vitest"; -import { type Quad, resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; +import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; import { clipPolygon, + polygonArea, polygonBounds, projectAndClipRegion, projectUnitPoint, + type Quad, rectPolygon, + regionBounds, } from "../geometry"; import type { CdpRunner } from "../shared"; describe("frame geometry projection", () => { + it("keeps region bounds separate from polygon area", () => { + const region = [ + rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), + rectPolygon({ x: 20, y: 5, w: 5, h: 5 }), + ]; + expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 }); + expect(polygonArea(region[0])).toBe(100); + }); + it("maps through the iframe content box instead of its border box", () => { const contentQuad: Quad = [ { x: 204, y: 306 }, diff --git a/apps/extension/src/tools/element-geometry.ts b/apps/extension/src/tools/element-geometry.ts index aa4c8b8..ccfd344 100644 --- a/apps/extension/src/tools/element-geometry.ts +++ b/apps/extension/src/tools/element-geometry.ts @@ -1,9 +1,15 @@ -// Shared CDP helpers for scrolling nodes into view and computing -// viewport-space bounding rectangles. Used by interaction tools (click -// centre) and observation tools (element screenshot clip). +// Resolve and scroll nodes inside one CDP target. Cross-frame projection +// belongs to frame-geometry.ts. import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; +import { + parseCdpQuad, + polygonBounds, + type Region, + rectPolygon, + type ViewportRect, +} from "./geometry"; import { type CdpRunner, isRpcError } from "./shared"; const ELEMENT_NOT_VISIBLE_MESSAGE = @@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError { return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE); } -/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */ -export interface ViewportRect { - x: number; - y: number; - width: number; - height: number; -} - -/** - * Compute the axis-aligned bounding box of an 8-double CDP content - * quad / box-model polygon. Exported for unit tests. - */ -export function quadBoundingRect(quad: number[]): ViewportRect | null { - if (quad.length !== 8) return null; - const xs = [quad[0], quad[2], quad[4], quad[6]]; - const ys = [quad[1], quad[3], quad[5], quad[7]]; - const minX = Math.min(...xs); - const maxX = Math.max(...xs); - const minY = Math.min(...ys); - const maxY = Math.max(...ys); - const width = maxX - minX; - const height = maxY - minY; - if (width <= 0 || height <= 0) return null; - return { x: minX, y: minY, width, height }; -} - -function rectToQuad(rect: ViewportRect): number[] { - return [ - rect.x, - rect.y, - rect.x + rect.width, - rect.y, - rect.x + rect.width, - rect.y + rect.height, - rect.x, - rect.y + rect.height, - ]; -} - /** * Resolve `backendNodeId` → CDP `objectId` so we can invoke * `Runtime.callFunctionOn` against the live JS object. @@ -112,52 +79,24 @@ export async function scrollNodeIntoView( } } -/** - * Viewport-space bounding box for `backendNodeId`. Prefers - * `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of - * visible descendant client rects for zero-size overflow containers. - */ -export async function nodeBoundingRect( +function validCdpRegion(quads: number[][] | undefined): Region { + return (quads ?? []).flatMap((raw) => { + const polygon = parseCdpQuad(raw); + return polygon && polygonBounds(polygon) ? [polygon] : []; + }); +} + +async function resolveVisibleContentRegion( cdp: CdpRunner, tabId: number, backendNodeId: number, -): Promise { - const polygons = await visibleContentPolygons(cdp, tabId, backendNodeId); - if (isRpcError(polygons)) return polygons; - const bounds = polygons.flatMap((polygon) => quadBoundingRect(polygon) ?? []); - const x = Math.min(...bounds.map((rect) => rect.x)); - const y = Math.min(...bounds.map((rect) => rect.y)); - const right = Math.max(...bounds.map((rect) => rect.x + rect.width)); - const bottom = Math.max(...bounds.map((rect) => rect.y + rect.height)); - return { x, y, width: right - x, height: bottom - y }; -} - -/** - * Compute the centroid of a CDP content quad. Quads are reported as - * 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4). - */ -export function quadCentre(quad: number[]): { x: number; y: number } { - const x = (quad[0] + quad[4]) / 2; - const y = (quad[1] + quad[5]) / 2; - return { x, y }; -} - -/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */ -export function boxCentre(box: number[]): { x: number; y: number } { - return quadCentre(box); -} - -async function visibleContentPolygons( - cdp: CdpRunner, - tabId: number, - backendNodeId: number, -): Promise { +): Promise { try { const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", { backendNodeId, }); - const visible = quads.quads?.filter((quad) => quad.length === 8 && quadBoundingRect(quad)); - if (visible && visible.length > 0) return visible; + const visible = validCdpRegion(quads.quads); + if (visible.length > 0) return visible; } catch (err) { console.debug("[bsk element-geometry] getContentQuads failed", err); } @@ -166,25 +105,32 @@ async function visibleContentPolygons( backendNodeId, }); const content = box.model?.content; - if (content && content.length === 8 && quadBoundingRect(content)) return [content]; + const fallback = validCdpRegion(content ? [content] : undefined); + if (fallback.length > 0) return fallback; } catch (err) { console.debug("[bsk element-geometry] getBoxModel failed", err); } const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId); - if (descendantRect) return [rectToQuad(descendantRect)]; + if (descendantRect) { + return [ + rectPolygon({ + x: descendantRect.x, + y: descendantRect.y, + w: descendantRect.width, + h: descendantRect.height, + }), + ]; + } return elementNotVisibleError(); } -/** - * Return a live content quad for a backend node in the target's own viewport. - * Frame-aware callers can project this quad through the owning frame chain. - */ -export async function nodeContentQuads( +/** Resolve every visible content region in the node's target-local viewport. */ +export async function nodeContentRegion( cdp: CdpRunner, tabId: number, backendNodeId: number, -): Promise { - return visibleContentPolygons(cdp, tabId, backendNodeId); +): Promise { + return resolveVisibleContentRegion(cdp, tabId, backendNodeId); } async function descendantBoundingRect( @@ -257,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null { if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null; return { x, y, width, height }; } - -/** - * Get the click point for `backendNodeId`. Prefers - * `DOM.getContentQuads` (richer for rotated / transformed elements); - * falls back to `DOM.getBoxModel`, then visible descendant bounds. - * Returns `permission_denied` with "element not visible" when every - * geometry path comes back empty / fails. - */ -export async function nodeCentre( - cdp: CdpRunner, - tabId: number, - backendNodeId: number, -): Promise<{ x: number; y: number } | RpcError> { - const polygons = await visibleContentPolygons(cdp, tabId, backendNodeId); - if (isRpcError(polygons)) return polygons; - return quadCentre(polygons[0]); -} diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index 65b2b7f..dcfb5e1 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -1,6 +1,6 @@ import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; import type { RpcError } from "@/transport/types"; -import { nodeContentQuads, scrollNodeIntoView, type ViewportRect } from "./element-geometry"; +import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { clipPolygon, type GeometryProjection, @@ -8,21 +8,18 @@ import { type Polygon, type ProjectiveEdge, parseCdpQuad, - polygonBounds, + polygonArea, polygonCentroid, projectRegionToViewport, type Quad, type Region, rectPolygon, + regionBounds, type Size, + type ViewportRect, } from "./geometry"; import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; -export type { Point, Quad, Region } from "./geometry"; -export { parseCdpQuad } from "./geometry"; - -export type FrameProjection = GeometryProjection; - export interface NodeAddress { target: CdpTarget; backendNodeId: number; @@ -124,7 +121,7 @@ export async function resolveFrameProjection( cdp: CdpRunner, graph: CdpFrameGraph, frameId: string, -): Promise { +): Promise { const byId = frameMap(graph); const frame = byId.get(frameId); if (!frame) return null; @@ -154,7 +151,7 @@ export async function resolveFrameProjection( const topViewport = edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - return topViewport ? { sourceViewport, sourceClips, edges, topViewport } : null; + return topViewport ? { sourceClips, edges, topViewport } : null; } async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise { @@ -215,9 +212,8 @@ export async function scrollElementAndFramesIntoView( function largestRegion(regions: Region): Polygon | null { let largest: { polygon: Polygon; area: number } | null = null; for (const polygon of regions) { - const bounds = polygonBounds(polygon); - if (!bounds) continue; - const area = bounds.width * bounds.height; + const area = polygonArea(polygon); + if (area <= 0) continue; if (!largest || area > largest.area) largest = { polygon, area }; } return largest?.polygon ?? null; @@ -241,16 +237,12 @@ export async function resolveNodeGeometry( if (scrollError) return scrollError; } - const local = await nodeContentQuads( + const localRegion = await nodeContentRegion( cdpRunnerForTarget(cdp, address.target), tabId, address.backendNodeId, ); - if (isRpcError(local)) return local; - const localRegion = local - .map((quad) => parseCdpQuad(quad)) - .filter((quad): quad is Quad => quad !== null); - if (localRegion.length === 0) return geometryError("DOM returned no valid content regions"); + if (isRpcError(localRegion)) return localRegion; const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; if (address.target.sessionId && !address.frameId) { @@ -276,7 +268,7 @@ export async function resolveNodeGeometry( .filter((polygon) => polygon.length >= 3); } - const topBounds = polygonBounds(topVisibleRegions.flat()); + const topBounds = regionBounds(topVisibleRegions); const actionRegion = largestRegion(topVisibleRegions); const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null; if (!topBounds || !actionPoint) { diff --git a/apps/extension/src/tools/geometry.ts b/apps/extension/src/tools/geometry.ts index b8de43d..fea8782 100644 --- a/apps/extension/src/tools/geometry.ts +++ b/apps/extension/src/tools/geometry.ts @@ -1,5 +1,3 @@ -import type { ViewportRect } from "./element-geometry"; - export interface Point { x: number; y: number; @@ -14,6 +12,13 @@ export interface Size { height: number; } +export interface ViewportRect { + x: number; + y: number; + width: number; + height: number; +} + export interface ProjectiveEdge { sourceViewport: Size; destinationQuad: Quad; @@ -21,7 +26,6 @@ export interface ProjectiveEdge { } export interface GeometryProjection { - sourceViewport: Size; sourceClips: Polygon[]; edges: ProjectiveEdge[]; topViewport: Size; @@ -175,6 +179,14 @@ export function polygonBounds(points: Polygon): ViewportRect | null { return { x, y, width: right - x, height: bottom - y }; } +export function regionBounds(region: Region): ViewportRect | null { + return polygonBounds(region.flat()); +} + +export function polygonArea(points: Polygon): number { + return Math.abs(polygonSignedArea(points)); +} + export function polygonCentroid(points: Polygon): Point | null { if (points.length < 3) return null; let twiceArea = 0; @@ -231,7 +243,7 @@ export function projectRectToViewport( projection: GeometryProjection, ): ViewportRect | null { if (!rect) return null; - return polygonBounds(projectRegionToViewport([rectPolygon(rect)], projection).flat()); + return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); } export function childFrameProjection( @@ -240,7 +252,6 @@ export function childFrameProjection( ): GeometryProjection { const destinationQuad = rectPolygon(ownerRectInParent); return { - sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h }, sourceClips: [], edges: [ { diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index cf78710..182cabd 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -29,7 +29,7 @@ import type { SelectResult, } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; -import { backendNodeToObject, boxCentre, nodeCentre, quadCentre } from "./element-geometry"; +import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; import { @@ -1107,9 +1107,6 @@ export async function handleSelect( export const __testing__ = { DEFAULT_TIMEOUT_MS, - quadCentre, - boxCentre, resolveBackendNode, - nodeCentre, isFillable, }; diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 7bb1221..806abdf 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -1057,7 +1057,6 @@ export async function captureViewModel( frameId: snapshotFrameId(doc0, strings), ownerFrameBackendNodeId: null, projection: { - sourceViewport: viewport, sourceClips: [], edges: [], topViewport: viewport, diff --git a/apps/extension/src/tools/vom/frame-capture.ts b/apps/extension/src/tools/vom/frame-capture.ts index e352b89..aef97ed 100644 --- a/apps/extension/src/tools/vom/frame-capture.ts +++ b/apps/extension/src/tools/vom/frame-capture.ts @@ -1,6 +1,6 @@ import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; -import { type FrameProjection, resolveFrameProjection } from "../frame-geometry"; -import { projectRectToViewport } from "../geometry"; +import { resolveFrameProjection } from "../frame-geometry"; +import { type GeometryProjection, projectRectToViewport } from "../geometry"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { @@ -31,7 +31,10 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise ({ ...node, rect: (() => {