feat(extension): support OOPIF refs with unified frame geometry

apps/extension/src/tools/frame-geometry.ts

@@ -0,0 +1,289 @@
+import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
+import type { RpcError } from "@/transport/types";
+import { nodeContentQuads, scrollNodeIntoView, type ViewportRect } from "./element-geometry";
+import {
+  clipPolygon,
+  type GeometryProjection,
+  type Point,
+  type Polygon,
+  type ProjectiveEdge,
+  parseCdpQuad,
+  polygonBounds,
+  polygonCentroid,
+  projectRegionToViewport,
+  type Quad,
+  type Region,
+  rectPolygon,
+  type Size,
+} from "./geometry";
+import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
+
+export type { Point, Quad, Region } from "./geometry";
+export { parseCdpQuad } from "./geometry";
+
+export type FrameProjection = GeometryProjection;
+
+export interface NodeAddress {
+  target: CdpTarget;
+  backendNodeId: number;
+  frameId?: string;
+}
+
+export interface ResolvedNodeGeometry {
+  topVisibleRegions: Region;
+  topBounds: ViewportRect;
+  actionPoint: Point;
+}
+
+function geometryError(message: string): RpcError {
+  return { code: "cdp_failed", message };
+}
+
+function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
+  return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
+}
+
+function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
+  const seen = new Set<string>();
+  let current = frame;
+  while (current.parentFrameId) {
+    if (seen.has(current.frameId)) return null;
+    seen.add(current.frameId);
+    const parent = byId.get(current.parentFrameId);
+    if (!parent || parent.target.sessionId !== current.target.sessionId) break;
+    current = parent;
+  }
+  return current;
+}
+
+function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
+  const byId = frameMap(graph);
+  const path: CdpFrame[] = [];
+  const seen = new Set<string>();
+  let current = byId.get(frameId);
+  if (!current) return null;
+  while (current.parentFrameId) {
+    if (seen.has(current.frameId)) return null;
+    seen.add(current.frameId);
+    path.push(current);
+    const parent = byId.get(current.parentFrameId);
+    if (!parent) return null;
+    current = parent;
+  }
+  return path;
+}
+
+async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
+  const metrics = await sendToCdpTarget<{
+    cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
+    layoutViewport?: { clientWidth?: number; clientHeight?: number };
+  }>(cdp, target, "Page.getLayoutMetrics", {});
+  const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
+  const width = viewport.clientWidth ?? 0;
+  const height = viewport.clientHeight ?? 0;
+  return width > 0 && height > 0 ? { width, height } : null;
+}
+
+async function ownerContentQuad(
+  cdp: CdpRunner,
+  parent: CdpFrame,
+  ownerBackendNodeId: number,
+): Promise<Quad | null> {
+  const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
+    cdp,
+    parent.target,
+    "DOM.getBoxModel",
+    { backendNodeId: ownerBackendNodeId },
+  );
+  return parseCdpQuad(result.model?.content);
+}
+
+async function sameTargetFrameClips(
+  cdp: CdpRunner,
+  byId: Map<string, CdpFrame>,
+  frame: CdpFrame,
+  targetRoot: CdpFrame,
+): Promise<Polygon[] | null> {
+  const clips: Polygon[] = [];
+  let current = frame;
+  const seen = new Set<string>();
+  while (current.frameId !== targetRoot.frameId) {
+    if (seen.has(current.frameId) || !current.parentFrameId) return null;
+    seen.add(current.frameId);
+    const parent = byId.get(current.parentFrameId);
+    if (!parent || current.ownerBackendNodeId === undefined) return null;
+    const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
+    if (!clip) return null;
+    clips.push(clip);
+    current = parent;
+  }
+  return clips;
+}
+
+export async function resolveFrameProjection(
+  cdp: CdpRunner,
+  graph: CdpFrameGraph,
+  frameId: string,
+): Promise<FrameProjection | null> {
+  const byId = frameMap(graph);
+  const frame = byId.get(frameId);
+  if (!frame) return null;
+  let targetRoot = targetRootFrame(byId, frame);
+  if (!targetRoot) return null;
+  const sourceViewport = await targetViewport(cdp, targetRoot.target);
+  if (!sourceViewport) return null;
+  const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
+  if (!sourceClips) return null;
+
+  const edges: ProjectiveEdge[] = [];
+  while (targetRoot.parentFrameId) {
+    const parent = byId.get(targetRoot.parentFrameId);
+    if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
+    const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
+    if (!destinationQuad) return null;
+    const source =
+      edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
+    if (!source) return null;
+    const parentTargetRoot = targetRootFrame(byId, parent);
+    if (!parentTargetRoot) return null;
+    const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
+    if (!destinationClips) return null;
+    edges.push({ sourceViewport: source, destinationQuad, destinationClips });
+    targetRoot = parentTargetRoot;
+  }
+
+  const topViewport =
+    edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
+  return topViewport ? { sourceViewport, sourceClips, edges, topViewport } : null;
+}
+
+async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
+  if (!cdp.getFrameGraph) return null;
+  try {
+    return await cdp.getFrameGraph(tabId);
+  } catch (error) {
+    console.debug("[bsk frame-geometry] frame graph resolution failed", error);
+    return null;
+  }
+}
+
+async function scrollFrameOwners(
+  cdp: CdpRunner,
+  tabId: number,
+  graph: CdpFrameGraph,
+  frameId: string,
+): Promise<RpcError | null> {
+  const byId = frameMap(graph);
+  const ancestry = frameAncestry(graph, frameId);
+  if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
+  for (const child of [...ancestry].reverse()) {
+    const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
+    if (!parent || child.ownerBackendNodeId === undefined) {
+      return geometryError(`could not resolve frame owner for ${child.frameId}`);
+    }
+    const error = await scrollNodeIntoView(
+      cdpRunnerForTarget(cdp, parent.target),
+      tabId,
+      child.ownerBackendNodeId,
+    );
+    if (error) return error;
+  }
+  return null;
+}
+
+export async function scrollElementAndFramesIntoView(
+  cdp: CdpRunner,
+  tabId: number,
+  target: CdpTarget,
+  backendNodeId: number,
+  frameId?: string,
+): Promise<RpcError | null> {
+  if (frameId) {
+    const graph = await loadFrameGraph(cdp, tabId);
+    if (graph) {
+      const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
+      if (error) return error;
+    } else {
+      return geometryError(`could not resolve frame graph for ${frameId}`);
+    }
+  } else if (target.sessionId) {
+    return geometryError("an OOPIF node address requires frameId");
+  }
+  return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
+}
+
+function largestRegion(regions: Region): Polygon | null {
+  let largest: { polygon: Polygon; area: number } | null = null;
+  for (const polygon of regions) {
+    const bounds = polygonBounds(polygon);
+    if (!bounds) continue;
+    const area = bounds.width * bounds.height;
+    if (!largest || area > largest.area) largest = { polygon, area };
+  }
+  return largest?.polygon ?? null;
+}
+
+export async function resolveNodeGeometry(
+  cdp: CdpRunner,
+  tabId: number,
+  address: NodeAddress,
+  options: { scrollIntoView?: boolean } = {},
+): Promise<ResolvedNodeGeometry | RpcError> {
+  try {
+    if (options.scrollIntoView) {
+      const scrollError = await scrollElementAndFramesIntoView(
+        cdp,
+        tabId,
+        address.target,
+        address.backendNodeId,
+        address.frameId,
+      );
+      if (scrollError) return scrollError;
+    }
+
+    const local = await nodeContentQuads(
+      cdpRunnerForTarget(cdp, address.target),
+      tabId,
+      address.backendNodeId,
+    );
+    if (isRpcError(local)) return local;
+    const localRegion = local
+      .map((quad) => parseCdpQuad(quad))
+      .filter((quad): quad is Quad => quad !== null);
+    if (localRegion.length === 0) return geometryError("DOM returned no valid content regions");
+
+    const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
+    if (address.target.sessionId && !address.frameId) {
+      return geometryError("an OOPIF node address requires frameId");
+    }
+    if (address.frameId && !graph) {
+      return geometryError(`could not resolve frame graph for ${address.frameId}`);
+    }
+
+    let topVisibleRegions: Region;
+    if (address.frameId && graph) {
+      const projection = await resolveFrameProjection(cdp, graph, address.frameId);
+      if (!projection)
+        return geometryError(`could not resolve frame geometry for ${address.frameId}`);
+      topVisibleRegions = projectRegionToViewport(localRegion, projection);
+    } else {
+      const viewport = await targetViewport(cdp, address.target);
+      if (!viewport) return geometryError("could not resolve top viewport geometry");
+      topVisibleRegions = localRegion
+        .map((polygon) =>
+          clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
+        )
+        .filter((polygon) => polygon.length >= 3);
+    }
+
+    const topBounds = polygonBounds(topVisibleRegions.flat());
+    const actionRegion = largestRegion(topVisibleRegions);
+    const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
+    if (!topBounds || !actionPoint) {
+      return { code: "permission_denied", message: "element not visible" };
+    }
+    return { topVisibleRegions, topBounds, actionPoint };
+  } catch (error) {
+    return geometryError(error instanceof Error ? error.message : String(error));
+  }
+}