figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extension): support OOPIF refs with unified frame geometry

Browse Source
This commit is contained in:
Ljy-0827
2026-08-18 19:28:16 +08:00
committed by Zhang GH
parent 22019cdf07
commit 42403e55f2
32 changed files with 2418 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/vom/__tests__/capture.test.ts
+4 -4
View File
@@ -698,7 +698,7 @@ describe("captureViewModel", () => {
const { nodes } = await captureViewModel(cdp, 4);
const div = nodes.find((n) => n.backendNodeId === 12);
expect(div?.localRect?.y).toBe(-200);
expect(div?.rect).toMatchObject({ y: -200, h: 800 });
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
@@ -919,7 +919,7 @@ describe("captureViewModel", () => {
expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toBeNull();
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
@@ -1091,10 +1091,10 @@ describe("captureViewModel", () => {
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toBeNull();
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
expect(input?.ownerFrameBackendNodeId).toBe(23);
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
expect(input?.rect).toBeNull();
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
});
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
apps/extension/src/tools/vom/__tests__/frame-capture.test.ts
+109 -37
View File
@@ -3,61 +3,133 @@ import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function node(frameId: string, backendNodeId: number): CapturedNode {
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "div",
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: null,
localRect: { x: 0, y: 0, w: 20, h: 20 },
paintOrder: 0,
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("captures AX trees for every same-target frame found in the DOM snapshot", async () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [node("main", 1)],
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map([[10, [node("child", 101)]]]),
frameNodes: new Map([
["main", [node("main", 1)]],
["child", [node("child", 101)]],
]),
frameOwnerBackendNodeIds: new Map([["child", 10]]),
frameParentIds: new Map([["child", "main"]]),
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") {
const frameId = (params as { frameId?: string })?.frameId ?? "main";
return {
nodes: [
{
nodeId: `${frameId}-root`,
frameId,
backendDOMNodeId: frameId === "main" ? 1 : 101,
},
],
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const documents = await captureFrameData({ send } as CdpRunner, 4, captured);
const trees = await captureFrameData(cdp, 4, captured);
expect(send).toHaveBeenCalledWith(4, "Accessibility.enable", {});
expect(documents.map((document) => document.frameId)).toEqual(["main", "child"]);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "main",
ownerBackendNodeId: 10,
axNodes: [{ nodeId: "child-root", frameId: "child", backendDOMNodeId: 101 }],
});
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
});
apps/extension/src/tools/vom/__tests__/frame-document.test.ts
+74 -10
View File
@@ -1,4 +1,5 @@
import { describe, expect, it } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
@@ -9,8 +10,7 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
frameId,
tag: "div",
attrs: {},
rect: null,
localRect: { x: 0, y: 0, w: 20, h: 20 },
rect: { x: 0, y: 0, w: 20, h: 20 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
@@ -30,9 +30,16 @@ function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
],
};
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
{
frameId: "main",
frame: graph.frames[0],
nodes: [
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
{
@@ -48,6 +55,7 @@ describe("buildFrameDocuments", () => {
];
const documents = buildFrameDocuments(
graph,
batches,
captured(
new Map([
@@ -64,14 +72,28 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
});
it("deduplicates overlapping AX results using the strongest ownership", () => {
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
],
};
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
{ frameId: "main", nodes: [{ ...childNode, frameId: "child" }] },
{ frameId: "child", nodes: [childNode] },
{
frame: graph.frames[0],
nodes: [{ ...childNode, frameId: "child" }],
},
{
frame: graph.frames[1],
nodes: [childNode],
},
];
const documents = buildFrameDocuments(
graph,
batches,
captured(
new Map([
@@ -85,22 +107,64 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
});
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
target: { tabId: 4, sessionId: "right-session" },
},
],
};
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
frame,
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
}));
const documents = buildFrameDocuments(
graph,
batches,
captured(
new Map([
["main", []],
["left", [domNode("left", 7)]],
["right", [domNode("right", 7)]],
]),
),
);
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
"left-node",
);
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
"right-node",
);
});
it("uses captured frame parent identity instead of guessing from backend ids", () => {
const frameNodes = new Map([
["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]],
["right", [domNode("right", 8)]],
["right", [domNode("right", 7)]],
["child", [domNode("child", 9)]],
]);
const capture = captured(frameNodes);
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
capture.frameParentIds = new Map([["child", "left"]]);
const batches = ["main", "left", "right", "child"].map((frameId) => ({
frameId,
const frames = ["main", "left", "right", "child"].map((frameId) => ({
frame: { frameId, target: { tabId: 4 } },
nodes: [],
}));
const documents = buildFrameDocuments(batches, capture);
const documents = buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left",
apps/extension/src/tools/vom/capture.ts
+24 -4
View File
@@ -7,6 +7,7 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
@@ -22,9 +23,9 @@ export interface CapturedNode {
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport geometry when this node belongs to the root document. */
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Owning document viewport-relative CSS px. */
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
@@ -317,6 +318,7 @@ interface ParseDocumentResult {
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: GeometryProjection | null;
scrollX: number;
scrollY: number;
}
@@ -869,7 +871,10 @@ function parseDocumentNodes(
w: b[2] / dpr,
h: b[3] / dpr,
};
rect = context.ownerFrameBackendNodeId === null ? localRect : null;
const bounds = context.projection
? projectRectToViewport(localRect, context.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
@@ -939,6 +944,7 @@ function parseChildFrameDocuments(
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
@@ -953,6 +959,7 @@ function parseChildFrameDocuments(
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
@@ -960,10 +967,16 @@ function parseChildFrameDocuments(
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
const projection =
parentContext.projection && iframeNode?.localRect
? childFrameProjection(parentContext.projection, iframeNode.localRect)
: null;
const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings),
ownerFrameBackendNodeId: iframeBackendId,
projection,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
@@ -982,6 +995,7 @@ function parseChildFrameDocuments(
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
nextVisited,
);
@@ -1042,6 +1056,12 @@ export async function captureViewModel(
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
projection: {
sourceViewport: viewport,
sourceClips: [],
edges: [],
topViewport: viewport,
},
scrollX,
scrollY,
};
@@ -1049,7 +1069,7 @@ export async function captureViewModel(
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
apps/extension/src/tools/vom/frame-capture.ts
+120 -19
View File
@@ -1,5 +1,8 @@
import type { CdpRunner } from "../shared";
import type { CapturedViewModel } from "./capture";
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { type FrameProjection, resolveFrameProjection } from "../frame-geometry";
import { projectRectToViewport } from "../geometry";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
@@ -18,39 +21,133 @@ function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
function capturedFrameIds(captured: CapturedViewModel): string[] {
const frameIds = new Set<string>();
if (captured.rootFrameId) frameIds.add(captured.rootFrameId);
for (const frameId of captured.frameNodes?.keys() ?? []) frameIds.add(frameId);
if (frameIds.size === 0) frameIds.add("root");
return [...frameIds];
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes(nodes: CapturedNode[], projection: FrameProjection): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})(),
}));
}
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
if (frame.ownerBackendNodeId === undefined) return undefined;
const parentNodes = frame.parentFrameId
? captured.frameNodes?.get(frame.parentFrameId)
: captured.nodes;
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
const owner = frameOwnerNode(captured, frame);
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
});
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
if (!projection) continue;
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
const frameIds = capturedFrameIds(captured);
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
await cdp.send(tabId, "Accessibility.enable", {});
const trees = await Promise.all(
frameIds.map(async (frameId): Promise<FrameAxBatch<T>> => {
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
const result = await cdp.send<{ nodes?: T[] }>(
tabId,
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frameId === "root" ? {} : { frameId },
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return { frameId, nodes: result.nodes ?? [] };
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", { frameId, err });
return { frameId, nodes: [] };
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
@@ -70,6 +167,10 @@ export async function captureFrameData<T extends FrameAxNode>(
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<CapturedFrameDocument<T>[]> {
const batches = await captureAxTrees<T>(cdp, tabId, captured, signal);
return buildFrameDocuments(batches, captured);
const graph = await discoverFrameGraph(cdp, tabId);
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
}
apps/extension/src/tools/vom/frame-document.ts
+69 -41
View File
@@ -1,3 +1,9 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture";
export interface FrameOwnedAxNode {
@@ -9,14 +15,11 @@ export interface FrameOwnedAxNode {
}
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
frameId: string;
frame: CdpFrame;
nodes: T[];
}
export interface FrameDocument<T extends FrameOwnedAxNode> {
frameId: string;
parentFrameId?: string;
ownerBackendNodeId?: number;
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string;
axNodes: T[];
domNodes: CapturedNode[];
@@ -29,30 +32,55 @@ interface Ownership {
interface OwnedCandidate<T> {
node: T;
target: CdpTarget;
ownership: Ownership;
}
function frameIds<T extends FrameOwnedAxNode>(
function targetNodeKey(target: CdpTarget, nodeId: string): string {
return `${cdpTargetKey(target)}:${nodeId}`;
}
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
return `${cdpTargetKey(target)}:${backendNodeId}`;
}
function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): string[] {
const ids = new Set<string>();
if (captured.rootFrameId) ids.add(captured.rootFrameId);
for (const frameId of captured.frameNodes?.keys() ?? []) ids.add(frameId);
for (const batch of batches) ids.add(batch.frameId);
return [...ids];
): CdpFrame[] {
const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
}
for (const frameId of captured.frameNodes?.keys() ?? []) {
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => {
const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
return {
...frame,
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
...(parentFrameId ? { parentFrameId } : {}),
};
});
}
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): FrameDocument<T>[] {
const frames = frameIds(batches, captured);
const knownFrames = new Set(frames);
const backendOwner = new Map<number, string>();
for (const frameId of frames) {
for (const node of captured.frameNodes?.get(frameId) ?? []) {
backendOwner.set(node.backendNodeId, frameId);
const frames = frameList(graph, batches, captured);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const backendOwner = new Map<string, string>();
for (const frame of frames) {
for (const node of captured.frameNodes?.get(frame.frameId) ?? []) {
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
}
}
@@ -64,15 +92,17 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && knownFrames.has(node.frameId)) {
if (node.frameId && frameById.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const owner = backendOwner.get(node.backendDOMNodeId);
if (owner) {
const ownership = { frameId: owner, strength: 3 };
const frameId = backendOwner.get(
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
);
if (frameId) {
const ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
@@ -91,28 +121,32 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
return ownership;
}
}
const ownership = { frameId: batch.frameId, strength: 1 };
const ownership = { frameId: batch.frame.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
for (const node of batch.nodes) {
const ownership = resolveOwnership(node);
const existing = candidates.get(node.nodeId);
const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) {
candidates.set(node.nodeId, { node, ownership });
candidates.set(key, { node, target: batch.frame.target, ownership });
}
}
}
const ownershipByNodeId = new Map(
[...candidates].map(([nodeId, candidate]) => [nodeId, candidate.ownership.frameId]),
const ownershipByTargetNode = new Map(
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
);
const axNodesByFrame = new Map<string, T[]>();
for (const { node, ownership } of candidates.values()) {
const parentFrameId = node.parentId ? ownershipByNodeId.get(node.parentId) : undefined;
for (const candidate of candidates.values()) {
const { node, target, ownership } = candidate;
const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
: undefined;
const childIds = node.childIds?.filter(
(childId) => ownershipByNodeId.get(childId) === ownership.frameId,
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
);
const ownedNode = {
...node,
@@ -127,16 +161,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
axNodesByFrame.set(ownership.frameId, nodes);
}
return frames.map((frameId) => ({
frameId,
...(captured.frameParentIds?.get(frameId)
? { parentFrameId: captured.frameParentIds.get(frameId) }
: {}),
...(captured.frameOwnerBackendNodeIds?.get(frameId) !== undefined
? { ownerBackendNodeId: captured.frameOwnerBackendNodeIds.get(frameId) }
: {}),
contextScopeId: frameId,
axNodes: axNodesByFrame.get(frameId) ?? [],
domNodes: captured.frameNodes?.get(frameId) ?? [],
return frames.map((frame) => ({
...frame,
contextScopeId: frame.frameId,
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
domNodes: captured.frameNodes?.get(frame.frameId) ?? [],
}));
}