mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): vom base algorithm and infrastructure
This commit is contained in:
Ljy-0827
@@ -0,0 +1,630 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
function fakeSnapshotReply() {
|
||||
// strings table
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"input",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
// styles columns follow requested computedStyles order: [position, pointer-events]
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 300, 200, 40],
|
||||
],
|
||||
paintOrders: [0, 50, 51],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCdp(snapshot: unknown) {
|
||||
return {
|
||||
send: vi.fn(async (_tab: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureViewModel", () => {
|
||||
it("parses nodes, attrs, rects, paint order and styles", async () => {
|
||||
const { nodes, viewport, iframeNodes } = await captureViewModel(
|
||||
makeCdp(fakeSnapshotReply()),
|
||||
4,
|
||||
);
|
||||
expect(iframeNodes.size).toBe(0);
|
||||
expect(viewport).toEqual({ width: 1000, height: 800 });
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
|
||||
expect(div?.rect).toEqual({ x: 0, y: 0, w: 1000, h: 800 });
|
||||
expect(div?.paintOrder).toBe(50);
|
||||
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
// click-blocker + the "中断" button. Capturing it makes the snapshot detect
|
||||
// the agent's own mask as a blocking layer and occlude the real page, so the
|
||||
// whole subtree must be dropped.
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_NAME,
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [i(OVERLAY_HOST_MARKER_ATTR), -1], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4, 5],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 40],
|
||||
[0, 0, 0, 0],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 750, 80, 40],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 9, 10],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const ids = nodes.map((n) => n.backendNodeId);
|
||||
// Real page nodes survive.
|
||||
expect(ids).toContain(12);
|
||||
// Overlay host + its inlined shadow subtree are gone.
|
||||
expect(ids).not.toContain(13);
|
||||
expect(ids).not.toContain(14);
|
||||
expect(ids).not.toContain(15);
|
||||
});
|
||||
|
||||
it("returns excludedBackendNodeIds for the overlay host subtree", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_NAME,
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [i(OVERLAY_HOST_MARKER_ATTR), -1], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4, 5],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 40],
|
||||
[0, 0, 0, 0],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 750, 80, 40],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 9, 10],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by tag name only", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
OVERLAY_HOST_NAME,
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"auto",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by marker attribute only", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"auto",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], [i(OVERLAY_HOST_MARKER_ATTR), -1]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
|
||||
it("parses the computed cursor from the third style column", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"position",
|
||||
"static",
|
||||
"fixed",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
// styles columns: [position, pointer-events, cursor]
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 40, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 2000, 3200],
|
||||
[0, 0, 2000, 1600],
|
||||
],
|
||||
paintOrders: [0, 50],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||
// Simulates a button with text "登录" and an <a> with text "忘记密码".
|
||||
// nodeValue is a parallel array: element nodes = -1, text nodes = string index.
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"a",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"登录",
|
||||
"忘记密码",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 1, 4],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("#text"), i("a"), i("#text")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [], [], []],
|
||||
nodeValue: [-1, -1, -1, i("登录"), -1, i("忘记密码")],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 4],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[100, 300, 200, 60],
|
||||
[100, 380, 120, 36],
|
||||
],
|
||||
paintOrders: [0, 10, 11],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const btn = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(btn?.textContent).toBe("登录");
|
||||
const link = nodes.find((n) => n.backendNodeId === 14);
|
||||
expect(link?.textContent).toBe("忘记密码");
|
||||
// Nodes without text children should have no textContent
|
||||
const body = nodes.find((n) => n.backendNodeId === 11);
|
||||
expect(body?.textContent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"src",
|
||||
"https://passport.jd.com/login",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("src"), i("https://passport.jd.com/login")]],
|
||||
// Sparse: node at array-index 3 (the iframe, backendNodeId=13) → documents[1]
|
||||
contentDocumentIndex: { index: [3], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 300, 800, 400],
|
||||
],
|
||||
paintOrders: [0, 50, 51],
|
||||
},
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [100, 101],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 200, 40]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 2000, 3200],
|
||||
[0, 400, 2000, 1600],
|
||||
],
|
||||
paintOrders: [0, 50],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
||||
});
|
||||
|
||||
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") return { nodeId: 99 };
|
||||
if (method === "DOM.describeNode") {
|
||||
return {
|
||||
node: {
|
||||
backendNodeId: 200,
|
||||
shadowRoots: [
|
||||
{ backendNodeId: 201, children: [{ backendNodeId: 202, children: [] }] },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const excluded = await collectOverlayExcludedBackendIds(cdp, 4);
|
||||
expect(excluded).toEqual(new Set([200, 201, 202]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,356 @@
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); we request exactly three computed styles so the
|
||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
rect: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
textContent?: string;
|
||||
}
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
interface LayoutMetricsReply {
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
interface CdpDomNode {
|
||||
backendNodeId?: number;
|
||||
children?: CdpDomNode[];
|
||||
shadowRoots?: CdpDomNode[];
|
||||
}
|
||||
|
||||
interface ParseDocumentResult {
|
||||
nodes: CapturedNode[];
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
||||
if (!layoutW || !cssW) return 1;
|
||||
const dpr = layoutW / cssW;
|
||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
out.add(node.backendNodeId);
|
||||
}
|
||||
for (const child of node.children ?? []) {
|
||||
collectBackendIdsFromDomNode(child, out);
|
||||
}
|
||||
for (const shadow of node.shadowRoots ?? []) {
|
||||
collectBackendIdsFromDomNode(shadow, out);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
|
||||
* collect every backendNodeId in its pierced subtree (open shadow included).
|
||||
*/
|
||||
export async function collectOverlayExcludedBackendIds(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<Set<number>> {
|
||||
const excluded = new Set<number>();
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
pierce: true,
|
||||
});
|
||||
const rootNodeId = doc.root?.nodeId;
|
||||
if (typeof rootNodeId !== "number") return excluded;
|
||||
|
||||
const found = await cdp.send<{ nodeId?: number }>(tabId, "DOM.querySelector", {
|
||||
nodeId: rootNodeId,
|
||||
selector: OVERLAY_HOST_SELECTOR,
|
||||
});
|
||||
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
||||
|
||||
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
||||
nodeId: found.nodeId,
|
||||
depth: -1,
|
||||
pierce: true,
|
||||
});
|
||||
collectBackendIdsFromDomNode(described.node, excluded);
|
||||
} catch (err) {
|
||||
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
||||
}
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param scrollX - horizontal scroll offset to subtract (CSS px)
|
||||
* @param scrollY - vertical scroll offset to subtract (CSS px)
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
scrollX: number,
|
||||
scrollY: number,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
rect = {
|
||||
x: b[0] / dpr - scrollX,
|
||||
y: b[1] / dpr - scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
textContent,
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
const scrollX = vpSrc.pageX ?? 0;
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
const doc0 = documents[0];
|
||||
if (!doc0?.nodes?.backendNodeId) {
|
||||
return { nodes: [], viewport, iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
// Build a map from node-array-index → sub-document-index using the sparse
|
||||
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
if (documents.length > 1) {
|
||||
const cdi = doc0.nodes?.contentDocumentIndex;
|
||||
if (cdi?.index && cdi.value) {
|
||||
for (let k = 0; k < cdi.index.length; k++) {
|
||||
const nodeArrayIdx = cdi.index[k];
|
||||
const docIdx = cdi.value[k];
|
||||
const subDoc = documents[docIdx];
|
||||
if (!subDoc) continue;
|
||||
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
|
||||
if (iframeBid === undefined) continue;
|
||||
// Sub-documents have their own scroll offsets; bounds are relative
|
||||
// to the iframe's own coordinate space (not the main viewport).
|
||||
const subScrollX = subDoc.scrollOffsetX ?? 0;
|
||||
const subScrollY = subDoc.scrollOffsetY ?? 0;
|
||||
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
|
||||
iframeNodes.set(iframeBid, subParsed.nodes);
|
||||
for (const id of subParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, viewport, iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
Reference in New Issue
Block a user