Compare commits

..
Author SHA1 Message Date
Ljy-0827 d7b2c6cccd fix(observation): validate document consistency during capture 2026-09-08 19:22:46 +08:00
Ljy-0827 1f6faa7a0a refactor(observation): organize capture around document facts 2026-09-08 19:16:47 +08:00
Ljy-0827 27dba3c57f fix(geometry): unify frame projection and screenshot coordinates 2026-09-08 17:42:27 +08:00
Zhang GH e8137b42cc Merge pull request #194 from Tencent/fix/dsh-runtime-peer-dependency
fix(dsh-plugin): remove obsolete client runtime host dependency
2026-09-08 17:34:33 +08:00
Zhang GH cb5dc148d5 fix(dsh-plugin): remove obsolete client runtime host dependency
Keep dsh-client-runtime as a development-only type dependency. Remove its peer metadata, client injection entry, and runtime external allowance so newer hosts do not preflight a package they no longer ship.

Fixes #193
2026-09-08 16:58:20 +08:00
43 changed files with 3201 additions and 3811 deletions

No files matched your search

@@ -29,6 +29,24 @@ function fakeApi() {
} }
describe("ChromiumCdp", () => { describe("ChromiumCdp", () => {
it("changes attachment identity only across real debugger attachments", async () => {
const { api, onEvent, onDetach } = fakeApi();
const cdp = new ChromiumCdp(api);
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
const first = cdp.getAttachmentId(4);
expect(first).toBeTruthy();
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
expect(cdp.getAttachmentId(4)).toBe(first);
onDetach.fire({ tabId: 4 }, "target_closed");
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
expect(cdp.getAttachmentId(4)).not.toBe(first);
await cdp.detach(4);
expect(cdp.getAttachmentId(4)).toBeUndefined();
});
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => { it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi(); const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation( (api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -2,6 +2,24 @@ import { describe, expect, it } from "vitest";
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph"; import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
describe("buildFrameGraph", () => { describe("buildFrameGraph", () => {
it("preserves the owning target's loader identity", () => {
const graph = buildFrameGraph([
{
target: { tabId: 4 },
tree: {
frame: { id: "root", loaderId: "root-loader" },
childFrames: [{ frame: { id: "child", loaderId: "stale-placeholder" } }],
},
},
{
target: { tabId: 4, sessionId: "child-session" },
tree: { frame: { id: "child", loaderId: "child-loader" } },
},
]);
expect(graph?.frames.find((frame) => frame.frameId === "root")?.loaderId).toBe("root-loader");
expect(graph?.frames.find((frame) => frame.frameId === "child")?.loaderId).toBe("child-loader");
});
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => { it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
const graph = buildFrameGraph([ const graph = buildFrameGraph([
{ {
@@ -164,6 +164,7 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
export class ChromiumCdp { export class ChromiumCdp {
private readonly api: CdpDebuggerApi; private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>(); private readonly attachedTabs = new Set<number>();
private readonly attachmentIds = new Map<number, string>();
private readonly attachInFlight = new Map<number, Promise<void>>(); private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>(); private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>(); private readonly tabOwners = new Map<number, Set<string>>();
@@ -198,6 +199,11 @@ export class ChromiumCdp {
this.bindFrameTargetHandler(); this.bindFrameTargetHandler();
} }
/** Identity of this actual debugger attachment, independent of frame topology. */
getAttachmentId(tabId: number): string | undefined {
return this.attachmentIds.get(tabId);
}
/** Attach to `tabId` if we haven't already in this driver. */ /** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> { async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching. // Returning a tab clears the cache before Chrome finishes detaching.
@@ -217,6 +223,7 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId); await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId); await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId); this.attachedTabs.add(tabId);
this.attachmentIds.set(tabId, crypto.randomUUID());
await this.enableFrameDiscovery({ tabId }).catch((err) => { await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err }); console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
}); });
@@ -444,6 +451,7 @@ export class ChromiumCdp {
this.attachInFlight.delete(tabId); this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return; if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId); this.attachedTabs.delete(tabId);
this.attachmentIds.delete(tabId);
this.clearDialogState(tabId); this.clearDialogState(tabId);
this.clearConsoleState(tabId); this.clearConsoleState(tabId);
this.clearNetworkState(tabId); this.clearNetworkState(tabId);
@@ -502,6 +510,7 @@ export class ChromiumCdp {
this.attachInFlight.clear(); this.attachInFlight.clear();
this.tabOwners.clear(); this.tabOwners.clear();
this.attachedTabs.clear(); this.attachedTabs.clear();
this.attachmentIds.clear();
this.dialogBuffers.clear(); this.dialogBuffers.clear();
this.dialogSequences.clear(); this.dialogSequences.clear();
this.consoleBuffers.clear(); this.consoleBuffers.clear();
@@ -848,6 +857,7 @@ export class ChromiumCdp {
const listener = (source: chrome.debugger.Debuggee, _reason: string) => { const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
if (typeof source.tabId === "number") { if (typeof source.tabId === "number") {
this.attachedTabs.delete(source.tabId); this.attachedTabs.delete(source.tabId);
this.attachmentIds.delete(source.tabId);
this.attachInFlight.delete(source.tabId); this.attachInFlight.delete(source.tabId);
this.tabOwners.delete(source.tabId); this.tabOwners.delete(source.tabId);
this.clearDialogState(source.tabId); this.clearDialogState(source.tabId);
@@ -8,6 +8,7 @@ export interface CdpFrame {
parentFrameId?: string; parentFrameId?: string;
ownerBackendNodeId?: number; ownerBackendNodeId?: number;
url?: string; url?: string;
loaderId?: string;
target: CdpTarget; target: CdpTarget;
} }
@@ -21,6 +22,7 @@ export interface CdpFrameTreeNode {
id: string; id: string;
parentId?: string; parentId?: string;
url?: string; url?: string;
loaderId?: string;
}; };
childFrames?: CdpFrameTreeNode[]; childFrames?: CdpFrameTreeNode[];
} }
@@ -48,6 +50,7 @@ function mergeFrame(
frameId: node.frame.id, frameId: node.frame.id,
...(parentFrameId ? { parentFrameId } : {}), ...(parentFrameId ? { parentFrameId } : {}),
...(url ? { url } : {}), ...(url ? { url } : {}),
...(node.frame.loaderId ? { loaderId: node.frame.loaderId } : {}),
target, target,
}); });
} }
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; import { resolveNodeGeometry } from "../frame-geometry";
import { import {
clipPolygon, clipPolygon,
polygonArea, polygonArea,
@@ -10,6 +10,7 @@ import {
rectPolygon, rectPolygon,
regionBounds, regionBounds,
} from "../geometry"; } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => { describe("frame geometry projection", () => {
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected child command ${method}`); throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"], }) as CdpRunner["sendToTarget"],
}; };
const projection = await resolveFrameProjection( const projection = await new GeometryContext(cdp, 4, {
cdp, rootFrameId: "main",
{ frames: [
rootFrameId: "main", { frameId: "main", target: { tabId: 4 } },
frames: [ {
{ frameId: "main", target: { tabId: 4 } }, frameId: "same-process-parent",
{ parentFrameId: "main",
frameId: "same-process-parent", ownerBackendNodeId: 10,
parentFrameId: "main", target: { tabId: 4 },
ownerBackendNodeId: 10, },
target: { tabId: 4 }, {
}, frameId: "oopif",
{ parentFrameId: "same-process-parent",
frameId: "oopif", ownerBackendNodeId: 20,
parentFrameId: "same-process-parent", target: { tabId: 4, sessionId: "oopif-session" },
ownerBackendNodeId: 20, },
target: { tabId: 4, sessionId: "oopif-session" }, ],
}, }).targetProjection("oopif");
],
},
"oopif",
);
expect(projection?.edges).toEqual([ expect(projection?.edges).toEqual([
{ {
@@ -0,0 +1,284 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target },
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
],
};
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 300, height: 200 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
});
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
}
describe("measurement geometry context", () => {
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const results = await Promise.all(
Array.from({ length: 20 }, () => context.targetProjection("child")),
);
expect(results.every((value) => value === results[0])).toBe(true);
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"Page.getLayoutMetrics",
"DOM.getBoxModel",
]);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
expect(cdp.calls.mock.calls).toHaveLength(4);
});
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const source = { target, frameId: "child" };
const projection = await context.snapshotProjection(source, 10, [], {
width: 1000,
height: 1000,
});
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
expect(projectSnapshotRect(input!, projection!)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
expect(projectSnapshotRect(scrolled!, projection!)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
1,
);
expect(
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
).toHaveLength(1);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ x: 10, y: 20 },
)!;
const projection = {
source: { target, frameId: "main" },
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
};
expect(projectSnapshotRect(input, projection)).toEqual({
x: 20,
y: 30,
width: 120,
height: 40,
});
expect(
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
).toBeNull();
expect(
projectSnapshotRect(input, {
...projection,
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
})) as CdpRunner["send"],
};
expect(
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 100,
height: 100,
}),
).toBeNull();
});
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
const cdp = driver();
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
{ target, frameId: "nested" },
10,
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
{ width: 1000, height: 1000 },
);
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ x: 0, y: 0 },
);
expect(projectSnapshotRect(input!, projection!)).toEqual({
x: 60,
y: 620,
width: 100,
height: 100,
});
});
it("rejects malformed frame ancestry without looping", async () => {
const cdp = driver();
const cycle: CdpFrameGraph = {
rootFrameId: "a",
frames: [
{ frameId: "a", parentFrameId: "b", target },
{ frameId: "b", parentFrameId: "a", target },
],
};
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
expect(cdp.calls).not.toHaveBeenCalled();
const orphan: CdpFrameGraph = {
rootFrameId: "a",
frames: [{ frameId: "a", parentFrameId: "missing", target }],
};
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
});
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
const cdp = driver();
expect(
await resolveNodeGeometry(
cdp,
4,
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ code: "cdp_failed" });
expect(cdp.calls).not.toHaveBeenCalled();
});
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
let active = 0;
let peak = 0;
const release: Array<() => void> = [];
const send = vi.fn(async () => {
active++;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
});
const controller = new AbortController();
const context = new GeometryContext(
{ send: send as CdpRunner["send"] },
4,
undefined,
controller.signal,
);
const tasks = Array.from({ length: 8 }, (_, index) =>
context.viewport({ tabId: 4, sessionId: String(index) }),
);
const settled = Promise.allSettled(tasks);
expect(send).toHaveBeenCalledTimes(4);
controller.abort();
for (const done of release) done();
const results = await settled;
expect(peak).toBe(4);
expect(send).toHaveBeenCalledTimes(4);
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
});
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
const controller = new AbortController();
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
cdp.calls.mockImplementation(async (tab, method) => {
const result = await original(tab, method);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
await expect(
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
}),
).rejects.toMatchObject({ name: "AbortError" });
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.releaseObject",
]);
});
it("reads live owner geometry only after scrolling", async () => {
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
let scrolled = false;
cdp.calls.mockImplementation(async (tab, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
scrolled = true;
return {};
}
if (method === "DOM.getContentQuads") {
expect(scrolled).toBe(true);
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
}
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
return original(tab, method);
});
expect(
await resolveNodeGeometry(
cdp,
4,
{ target, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
1,
);
});
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
).toBeNull();
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
).toBeNull();
});
it("retries a rejected measurement only in a new operation", async () => {
const cdp = driver();
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
const context = new GeometryContext(cdp, 4);
await expect(context.viewport(target)).rejects.toThrow("unavailable");
await expect(context.viewport(target)).rejects.toThrow("unavailable");
expect(cdp.calls).toHaveBeenCalledTimes(1);
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
width: 1000,
height: 1000,
});
});
});
@@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared"; import type { CdpRunner } from "@/tools/shared";
import type { BuildVomSceneOptions, VomFrameDocument } from "../observation";
import { import {
buildFrameVomScene, buildFrameVomScene,
buildVomScene,
type CdpAxNode, type CdpAxNode,
captureVomObservation, captureVomObservation,
handleGetHtml, handleGetHtml,
@@ -17,7 +17,35 @@ import {
type ScreenshotDeps, type ScreenshotDeps,
stripDataUrlPrefix, stripDataUrlPrefix,
} from "../observation"; } from "../observation";
import type { CapturedNode, CapturedViewModel } from "../vom/capture"; import type { CapturedNode, CapturedSceneInput } from "../vom/facts";
interface SceneFixture extends CapturedSceneInput {
documents?: VomFrameDocument[];
}
// Only supplies the common root fixture; child documents declare their own ownership.
function buildTestScene(
axNodes: CdpAxNode[],
captured: SceneFixture,
options: BuildVomSceneOptions = {},
) {
const rootFrameId = captured.rootFrameId ?? "root";
return buildFrameVomScene(
[
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: { tabId: 7 },
url: options.pageUrl,
domNodes: captured.nodes,
axNodes,
},
...(captured.documents ?? []),
],
captured,
options,
);
}
function fakeAgentWindow(ids: number[]) { function fakeAgentWindow(ids: number[]) {
let i = 0; let i = 0;
@@ -266,7 +294,15 @@ describe("handleScreenshot", () => {
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined(); expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
}); });
it("captures a clipped PNG when ref is given", async () => { it.each([
{ zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } },
{ zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } },
{ zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } },
])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({
zoom,
scroll,
expected,
}) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11"); const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, { tabId: 7 }); ctx.refStore.set("e5", 999, { tabId: 7 });
@@ -274,6 +310,10 @@ describe("handleScreenshot", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }), "Page.captureScreenshot": () => ({ data: TINY_PNG }),
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll },
cssVisualViewport: { zoom },
}),
}); });
const captureVisibleTab = vi.fn(); const captureVisibleTab = vi.fn();
const res = await handleScreenshot( const res = await handleScreenshot(
@@ -297,7 +337,8 @@ describe("handleScreenshot", () => {
clip?: { x: number; y: number; width: number; height: number }; clip?: { x: number; y: number; width: number; height: number };
} }
)?.clip; )?.clip;
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); expect(clip).toMatchObject(expected);
expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1);
}); });
it("returns not_found for unknown ref", async () => { it("returns not_found for unknown ref", async () => {
@@ -509,10 +550,10 @@ describe("handleScreenshot overlay suppression", () => {
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// buildVomScene // Semantic scene fixtures
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
describe("buildVomScene", () => { describe("buildFrameVomScene", () => {
it("merges multiple sibling and nested frames without mixing their targets", () => { it("merges multiple sibling and nested frames without mixing their targets", () => {
const node = ( const node = (
backendNodeId: number, backendNodeId: number,
@@ -550,25 +591,10 @@ describe("buildVomScene", () => {
node(300, null, "nested", "body"), node(300, null, "nested", "body"),
node(301, 300, "nested", "input", { placeholder: "Nested field" }), node(301, 300, "nested", "input", { placeholder: "Nested field" }),
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1200, height: 800 }, viewport: { width: 1200, height: 800 },
nodes: mainNodes, nodes: mainNodes,
iframeNodes: new Map([
[10, firstNodes],
[20, secondNodes],
[210, nestedNodes],
]),
frameNodes: new Map([
["main", mainNodes],
["first", firstNodes],
["second", secondNodes],
["nested", nestedNodes],
]),
frameOwnerBackendNodeIds: new Map([
["first", 10],
["second", 20],
["nested", 210],
]),
rootFrameId: "main", rootFrameId: "main",
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}; };
@@ -667,11 +693,10 @@ describe("buildVomScene", () => {
position: "static", position: "static",
pointerEvents: "auto", pointerEvents: "auto",
}; };
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 800, height: 600 }, viewport: { width: 800, height: 600 },
nodes: [], nodes: [],
iframeNodes: new Map(),
frameNodes: new Map([["child", [childNode]]]),
rootFrameId: "main", rootFrameId: "main",
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}; };
@@ -733,14 +758,10 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 800, height: 600 }, viewport: { width: 800, height: 600 },
nodes: mainNodes, nodes: mainNodes,
iframeNodes: new Map(),
frameNodes: new Map([
["main", mainNodes],
["child", []],
]),
rootFrameId: "main", rootFrameId: "main",
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}; };
@@ -816,9 +837,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 200, backendDOMNodeId: 200,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -844,11 +864,11 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.viewport).toEqual({ width: 1000, height: 800 }); expect(scene.viewport).toEqual({ width: 1000, height: 800 });
expect(scene.nodes[0]).toEqual( expect(scene.nodes[0]).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 100, backendNodeId: 100,
parentId: null, parentId: null,
role: "RootWebArea", role: "RootWebArea",
name: "Example", name: "Example",
@@ -860,14 +880,14 @@ describe("buildVomScene", () => {
); );
expect(scene.nodes[1]).toEqual( expect(scene.nodes[1]).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 200, backendNodeId: 200,
parentId: 100, parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
role: "button", role: "button",
name: "Submit", name: "Submit",
tag: "button", tag: "button",
rect: { x: 20, y: 20, w: 120, h: 40 }, rect: { x: 20, y: 20, w: 120, h: 40 },
domParentId: 100, domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
domAncestorIds: [100], domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id],
attrs: {}, attrs: {},
}), }),
); );
@@ -896,9 +916,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 300, backendDOMNodeId: 300,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -934,10 +953,10 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org"); expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org");
expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined(); expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined();
}); });
it("uses the nearest backend AX ancestor as parentId", () => { it("uses the nearest backend AX ancestor as parentId", () => {
@@ -962,9 +981,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30, backendDOMNodeId: 30,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 400, height: 300 }, viewport: { width: 400, height: 300 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -990,10 +1008,12 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]); expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]);
expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10); expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe(
scene.nodes.find((node) => node.backendNodeId === 10)!.id,
);
}); });
it("maps iframe sub-document controls to VomNodes", () => { it("maps iframe sub-document controls to VomNodes", () => {
@@ -1011,7 +1031,7 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20, backendDOMNodeId: 20,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
nodes: [ nodes: [
{ {
@@ -1035,10 +1055,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
iframeNodes: new Map([ documents: [
[ {
20, frameId: "child",
[ contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{ {
backendNodeId: 101, backendNodeId: 101,
parentBackendNodeId: null, parentBackendNodeId: null,
@@ -1060,26 +1085,25 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
], },
]), ],
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}; };
const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) => const scene = buildTestScene(axNodes, captured);
[101, 102].includes(node.id), const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!));
);
expect(iframeControls).toEqual([ expect(iframeControls).toEqual([
expect.objectContaining({ expect.objectContaining({
id: 101, backendNodeId: 101,
parentId: 20, parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "textbox", role: "textbox",
name: "请输入手机号", name: "请输入手机号",
sensitive: false, sensitive: false,
}), }),
expect.objectContaining({ expect.objectContaining({
id: 102, backendNodeId: 102,
parentId: 20, parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "textbox", role: "textbox",
name: "密码", name: "密码",
sensitive: true, sensitive: true,
@@ -1100,11 +1124,9 @@ describe("buildVomScene", () => {
parentId: "1", parentId: "1",
role: { type: "role", value: "Iframe" }, role: { type: "role", value: "Iframe" },
backendDOMNodeId: 20, backendDOMNodeId: 20,
childIds: ["3"],
}, },
{ {
nodeId: "3", nodeId: "3",
parentId: "2",
role: { type: "role", value: "textbox" }, role: { type: "role", value: "textbox" },
name: { type: "x", value: "输入密码" }, name: { type: "x", value: "输入密码" },
value: { value: "iframe-secret" }, value: { value: "iframe-secret" },
@@ -1112,7 +1134,7 @@ describe("buildVomScene", () => {
properties: [{ name: "inputType", value: { value: "password" } }], properties: [{ name: "inputType", value: { value: "password" } }],
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
nodes: [ nodes: [
{ {
@@ -1136,10 +1158,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
iframeNodes: new Map([ documents: [
[ {
20, frameId: "child",
[ contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [axNodes[2]],
domNodes: [
{ {
backendNodeId: 102, backendNodeId: 102,
parentBackendNodeId: null, parentBackendNodeId: null,
@@ -1151,25 +1178,27 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
], },
]), ],
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}; };
const passwordNodes = buildVomScene(axNodes, captured).nodes.filter( const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter(
(node) => node.role === "textbox" && node.name === "输入密码", (node) => node.role === "textbox" && node.name === "输入密码",
); );
expect(passwordNodes).toHaveLength(1); expect(passwordNodes).toHaveLength(1);
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); expect(passwordNodes[0]).toEqual(
expect.objectContaining({ backendNodeId: 202, sensitive: true }),
);
expect(passwordNodes[0].value).toBeUndefined(); expect(passwordNodes[0].value).toBeUndefined();
const rendered = renderVom(buildVomScene(axNodes, captured)).text; const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text;
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"'); expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
expect(rendered).not.toContain("iframe-secret"); expect(rendered).not.toContain("iframe-secret");
}); });
it("keeps unnamed iframe controls but skips unnamed iframe links", () => { it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
const scene = buildVomScene( const scene = buildTestScene(
[ [
{ {
nodeId: "1", nodeId: "1",
@@ -1208,10 +1237,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
iframeNodes: new Map([ documents: [
[ {
20, frameId: "child",
[ contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{ {
backendNodeId: 201, backendNodeId: 201,
parentBackendNodeId: null, parentBackendNodeId: null,
@@ -1233,25 +1267,25 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
], },
]), ],
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}, },
); );
expect(scene.nodes.find((node) => node.id === 201)).toEqual( expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 201, backendNodeId: 201,
parentId: 20, parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "button", role: "button",
}), }),
); );
expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name"); expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name");
expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined(); expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined();
}); });
it("preserves captured-only iframe anchors for iframe sub-document controls", () => { it("preserves captured-only iframe anchors for iframe sub-document controls", () => {
const scene = buildVomScene( const scene = buildTestScene(
[ [
{ {
nodeId: "1", nodeId: "1",
@@ -1284,10 +1318,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
iframeNodes: new Map<number, CapturedNode[]>([ documents: [
[ {
20, frameId: "child",
[ contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{ {
backendNodeId: 201, backendNodeId: 201,
parentBackendNodeId: null, parentBackendNodeId: null,
@@ -1309,16 +1348,16 @@ describe("buildVomScene", () => {
pointerEvents: "auto", pointerEvents: "auto",
}, },
], ],
], },
]), ],
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
}, },
); );
expect(scene.nodes.find((node) => node.id === 20)).toEqual( expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 20, backendNodeId: 20,
parentId: 10, parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id,
role: "Iframe", role: "Iframe",
name: "支付验证", name: "支付验证",
}), }),
@@ -1339,9 +1378,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10, backendDOMNodeId: 10,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 320, height: 240 }, viewport: { width: 320, height: 240 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1362,9 +1400,9 @@ describe("buildVomScene", () => {
], ],
}; };
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 10, backendNodeId: 10,
sensitive: true, sensitive: true,
}), }),
); );
@@ -1379,9 +1417,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10, backendDOMNodeId: 10,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1398,9 +1435,9 @@ describe("buildVomScene", () => {
], ],
}; };
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 10, backendNodeId: 10,
role: "textbox", role: "textbox",
name: "+86", name: "+86",
placeholder: "输入手机号", placeholder: "输入手机号",
@@ -1418,9 +1455,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10, backendDOMNodeId: 10,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1437,8 +1473,13 @@ describe("buildVomScene", () => {
], ],
}; };
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }), expect.objectContaining({
backendNodeId: 10,
role: "textbox",
name: "Email",
value: "a@b.com",
}),
); );
}); });
@@ -1457,9 +1498,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20, backendDOMNodeId: 20,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1497,9 +1537,9 @@ describe("buildVomScene", () => {
], ],
}; };
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( expect(
expect.objectContaining({ role: "textbox", name: "验证码" }), buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
); ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" }));
}); });
it("uses preceding AX static text as a form control label", () => { it("uses preceding AX static text as a form control label", () => {
@@ -1523,9 +1563,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20, backendDOMNodeId: 20,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1542,13 +1581,13 @@ describe("buildVomScene", () => {
], ],
}; };
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( expect(
expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }), buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
); ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }));
}); });
it("marks runtime default input values without treating them as ordinary filled input", () => { it("marks runtime default input values without treating them as ordinary filled input", () => {
const scene = buildVomScene( const scene = buildTestScene(
[ [
{ {
nodeId: "1", nodeId: "1",
@@ -1558,7 +1597,6 @@ describe("buildVomScene", () => {
], ],
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1602,9 +1640,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20, backendDOMNodeId: 20,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1654,9 +1691,14 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual( expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }), expect.objectContaining({
backendNodeId: 20,
role: "button",
name: "close",
cursor: "pointer",
}),
); );
const rendered = renderVom(scene); const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"'); expect(rendered.text).toContain('@e1 button "close"');
@@ -1690,9 +1732,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30, backendDOMNodeId: 30,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1731,9 +1772,9 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined(); expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined();
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link"); expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link");
}); });
it("promotes only the outermost clickable in a nested pointer chain", () => { it("promotes only the outermost clickable in a nested pointer chain", () => {
@@ -1758,9 +1799,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30, backendDOMNodeId: 30,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1799,11 +1839,15 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual( expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }), expect.objectContaining({
backendNodeId: 20,
role: "button",
attrs: { "aria-label": "收藏" },
}),
); );
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined(); expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined();
const rendered = renderVom(scene); const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
@@ -1827,9 +1871,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20, backendDOMNodeId: 20,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1877,10 +1920,10 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.activeScopeBlocks).toEqual([ expect(scene.activeScopeBlocks).toEqual([
{ {
triggerId: 20, triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
label: "Reviews (12)", label: "Reviews (12)",
lines: ["Jane - ear cups are small", "Bob - great sound"], lines: ["Jane - ear cups are small", "Bob - great sound"],
}, },
@@ -1904,11 +1947,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20, backendDOMNodeId: 20,
}, },
]; ];
const scene = buildVomScene( const scene = buildTestScene(
axNodes, axNodes,
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -1941,7 +1983,11 @@ describe("buildVomScene", () => {
); );
expect(scene.surfaces).toEqual([ expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, {
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
triggerAction: "hover",
subItems: ["Shoes", "Bags"],
},
]); ]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
}); });
@@ -1962,11 +2008,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21, backendDOMNodeId: 21,
}, },
]; ];
const scene = buildVomScene( const scene = buildTestScene(
axNodes, axNodes,
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2027,11 +2072,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21, backendDOMNodeId: 21,
}, },
]; ];
const scene = buildVomScene( const scene = buildTestScene(
axNodes, axNodes,
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2076,7 +2120,11 @@ describe("buildVomScene", () => {
); );
expect(scene.surfaces).toEqual([ expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, {
triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id,
triggerAction: "hover",
subItems: ["My profile"],
},
]); ]);
}); });
@@ -2096,11 +2144,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21, backendDOMNodeId: 21,
}, },
]; ];
const scene = buildVomScene( const scene = buildTestScene(
axNodes, axNodes,
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2151,7 +2198,11 @@ describe("buildVomScene", () => {
); );
expect(scene.surfaces).toEqual([ expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, {
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
]); ]);
expect(renderVom(scene).text).toContain( expect(renderVom(scene).text).toContain(
'@e1 button "image" [hover first: My profile | Sign out]', '@e1 button "image" [hover first: My profile | Sign out]',
@@ -2174,11 +2225,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 42, backendDOMNodeId: 42,
}, },
]; ];
const scene = buildVomScene( const scene = buildTestScene(
axNodes, axNodes,
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2248,9 +2298,8 @@ describe("buildVomScene", () => {
properties: [{ name: "inputType", value: { value: "credit-card" } }], properties: [{ name: "inputType", value: { value: "credit-card" } }],
}, },
]; ];
const scene = buildVomScene(axNodes, { const scene = buildTestScene(axNodes, {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2297,7 +2346,7 @@ describe("buildVomScene", () => {
], ],
}); });
expect(scene.nodes.find((node) => node.id === 20)).toEqual( expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
expect.objectContaining({ expect.objectContaining({
name: "Products [expanded]", name: "Products [expanded]",
attrs: expect.objectContaining({ attrs: expect.objectContaining({
@@ -2307,13 +2356,17 @@ describe("buildVomScene", () => {
}), }),
); );
expect(scene.activeScopeBlocks).toEqual([ expect(scene.activeScopeBlocks).toEqual([
{ triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] }, {
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
label: "Products [expanded]",
lines: ["Shoes Bags"],
},
]); ]);
expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true); expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true);
}); });
it("aggregates AX virtual text into unnamed structural nodes", () => { it("aggregates AX virtual text into unnamed structural nodes", () => {
const scene = buildVomScene( const scene = buildTestScene(
[ [
{ {
nodeId: "1", nodeId: "1",
@@ -2337,7 +2390,6 @@ describe("buildVomScene", () => {
], ],
{ {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2364,13 +2416,12 @@ describe("buildVomScene", () => {
}, },
); );
expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text"); expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text");
}); });
it("marks captured dialog elements as modal without AX role or aria-modal", () => { it("marks captured dialog elements as modal without AX role or aria-modal", () => {
const scene = buildVomScene([], { const scene = buildTestScene([], {
viewport: { width: 640, height: 480 }, viewport: { width: 640, height: 480 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
nodes: [ nodes: [
{ {
@@ -2388,7 +2439,7 @@ describe("buildVomScene", () => {
expect(scene.nodes[0]).toEqual( expect(scene.nodes[0]).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 50, backendNodeId: 50,
modal: true, modal: true,
}), }),
); );
@@ -2417,9 +2468,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 202, backendDOMNodeId: 202,
}, },
]; ];
const captured: CapturedViewModel = { const captured: SceneFixture = {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set([200, 201, 202]), excludedBackendNodeIds: new Set([200, 201, 202]),
nodes: [ nodes: [
{ {
@@ -2435,9 +2485,9 @@ describe("buildVomScene", () => {
], ],
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined(); expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined();
expect(scene.nodes.find((n) => n.id === 2)).toEqual( expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual(
expect.objectContaining({ role: "button", name: "Real Button" }), expect.objectContaining({ role: "button", name: "Real Button" }),
); );
}); });
@@ -2465,13 +2515,12 @@ describe("buildVomScene", () => {
backendDOMNodeId: 203, backendDOMNodeId: 203,
}, },
]; ];
const scene = buildVomScene(axNodes, { const scene = buildTestScene(axNodes, {
viewport: { width: 1000, height: 800 }, viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set([200]), excludedBackendNodeIds: new Set([200]),
nodes: [], nodes: [],
}); });
expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined(); expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined();
}); });
}); });
@@ -2576,6 +2625,7 @@ describe("handleSnapshot", () => {
strings, strings,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [i("body"), i("button")], nodeName: [i("body"), i("button")],
@@ -2667,6 +2717,7 @@ describe("handleSnapshot", () => {
strings, strings,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [i("body"), i("button")], nodeName: [i("body"), i("button")],
@@ -2761,6 +2812,7 @@ describe("handleSnapshot", () => {
strings, strings,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [i("body"), i("button")], nodeName: [i("body"), i("button")],
@@ -3025,6 +3077,7 @@ describe("handleSnapshot", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2], parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("input")], nodeName: [i("html"), i("body"), i("div"), i("input")],
@@ -3082,6 +3135,7 @@ describe("handleSnapshot", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1, 1, 1], parentIndex: [-1, 0, 1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")], nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
@@ -3215,12 +3269,15 @@ describe("handleSnapshot", () => {
const send = vi.fn(async (_tabId: number, method: string) => { const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.getLayoutMetrics") return VP_METRICS; if (method === "Page.getLayoutMetrics") return VP_METRICS;
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[0]] };
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
throw new Error(`unexpected root CDP method ${method}`); throw new Error(`unexpected root CDP method ${method}`);
}); });
const sendToTarget = vi.fn(async (_target, method: string) => { const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Accessibility.enable") return {}; if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[1]] };
if (method === "Accessibility.getFullAXTree") return { nodes: childAx }; if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
throw new Error(`unexpected child CDP method ${method}`); throw new Error(`unexpected child CDP method ${method}`);
}); });
@@ -3321,6 +3378,7 @@ describe("handleSnapshot", () => {
strings, strings,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1, 3], parentIndex: [-1, 0, 1, 1, 3],
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")], nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
@@ -3608,6 +3666,7 @@ describe("handleSnapshot", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -3634,6 +3693,7 @@ describe("handleSnapshot", () => {
}, },
// documents[1]: cross-origin login form // documents[1]: cross-origin login form
{ {
frameId: "child",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -3812,6 +3872,7 @@ describe("handleSnapshot", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4, 5], parentIndex: [-1, 0, 1, 0, 3, 4, 5],
nodeName: [ nodeName: [
+46 -151
View File
@@ -1,24 +1,21 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types"; import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import { import {
clipPolygon, clipPolygon,
type GeometryProjection,
type Point, type Point,
type Polygon, type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea, polygonArea,
polygonCentroid, polygonCentroid,
projectRegionToViewport, projectRegionToViewport,
type Quad,
type Region, type Region,
rectPolygon, rectPolygon,
regionBounds, regionBounds,
type Size,
type ViewportRect, type ViewportRect,
} from "./geometry"; } from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; import type { CssViewport } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
export interface NodeAddress { export interface NodeAddress {
target: CdpTarget; target: CdpTarget;
@@ -27,6 +24,8 @@ export interface NodeAddress {
} }
export interface ResolvedNodeGeometry { export interface ResolvedNodeGeometry {
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
topViewport: CssViewport;
topVisibleRegions: Region; topVisibleRegions: Region;
topBounds: ViewportRect; topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */ /** Point in the top-level tab viewport, used by root-target input events. */
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
return { code: "cdp_failed", message }; return { code: "cdp_failed", message };
} }
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners( async function scrollFrameOwners(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
graph: CdpFrameGraph, context: GeometryContext,
frameId: string, frameId: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const byId = frameMap(graph); const ancestry = await context.ancestry(frameId);
const ancestry = frameAncestry(graph, frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) { for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) { if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`); return geometryError(`could not resolve frame owner for ${child.frameId}`);
} }
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
return null; return null;
} }
async function scrollElementWithFrameGraph( async function scrollElementWithContext(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
target: CdpTarget, target: CdpTarget,
backendNodeId: number, backendNodeId: number,
frameId: string | undefined, frameId: string | undefined,
graph: CdpFrameGraph | null, context: GeometryContext,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
if (frameId) { if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); const frame = await context.frame(frameId);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId); if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
return geometryError("node frame does not belong to its target");
}
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
if (error) return error; if (error) return error;
} else if (target.sessionId) { } else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
backendNodeId: number, backendNodeId: number,
frameId?: string, frameId?: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); return scrollElementWithContext(
cdp,
tabId,
target,
backendNodeId,
frameId,
new GeometryContext(cdp, tabId),
);
} }
function largestRegion(regions: Region): Polygon | null { function largestRegion(regions: Region): Polygon | null {
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
if (address.target.sessionId && !address.frameId) { if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
} }
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
const context = new GeometryContext(cdp, tabId);
const graph = address.frameId ? await context.graph() : null;
if (address.frameId && graph) {
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
return geometryError("node frame does not belong to its target");
}
}
if (address.frameId && !graph) { if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`); return geometryError(`could not resolve frame graph for ${address.frameId}`);
} }
if (options.scrollIntoView) { if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph( const scrollError = await scrollElementWithContext(
cdp, cdp,
tabId, tabId,
address.target, address.target,
address.backendNodeId, address.backendNodeId,
address.frameId, address.frameId,
graph, context,
); );
if (scrollError) return scrollError; if (scrollError) return scrollError;
} }
// Only topology was read above. Begin live measurements after scrolling.
const localRegion = await nodeContentRegion( const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target), cdpRunnerForTarget(cdp, address.target),
tabId, tabId,
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
let topVisibleRegions: Region; let topVisibleRegions: Region;
let targetActionPoint: Point | null = null; let targetActionPoint: Point | null = null;
if (address.frameId && graph) { if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId); const projection = await context.targetProjection(address.frameId);
if (!projection) if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`); return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection); topVisibleRegions = projectRegionToViewport(localRegion, projection);
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
} }
} else { } else {
const viewport = await targetViewport(cdp, address.target); const viewport = await context.viewport(address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry"); if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion topVisibleRegions = localRegion
.map((polygon) => .map((polygon) =>
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
return { code: "permission_denied", message: "element not visible in its target" }; return { code: "permission_denied", message: "element not visible in its target" };
} }
targetActionPoint ??= actionPoint; targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; return {
topVisibleRegions,
topBounds,
actionPoint,
targetActionPoint,
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
};
} catch (error) { } catch (error) {
return geometryError(error instanceof Error ? error.message : String(error)); return geometryError(error instanceof Error ? error.message : String(error));
} }
-19
View File
@@ -245,22 +245,3 @@ export function projectRectToViewport(
if (!rect) return null; if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
} }
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
@@ -0,0 +1,84 @@
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { GeometryProjection, ViewportRect } from "../geometry";
import { projectRectToViewport } from "../geometry";
export interface CoordinateOwner {
target: CdpTarget;
frameId?: string;
}
export interface FrameViewportRect {
space: "frame-viewport-css";
owner: CoordinateOwner;
rect: ViewportRect;
}
export interface SnapshotProjection {
source: CoordinateOwner;
geometry: GeometryProjection;
}
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
scroll: { x: number; y: number },
): FrameViewportRect | null {
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const [x, y, width, height] = bounds;
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
};
}
export function projectSnapshotRect(
input: FrameViewportRect,
projection: SnapshotProjection,
): ViewportRect | null {
if (
input.owner.frameId !== projection.source.frameId ||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
) {
return null;
}
const { x, y, width: w, height: h } = input.rect;
return projectRectToViewport({ x, y, w, h }, projection.geometry);
}
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
export interface CssViewport {
width: number;
height: number;
scrollX: number;
scrollY: number;
cssToDip: number;
}
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
export function screenshotPageRect(
rect: ViewportRect,
viewport: CssViewport,
): { space: "page-dip"; rect: ViewportRect } | null {
const { cssToDip, scrollX, scrollY } = viewport;
if (
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
cssToDip <= 0 ||
rect.width <= 0 ||
rect.height <= 0
)
return null;
return {
space: "page-dip",
rect: {
x: (rect.x + scrollX) * cssToDip,
y: (rect.y + scrollY) * cssToDip,
width: rect.width * cssToDip,
height: rect.height * cssToDip,
},
};
}
@@ -0,0 +1,297 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import {
type GeometryProjection,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
type Quad,
type Size,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjection } from "./coordinate-types";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
return {
width: value?.clientWidth ?? 0,
height: value?.clientHeight ?? 0,
scrollX: value?.pageX ?? 0,
scrollY: value?.pageY ?? 0,
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
};
}
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
private active = 0;
private readonly waiting: Array<() => void> = [];
private graphPromise?: Promise<CdpFrameGraph | null>;
private framesPromise?: Promise<Map<string, CdpFrame>>;
constructor(
private readonly cdp: CdpRunner,
private readonly tabId: number,
graph?: CdpFrameGraph,
private readonly signal?: AbortSignal,
) {
if (graph) this.graphPromise = Promise.resolve(graph);
}
private async request<T>(
target: CdpTarget,
method: string,
params: object,
cleanup = false,
): Promise<T> {
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
else this.active++;
try {
if (!cleanup && this.signal?.aborted)
throw new DOMException("geometry aborted", "AbortError");
return await sendToCdpTarget<T>(this.cdp, target, method, params);
} finally {
const next = this.waiting.shift();
if (next) next();
else this.active--;
}
}
graph(): Promise<CdpFrameGraph | null> {
this.graphPromise ??=
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
return this.graphPromise;
}
private frames(): Promise<Map<string, CdpFrame>> {
this.framesPromise ??= this.graph().then(
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
);
return this.framesPromise;
}
async frame(frameId: string): Promise<CdpFrame | undefined> {
return (await this.frames()).get(frameId);
}
/** Topology only: safe to resolve before the caller scrolls frame owners. */
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
const frames = await this.frames();
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = frames.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
const key = cdpTargetKey(target);
let promise = this.metrics.get(key);
if (!promise) {
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
this.metrics.set(key, promise);
}
return promise;
}
async viewport(target: CdpTarget): Promise<Size | null> {
const { width, height } = cssViewport(await this.layoutMetrics(target));
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
? { width, height }
: null;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
if (!promise) {
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
backendNodeId,
}).then((reply) => parseCdpQuad(reply.model?.content));
this.owners.set(key, promise);
}
return promise;
}
/** Same-target owner quads are already target-relative, including nested owners. */
async snapshotProjection(
source: CoordinateOwner,
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
): Promise<SnapshotProjection | null> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
const owner = await promise;
return owner
? {
source,
geometry: {
sourceClips: [],
edges: [
{
sourceViewport: owner.size,
destinationQuad: owner.quad,
destinationClips: ancestorClips,
},
],
topViewport: viewport,
},
}
: null;
}
private async snapshotOwner(
target: CdpTarget,
backendNodeId: number,
): Promise<{ quad: Quad; size: Size } | null> {
const quad = await this.ownerContent(target, backendNodeId);
if (!quad) return null;
const resolved = await this.request<{ object?: { objectId?: string } }>(
target,
"DOM.resolveNode",
{ backendNodeId },
);
const objectId = resolved.object?.objectId;
if (!objectId) return null;
try {
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this.isConnected) return null;
const style = getComputedStyle(this);
return {
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
};
}`,
returnByValue: true,
},
);
const size = reply.result?.value;
return !reply.exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? { quad, size }
: null;
} finally {
// Cleanup must still run if cancellation arrived after resolving the object.
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
}
}
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
current = parent;
}
return current;
}
private async clips(
frames: Map<string, CdpFrame>,
frame: CdpFrame,
root: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== root.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
if (!quad) return null;
clips.push(quad);
current = parent;
}
return clips;
}
/** Input is already in the frame's CDP target viewport, not its local viewport. */
targetProjection(frameId: string): Promise<GeometryProjection | null> {
let promise = this.projections.get(frameId);
if (!promise) {
promise = this.buildTargetProjection(frameId);
this.projections.set(frameId, promise);
}
return promise;
}
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
const frames = await this.frames();
const frame = frames.get(frameId);
if (!frame) return null;
let root = this.targetRoot(frames, frame);
if (!root) return null;
const sourceViewport = await this.viewport(root.target);
if (!sourceViewport) return null;
const sourceClips = await this.clips(frames, frame, root);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
const seen = new Set<string>();
while (root.parentFrameId) {
if (seen.has(root.frameId)) return null;
seen.add(root.frameId);
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
}
+14 -221
View File
@@ -1,3 +1,4 @@
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target // and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and // tab (defaulting to the Agent Window's active tab when omitted) and
@@ -34,6 +35,7 @@ import type {
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry"; import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { import {
type ChromeTabsApi, type ChromeTabsApi,
enforceToolTargetScope, enforceToolTargetScope,
@@ -47,15 +49,9 @@ import {
type ToolEffect, type ToolEffect,
} from "./shared"; } from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref"; import { resolveSnapshotRef } from "./snapshot-ref";
import { import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
type CapturedNode, import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
type CapturedSurfaceProbe, import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { withOverlayBypass } from "./vom/hover-perception"; import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment"; import { probeTooltipNames } from "./vom/name-enrichment";
import { import {
@@ -197,15 +193,14 @@ async function captureElementScreenshot(
if (isRpcError(geometry)) return geometry; if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds; const rect = geometry.topBounds;
const clip = screenshotPageRect(rect, geometry.topViewport);
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
try { try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
format: "png", format: "png",
clip: { clip: {
x: rect.x, ...clip.rect,
y: rect.y,
width: rect.width,
height: rect.height,
scale: 1, scale: 1,
}, },
}); });
@@ -530,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
function buildConditionalSurfaces( function buildConditionalSurfaces(
nodes: VomNode[], nodes: VomNode[],
captured: CapturedViewModel, captured: CapturedSceneInput,
probes: CapturedSurfaceProbe[], probes: CapturedSurfaceProbe[],
): CondSurface[] { ): CondSurface[] {
if (probes.length === 0) return []; if (probes.length === 0) return [];
@@ -660,161 +655,9 @@ export interface BuildVomSceneOptions {
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>; export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function legacyFrameDocuments(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
pageUrl?: string,
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
): VomFrameDocument[] {
const rootFrameId = captured.rootFrameId ?? "root";
const documents: VomFrameDocument[] = [
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: rootTarget,
...(pageUrl ? { url: pageUrl } : {}),
axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
},
];
const pending = [...captured.iframeNodes.entries()];
let progress = true;
let nextSyntheticFrame = 1;
while (pending.length > 0 && progress) {
progress = false;
for (let index = pending.length - 1; index >= 0; index -= 1) {
const [ownerBackendNodeId, domNodes] = pending[index];
const parent = documents.find((document) =>
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
);
if (!parent) continue;
const frameId =
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
documents.push({
frameId,
parentFrameId: parent.frameId,
ownerBackendNodeId,
contextScopeId: frameId,
target: parent.target,
axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
});
pending.splice(index, 1);
progress = true;
}
}
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const backendFrame = new Map<number, string>();
const childFrameByOwner = new Map<number, string>();
for (const document of documents) {
if (document.ownerBackendNodeId !== undefined) {
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
}
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
}
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
const ownership = new Map<string, string>();
const resolving = new Set<string>();
const frameForAx = (node: CdpAxNode): string => {
const cached = ownership.get(node.nodeId);
if (cached) return cached;
let frameId: string | undefined;
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
if (!frameId && typeof node.backendDOMNodeId === "number") {
frameId = backendFrame.get(node.backendDOMNodeId);
}
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
const parent = axById.get(node.parentId);
if (parent) {
frameId =
typeof parent.backendDOMNodeId === "number"
? childFrameByOwner.get(parent.backendDOMNodeId)
: undefined;
if (!frameId) {
resolving.add(node.nodeId);
frameId = frameForAx(parent);
resolving.delete(node.nodeId);
}
}
}
frameId ??= rootFrameId;
ownership.set(node.nodeId, frameId);
return frameId;
};
for (const node of axNodes) frameForAx(node);
for (const node of axNodes) {
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
const document = documentByFrameId.get(frameId) ?? documents[0];
document.axNodes.push({
...node,
frameId,
...(node.parentId && ownership.get(node.parentId) === frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(node.childIds
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
: {}),
});
}
return documents;
}
function withLegacyBackendIds(scene: VomScene): VomScene {
const used = new Set<number>();
const idMap = new Map<number, number>();
let nextVirtualId = -1;
for (const node of scene.nodes) {
const preferred = node.backendNodeId;
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
used.add(id);
idMap.set(node.id, id);
}
return {
...scene,
nodes: scene.nodes.map((node) => ({
...node,
id: idMap.get(node.id) as number,
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
: {}),
})),
...(scene.surfaces
? {
surfaces: scene.surfaces.map((surface) => ({
...surface,
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
})),
}
: {}),
...(scene.activeScopeBlocks
? {
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
...block,
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
})),
}
: {}),
};
}
export function buildVomScene(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
return withLegacyBackendIds(
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
);
}
export function buildFrameVomScene( export function buildFrameVomScene(
documents: VomFrameDocument[], documents: VomFrameDocument[],
captured: CapturedViewModel, captured: CapturedSceneInput,
options: BuildVomSceneOptions = {}, options: BuildVomSceneOptions = {},
): VomScene { ): VomScene {
const scene = buildSemanticVomScene({ const scene = buildSemanticVomScene({
@@ -830,7 +673,7 @@ export function buildFrameVomScene(
function attachCapturedSceneAnnotations( function attachCapturedSceneAnnotations(
scene: VomScene, scene: VomScene,
documents: VomFrameDocument[], documents: VomFrameDocument[],
captured: CapturedViewModel, captured: CapturedSceneInput,
surfaceProbes: CapturedSurfaceProbe[], surfaceProbes: CapturedSurfaceProbe[],
): VomScene { ): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
@@ -980,51 +823,6 @@ export async function handleGetHtml(
} }
} }
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 };
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
throwIfAborted(signal, "observation");
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: source.clientWidth ?? 0,
height: source.clientHeight ?? 0,
};
} catch (error) {
if (isAbortError(error)) throw error;
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation");
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
options: CaptureVomObservationOptions,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { signal: options.signal });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal);
}
}
export interface HoverProbeOutcome { export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */ /** Whether any active hover was dispatched during this observation. */
performed: boolean; performed: boolean;
@@ -1057,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
async function runHoverProbes( async function runHoverProbes(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
captured: CapturedViewModel, captured: CapturedSceneInput,
documents: VomFrameDocument[], documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>, staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions, options: CaptureVomObservationOptions,
@@ -1096,14 +894,9 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url); await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options); const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal);
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
throwIfAborted(options.signal, "observation");
const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
: documents;
const semanticGraph = buildSemanticGraph({ const semanticGraph = buildSemanticGraph({
documents: normalizedDocuments, documents: normalizedDocuments,
viewport: captured.viewport, viewport: captured.viewport,
+1
View File
@@ -55,6 +55,7 @@ export interface CdpRunner {
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>; sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
detach?(tabId: number): Promise<void>; detach?(tabId: number): Promise<void>;
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>; getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
getAttachmentId?(tabId: number): string | undefined;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>; ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void; trackSessionTab?(sessionId: string, tabId: number): void;
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>; releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
@@ -0,0 +1,471 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES } from "../snapshot";
function fixture(
options: {
frames?: CdpFrame[];
after?: Record<string, { loader?: string; element?: number; missing?: boolean }>;
fail?: string;
missingIdentity?: boolean;
omitDocument?: string;
attachmentChanged?: boolean;
overlay?: string;
} = {},
) {
const frames = options.frames ?? [
{ frameId: "main", loaderId: "main-loader", target: { tabId: 4 } },
{
frameId: "same",
parentFrameId: "main",
ownerBackendNodeId: 3,
loaderId: "same-loader",
target: { tabId: 4 },
},
{
frameId: "nested",
parentFrameId: "same",
ownerBackendNodeId: 13,
loaderId: "nested-loader",
target: { tabId: 4 },
},
{
frameId: "remote",
parentFrameId: "main",
ownerBackendNodeId: 4,
loaderId: "remote-loader",
target: { tabId: 4, sessionId: "remote" },
},
];
const elements = new Map(
frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]),
);
const contexts = new Map(frames.map((frame, i) => [i + 1, frame.frameId]));
const reads = new Map<string, number>();
const logs: Array<{ target: CdpTarget; method: string; params: Record<string, unknown> }> = [];
let snapshots = 0;
const targetCount = new Set(frames.map((frame) => cdpTargetKey(frame.target))).size;
const send = vi.fn(
async <T>(target: CdpTarget, method: string, params: object = {}): Promise<T> => {
logs.push({ target, method, params: params as Record<string, unknown> });
const args = params as Record<string, string | number>;
if (options.fail === `${target.sessionId ?? "main"}:${method}`)
throw new Error("fixture failure");
let result: unknown = {};
if (method === "Page.createIsolatedWorld")
result = {
executionContextId: frames.findIndex((frame) => frame.frameId === args.frameId) + 1,
};
if (method === "Runtime.evaluate")
result = { result: { objectId: contexts.get(Number(args.contextId)) } };
if (method === "DOM.describeNode") {
const id = String(args.objectId);
const count = reads.get(id) ?? 0;
reads.set(id, count + 1);
result = options.missingIdentity
? {}
: {
node: {
backendNodeId:
count > 0 ? (options.after?.[id]?.element ?? elements.get(id)) : elements.get(id),
},
};
}
if (method === "Page.getFrameTree") {
const own = frames.filter(
(frame) =>
cdpTargetKey(frame.target) === cdpTargetKey(target) &&
!options.after?.[frame.frameId]?.missing,
);
const trees = new Map(
own.map((frame) => [
frame.frameId,
{
frame: {
id: frame.frameId,
loaderId: options.after?.[frame.frameId]?.loader ?? frame.loaderId,
},
childFrames: [] as unknown[],
},
]),
);
for (const frame of own)
if (frame.parentFrameId && trees.has(frame.parentFrameId))
trees.get(frame.parentFrameId)!.childFrames.push(trees.get(frame.frameId));
result = { frameTree: trees.get(own[0]?.frameId) };
}
if (method === "Page.getLayoutMetrics")
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOMSnapshot.captureSnapshot") {
snapshots++;
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
result = {
strings: [
"#document",
"html",
"button",
OVERLAY_HOST_MARKER_ATTR,
"",
"visible",
"1",
"static",
"auto",
],
documents: frames
.filter(
(frame) =>
cdpTargetKey(frame.target) === cdpTargetKey(target) &&
frame.frameId !== options.omitDocument,
)
.map((frame) => {
const element = elements.get(frame.frameId)!;
return {
frameId: frame.frameId,
nodes: {
backendNodeId: [element - 1, element + 10000, element, element + 1],
nodeName: [0, 1, 1, 2],
nodeType: [9, 10, 1, 1],
parentIndex: [-1, 0, 0, 2],
attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []],
},
layout: {
nodeIndex: [2, 3],
bounds: [
[0, 0, 1000, 800],
[10, 20, 100, 40],
],
styles: [
[7, 8, 8, 5, 6],
[7, 8, 8, 5, 6],
],
},
};
}),
};
}
if (method === "Accessibility.getFullAXTree") {
const frameId = String(args.frameId);
result = {
nodes: [
{
nodeId: `${frameId}-button`,
frameId,
backendDOMNodeId: elements.get(frameId)! + 1,
role: { value: "button" },
name: { value: frameId },
},
],
};
}
return result as T;
},
);
const cdp: CdpRunner = {
getAttachmentId: () =>
options.attachmentChanged && snapshots === targetCount ? "new-attachment" : "attachment",
getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }),
send: (tabId, method, params) =>
(send as NonNullable<CdpRunner["sendToTarget"]>)({ tabId }, method, params),
sendToTarget: send as NonNullable<CdpRunner["sendToTarget"]>,
};
return { cdp, logs, elements };
}
describe("captureObservationFacts", () => {
it("collects each target once, validates every document and scopes equal backend IDs", async () => {
const { cdp, logs } = fixture();
const facts = await captureObservationFacts(cdp, 4);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4);
expect(logs.filter((call) => call.method === "DOM.describeNode")).toHaveLength(8);
expect(logs.filter((call) => call.method === "Runtime.releaseObject")).toHaveLength(8);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
expect(main.index.children.get(1)).toEqual([2]);
expect(main.index.nodes.get(10001)?.nodeType).toBe(10);
expect(main.identity?.documentElementBackendNodeId).toBe(1);
expect(remote.axNodes[0].frameId).toBe("remote");
expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2));
expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt);
});
it.each([
{ loader: "new-loader" },
{ element: 99 },
{ missing: true },
])("isolates changed child identity %j and dependent descendants", async (change) => {
const { cdp } = fixture({ after: { same: change } });
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "same", reason: "document-changed" }),
);
});
it.each([
{ after: { main: { element: 99 } } },
{ attachmentChanged: true },
])("rejects a root replacement or reattachment %j", async (options) => {
await expect(captureObservationFacts(fixture(options).cdp, 4)).rejects.toThrow(
"document identity",
);
});
it("does not retry failed or missing documents and retains valid AX-only semantics", async () => {
const { cdp, logs } = fixture({
fail: "remote:DOMSnapshot.captureSnapshot",
omitDocument: "same",
});
const facts = await captureObservationFacts(cdp, 4);
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(remote.domNodes).toHaveLength(0);
expect(remote.axNodes).toHaveLength(1);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }),
);
expect(facts.issues).toContainEqual(
expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }),
);
});
it("keeps missing identity explicit without manufacturing a verified document", async () => {
const facts = await captureObservationFacts(fixture({ missingIdentity: true }).cdp, 4);
expect(facts.documents.every((doc) => !doc.identity)).toBe(true);
expect(facts.issues.filter((issue) => issue.reason === "identity-unverified")).toHaveLength(4);
});
it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => {
const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4);
const { documents } = semanticCapture(facts);
const graph = buildSemanticGraph({
documents,
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
});
expect(
[...graph.nodes.values()].find(
(node) => node.frameId === "remote" && node.backendNodeId === 2,
)?.excluded,
).toBe(true);
expect(
[...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2)
?.excluded,
).toBe(false);
});
it("bounds collection across many targets and stops scheduling after cancellation", async () => {
const frames = Array.from({ length: 12 }, (_, i) => ({
frameId: `f${i}`,
target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) },
}));
const { cdp } = fixture({ frames });
let active = 0,
peak = 0;
const original = cdp.sendToTarget!;
const send: NonNullable<CdpRunner["sendToTarget"]> = async (target, method, params) => {
active++;
peak = Math.max(peak, active);
try {
await new Promise((resolve) => setTimeout(resolve, 1));
return await original(target, method, params);
} finally {
active--;
}
};
cdp.sendToTarget = send;
cdp.send = (tabId, method, params) => send({ tabId }, method, params);
await captureObservationFacts(cdp, 4);
// This fixture has no frame edges, so no concurrent owner requests.
expect(peak).toBeLessThanOrEqual(4);
expect(peak).toBeGreaterThan(1);
const controller = new AbortController();
controller.abort();
await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(active).toBe(0);
});
});
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4));
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
trees
.find((doc) => doc.frameId === "left")
?.domNodes?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
trees
.find((doc) => doc.frameId === "right")
?.domNodes?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(
new Map(
trees
.filter((doc) => doc.ownerBackendNodeId !== undefined)
.map((doc) => [doc.frameId, doc.ownerBackendNodeId]),
),
).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(
new Map(
trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]),
),
).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const { documents } = semanticCapture(await captureObservationFacts(cdp, 4));
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
});
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
// Minimal but format-accurate captureSnapshot reply: a body with one // Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input. // fixed full-screen overlay div carrying a password input.
@@ -26,6 +27,7 @@ function fakeSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2], parentIndex: [-1, 0, 1, 2],
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
@@ -73,6 +75,7 @@ function hoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1], nodeType: [1, 1, 1],
@@ -117,6 +120,7 @@ function twoHoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1], parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
@@ -170,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2, 3], parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1], nodeType: [1, 1, 1, 1, 1],
@@ -243,13 +248,11 @@ function makeCdp(snapshot: unknown) {
}; };
} }
describe("captureViewModel", () => { describe("snapshot capture and hover", () => {
it("parses nodes, attrs, rects, paint order and styles", async () => { it("parses nodes, attrs, rects, paint order and styles", async () => {
const { nodes, viewport, iframeNodes } = await captureViewModel( const { nodes, viewport } = semanticCapture(
makeCdp(fakeSnapshotReply()), await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
4, ).captured;
);
expect(iframeNodes.size).toBe(0);
expect(viewport).toEqual({ width: 1000, height: 800 }); expect(viewport).toEqual({ width: 1000, height: 800 });
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 }); expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
@@ -261,35 +264,11 @@ describe("captureViewModel", () => {
it("never hovers the page while capturing", async () => { it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply()); const cdp = makeCdp(fakeSnapshotReply());
await captureViewModel(cdp, 4); semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
}); });
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formPlaceholder: "secret",
attrs: { type: "password" },
});
expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({
serializationOptions: expect.objectContaining({ serialization: "deep" }),
}),
);
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => { it("runs hover surface probes when explicitly enabled", async () => {
let hoverStateCalls = 0; let hoverStateCalls = 0;
const cdp = { const cdp = {
@@ -329,7 +308,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([ expect(surfaceProbes).toEqual([
@@ -389,7 +368,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([ expect(surfaceProbes).toEqual([
@@ -434,7 +413,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
await probeHoverSurfaces(cdp, 4, captured.nodes); await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1); expect(hoverMoves).toBe(1);
@@ -466,6 +445,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -496,7 +476,7 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
const ids = nodes.map((n) => n.backendNodeId); const ids = nodes.map((n) => n.backendNodeId);
// Real page nodes survive. // Real page nodes survive.
expect(ids).toContain(12); expect(ids).toContain(12);
@@ -527,6 +507,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -557,7 +538,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
}); });
@@ -578,6 +561,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)], nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
@@ -600,7 +584,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13])); expect(excludedBackendNodeIds).toEqual(new Set([13]));
}); });
@@ -623,6 +609,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -645,7 +632,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13])); expect(excludedBackendNodeIds).toEqual(new Set([13]));
}); });
@@ -668,6 +657,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -690,7 +680,7 @@ describe("captureViewModel", () => {
}, },
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer"); expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
}); });
@@ -702,6 +692,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1, 1], parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
@@ -728,7 +719,7 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
@@ -748,19 +739,20 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
expect(div?.localRect?.y).toBe(-200); expect(div?.localRect?.y).toBe(-200);
expect(div?.rect).toMatchObject({ y: 0, h: 600 }); expect(div?.rect).toMatchObject({ y: 0, h: 600 });
}); });
it("normalizes device-pixel bounds by devicePixelRatio", async () => { it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -795,13 +787,19 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
x: 0, x: 0,
y: 0, y: 0,
w: 1000, w: 1000,
h: 800, h: 800,
}); });
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 2000,
h: 1600,
});
}); });
it("extracts textContent from #text child nodes via nodeValue", async () => { it("extracts textContent from #text child nodes via nodeValue", async () => {
@@ -825,6 +823,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -863,7 +862,7 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const btn = nodes.find((n) => n.backendNodeId === 12); const btn = nodes.find((n) => n.backendNodeId === 12);
expect(btn?.textContent).toBe("登录"); expect(btn?.textContent).toBe("登录");
const link = nodes.find((n) => n.backendNodeId === 14); const link = nodes.find((n) => n.backendNodeId === 14);
@@ -873,7 +872,7 @@ describe("captureViewModel", () => {
expect(body?.textContent).toBeUndefined(); expect(body?.textContent).toBeUndefined();
}); });
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => { it("retains explicit iframe document ownership and projected nodes", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -950,36 +949,33 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { const facts = await captureObservationFacts(cdp, 4);
nodes, const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
iframeNodes, const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
frameNodes, expect(facts.documents).toHaveLength(2);
frameOwnerBackendNodeIds, expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
frameParentIds, const subNodes = child.domNodes;
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
const input = subNodes.find((n) => n.backendNodeId === 101); const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input"); expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text"); expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13); expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 }); expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 }); expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame"); expect(facts.rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes); expect(child.frame.ownerBackendNodeId).toBe(13);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13); expect(child.frame.parentFrameId).toBe("main-frame");
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
}); });
it("captures iframe documents without requiring owner-frame geometry", async () => { it("projects iframe documents independently of owner snapshot geometry", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -997,6 +993,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -1014,6 +1011,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "child",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -1040,15 +1038,23 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { documents } = await captureObservationFacts(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); const input = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toBeNull(); expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
}); });
it("recursively normalizes nested iframe sub-documents", async () => { it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1069,6 +1075,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")], nodeName: [i("html"), i("body"), i("iframe")],
@@ -1090,6 +1097,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "child",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")], nodeName: [i("html"), i("body"), i("iframe")],
@@ -1111,6 +1119,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "nested",
nodes: { nodes: {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [i("body"), i("input")], nodeName: [i("body"), i("input")],
@@ -1127,21 +1136,52 @@ describe("captureViewModel", () => {
], ],
}; };
const cdp = { const cdp = {
send: vi.fn(async (_t: number, method: string) => { send: vi.fn(
if (method === "DOMSnapshot.enable") return {}; async (
if (method === "DOMSnapshot.captureSnapshot") return snapshot; _t: number,
if (method === "Page.getLayoutMetrics") { method: string,
return { params?: { backendNodeId?: number; objectId?: string },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, ) => {
}; if (method === "DOMSnapshot.enable") return {};
} if (method === "DOMSnapshot.captureSnapshot") return snapshot;
throw new Error(method); if (method === "Page.getLayoutMetrics") {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return {
model: {
content:
params?.backendNodeId === 13
? [10, 20, 310, 20, 310, 220, 10, 220]
: [15, 26, 115, 26, 115, 106, 15, 106],
},
};
if (method === "DOM.resolveNode")
return { object: { objectId: String(params?.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return {
result: {
value:
params?.objectId === "13"
? { width: 300, height: 200 }
: { width: 100, height: 80 },
},
};
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
},
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { documents } = await captureObservationFacts(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23); const nestedIframe = documents
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31); .find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 23);
const input = documents
.find((doc) => doc.frame.frameId === "nested")
?.domNodes?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 }); expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 }); expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
@@ -1150,13 +1190,14 @@ describe("captureViewModel", () => {
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
}); });
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -1191,8 +1232,8 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
}); });
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => { it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -0,0 +1,156 @@
import { describe, expect, it } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
backendNodeId: id,
parentBackendNodeId: parent,
tag: "div",
attrs: {},
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("document facts", () => {
it.each([
1_000, 10_000, 100_000,
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
for (const wide of [false, true]) {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
return i ? (wide ? 0 : i - 1) : null;
},
});
return item;
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.ancestry.get(count - 1)).toEqual({ complete: true, overlay: false });
expect(index.children.get(0)?.length).toBe(wide ? count - 1 : 1);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("marks cycles and orphans incomplete and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
{ ...node(2, 1), tag: "#document-fragment" },
host,
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 6),
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
for (const id of [4, 5, 6, 7]) expect(index.ancestry.get(id)?.complete).toBe(false);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("does not treat a missing snapshot parent as a complete root", async () => {
const { nodes } = await decodeDocument(
{ nodes: { backendNodeId: [1], nodeName: [0], parentIndex: [99] } },
["div"],
);
const index = await buildDocumentIndex(nodes);
expect(index.ancestry.get(1)?.complete).toBe(false);
});
it("retains raw box units and independent style/interaction evidence", async () => {
const strings = [
"html",
"canvas",
"aria-hidden",
"true",
"inert",
"",
"hidden",
"visible",
"0",
"auto",
"scroll",
"matrix(2,0,0,2,0,0)",
];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility"
? 7
: style === "opacity"
? 8
: style === "overflow-x"
? 10
: style === "transform"
? 11
: 9,
);
const { nodes } = await decodeDocument(
{
nodes: {
backendNodeId: [1, 2],
parentIndex: [-1, 0],
nodeName: [0, 1],
attributes: [[2, 3, 4, 5], []],
},
layout: {
nodeIndex: [0, 1],
bounds: [
[0, 0, 200, 100],
[10, 20, 120, 40],
],
clientRects: [
[5, 5, 90, 40],
[0, 0, 120, 40],
],
styles: [styles, styles],
},
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toMatchObject({
boundsSpace: "snapshot-document-css",
clientSpace: "unscaled-client-offset-css",
clientRect: [5, 5, 90, 40],
styles: {
visibility: "visible",
opacity: "0",
"overflow-x": "scroll",
transform: "matrix(2,0,0,2,0,0)",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("yields during large inputs so cancellation interrupts indexing", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
for (const item of input)
Object.defineProperty(item, "attrs", {
get: () => {
reads++;
if (reads === 100) controller.abort();
return {};
},
});
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(reads).toBeLessThanOrEqual(356);
});
});
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared"; import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { captureViewModel } from "../capture"; import { captureObservationFacts } from "../capture-coordinator";
interface Control { interface Control {
id: number; id: number;
@@ -106,17 +106,19 @@ describe("form state identity", () => {
[11, state("alice@example.com")], [11, state("alice@example.com")],
]), ]),
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({ expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "", formValue: "",
formState: "empty", formState: "empty",
formPlaceholder: "snapshot placeholder", formPlaceholder: "snapshot placeholder",
}); });
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com", formValue: "alice@example.com",
}); });
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" }); expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)).toMatchObject({
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false); formValue: "上海",
});
expect(captured.documents[0].domNodes.some((n) => n.backendNodeId === 99)).toBe(false);
}); });
it("does not shift state between frames when one frame is absent from the runtime batch", async () => { it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
@@ -133,8 +135,10 @@ describe("form state identity", () => {
[21, state("second frame")], [21, state("second frame")],
]), ]),
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
const controls = [...captured.iframeNodes.values()].flat(); const controls = captured.documents
.filter((doc) => doc.frame.parentFrameId)
.flatMap((doc) => doc.domNodes);
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame"); expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame"); expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame"); expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
@@ -151,8 +155,13 @@ describe("form state identity", () => {
}); });
const cdp = { send, sendToTarget } as unknown as CdpRunner; const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) { for (const sessionId of ["left-frame", "right-frame"]) {
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7); const captured = await captureObservationFacts(
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId); cdpRunnerForTarget(cdp, { tabId: 7, sessionId }),
7,
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
sessionId,
);
expect(sendToTarget).toHaveBeenCalledWith( expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId }, { tabId: 7, sessionId },
"Runtime.releaseObjectGroup", "Runtime.releaseObjectGroup",
@@ -178,8 +187,8 @@ describe("form state identity", () => {
result.result.deepSerializedValue.value[0].value[1].value = "not json"; result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result; return result;
}); });
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value", formValue: "snapshot value",
formPlaceholder: "snapshot placeholder", formPlaceholder: "snapshot placeholder",
}); });
@@ -192,7 +201,7 @@ describe("form state identity", () => {
controller.abort(); controller.abort();
return reply([[11, state("new")]]); return reply([[11, state("new")]]);
}); });
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({ await expect(captureObservationFacts(cdp, 7, controller.signal)).rejects.toMatchObject({
name: "AbortError", name: "AbortError",
}); });
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
@@ -216,9 +225,13 @@ describe("form state identity", () => {
return result; return result;
}, },
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value"); expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value"); "snapshot value",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)?.formValue).toBe(
"live value",
);
}); });
it.each([ it.each([
@@ -228,8 +241,8 @@ describe("form state identity", () => {
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () => const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]), reply([[11, { ...state("secret"), sensitive: true }]]),
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
const control = captured.nodes.find((n) => n.backendNodeId === 11); const control = captured.documents[0].domNodes.find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled"); expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined(); expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined(); expect(control?.formDefaultValue).toBeUndefined();
@@ -257,9 +270,13 @@ describe("form state identity", () => {
expect(entries).toHaveLength(250); expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)])); return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
}); });
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250"); expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 250)?.formValue).toBe(
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old"); "live 250",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 251)?.formValue).toBe(
"old",
);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1); expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect( expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"), send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
@@ -1,198 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const documents = await captureFrameData(cdp, 4, captured);
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
});
@@ -1,7 +1,12 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture"; import type { CapturedNode } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document"; import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDomInput,
type FrameOwnedAxNode,
} from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode { function domNode(frameId: string, backendNodeId: number): CapturedNode {
return { return {
@@ -17,19 +22,16 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
}; };
} }
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel { function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
return { return {
nodes: frameNodes.get("main") ?? [], nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes, frameNodes,
rootFrameId: "main", rootFrameId: "main",
excludedBackendNodeIds: new Set(),
}; };
} }
describe("buildFrameDocuments", () => { describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => { it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -54,7 +56,7 @@ describe("buildFrameDocuments", () => {
}, },
]; ];
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -72,7 +74,7 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child"); expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
}); });
it("deduplicates overlapping same-target AX results using the strongest ownership", () => { it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -92,7 +94,7 @@ describe("buildFrameDocuments", () => {
}, },
]; ];
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -107,7 +109,7 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1); expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
}); });
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => { it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -129,7 +131,7 @@ describe("buildFrameDocuments", () => {
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }], nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
})); }));
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -149,7 +151,7 @@ describe("buildFrameDocuments", () => {
); );
}); });
it("uses captured frame parent identity instead of guessing from backend ids", () => { it("uses captured frame parent identity instead of guessing from backend ids", async () => {
const frameNodes = new Map([ const frameNodes = new Map([
["main", [domNode("main", 1)]], ["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]], ["left", [domNode("left", 7)]],
@@ -164,11 +166,42 @@ describe("buildFrameDocuments", () => {
nodes: [], nodes: [],
})); }));
const documents = buildFrameDocuments(null, frames, capture); const documents = await buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({ expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left", parentFrameId: "left",
ownerBackendNodeId: 7, ownerBackendNodeId: 7,
}); });
}); });
it("rejects explicit AX frame ownership in a different target", async () => {
const frames = [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
];
const unresolved = vi.fn();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames },
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
captured(new Map()),
undefined,
unresolved,
);
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
expect(unresolved).toHaveBeenCalledOnce();
});
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
nodeId: String(i),
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
})).reverse();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames: [frame] },
[{ frame, nodes }],
captured(new Map()),
);
expect(documents[0].axNodes).toHaveLength(10000);
expect(documents[0].axNodes[0].frameId).toBe("main");
});
}); });
@@ -0,0 +1,382 @@
import {
buildFrameGraph,
type CdpFrame,
type CdpFrameGraph,
type CdpFrameTreeNode,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { collectOverlayExcludedBackendIds } from "./capture";
import { readDocumentIdentity, sameDocument } from "./document-identity";
import type { CapturedSceneInput } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
type DocumentIdentity,
isCaptureAbort,
type ObservationFacts,
throwCaptureAborted,
} from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
import { REQUESTED_STYLES, type SnapshotReply, snapshotFrameId } from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
before: Map<string, DocumentIdentity>;
documents: NormalizedDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
}
/** A fixed worker pool scoped to one capture; each worker issues at most one
* collection request at a time. Geometry has its own measurement-phase bound. */
async function collectTargets<T>(
items: readonly T[],
collect: (item: T) => Promise<void>,
signal?: AbortSignal,
): Promise<void> {
let cursor = 0;
await Promise.all(
Array.from({ length: Math.min(4, items.length) }, async () => {
while (cursor < items.length) {
throwCaptureAborted(signal);
const item = items[cursor++];
await collect(item);
}
}),
);
}
/** When topology discovery is unavailable, snapshot documents still carry
* explicit target/frame provenance. Only explicit contentDocumentIndex edges
* establish ancestry; a foreign frame is never renamed to fit a request. */
function snapshotFrames(snapshot: SnapshotReply, target: CdpTarget): CdpFrame[] {
const raw = snapshot.documents ?? [];
const strings = snapshot.strings ?? [];
const frames = raw.map((doc) => {
const frameId = snapshotFrameId(doc, strings);
return frameId ? ({ frameId, target } as CdpFrame) : undefined;
});
for (let i = 0; i < raw.length; i++) {
const edges = raw[i].nodes?.contentDocumentIndex;
const parent = frames[i];
if (!parent || !edges) continue;
for (let e = 0; e < edges.index.length; e++) {
const child = frames[edges.value[e]];
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
if (child && owner !== undefined && child !== parent) {
child.parentFrameId = parent.frameId;
child.ownerBackendNodeId = owner;
}
}
}
return frames.filter((frame): frame is CdpFrame => !!frame);
}
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
): Promise<ObservationFacts<T>> {
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
try {
graph = await cdp.getFrameGraph?.(tabId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const geometry = new GeometryContext(cdp, tabId, graph, signal);
const issues: CaptureIssue[] = [];
const groups = new Map<string, TargetBatch<T>>();
for (const frame of graph?.frames ?? []) {
const key = cdpTargetKey(frame.target);
let group = groups.get(key);
if (!group) {
group = {
target: frame.target,
frames: [],
before: new Map(),
documents: [],
ax: [],
fallbackExcluded: new Set(),
};
groups.set(key, group);
}
group.frames.push(frame);
}
if (!groups.size)
groups.set(cdpTargetKey({ tabId }), {
target: { tabId },
frames: [],
before: new Map(),
documents: [],
ax: [],
fallbackExcluded: new Set(),
});
const batches = [...groups.values()];
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
let firstFailure: unknown;
await collectTargets(
batches,
async (batch) => {
const scoped = cdpRunnerForTarget(cdp, batch.target);
for (const frame of batch.frames) {
const identity = await readDocumentIdentity(cdp, frame, signal);
if (identity) batch.before.set(frame.frameId, identity);
}
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwCaptureAborted(signal);
if (!graph) batch.frames = snapshotFrames(snapshot, batch.target);
batch.documents = await normalizeSnapshot(
snapshot,
batch.target,
batch.frames,
geometry,
issues,
signal,
);
if (
!(await enrichFormControlStates(
scoped,
tabId,
batch.documents.map((doc) => doc.domNodes),
signal,
))
)
unavailable(batch, "forms");
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
for (const frame of batch.frames)
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "dom");
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
}
if (!batch.frames.length && !graph)
batch.frames = [{ frameId: "root", target: batch.target }];
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "Accessibility.enable", {});
} catch (error) {
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax");
return;
}
for (const frame of batch.frames) {
try {
throwCaptureAborted(signal);
const result = await scoped.send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
batch.ax.push({ frame, nodes: result.nodes ?? [] });
} catch (error) {
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
}
},
signal,
);
// Validate after every target's static data and geometry have been collected.
const identities = new Map<string, DocumentIdentity>();
const invalid = new Set<string>();
await collectTargets(
batches,
async (batch) => {
let fresh: Map<string, CdpFrame> | undefined;
if (batch.before.size) {
try {
throwCaptureAborted(signal);
const reply = await sendToCdpTarget<{ frameTree?: CdpFrameTreeNode }>(
cdp,
batch.target,
"Page.getFrameTree",
{},
);
const tree = reply.frameTree
? buildFrameGraph([{ target: batch.target, tree: reply.frameTree }])
: null;
fresh = new Map(tree?.frames.map((frame) => [frame.frameId, frame]));
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc]));
for (const frame of batch.frames) {
const before = batch.before.get(frame.frameId);
const currentFrame = fresh?.get(frame.frameId);
const after = currentFrame
? await readDocumentIdentity(cdp, currentFrame, signal)
: undefined;
const doc = docs.get(frame.frameId);
const snapshotMismatch =
before &&
doc?.documentElementBackendNodeId !== undefined &&
before.documentElementBackendNodeId !== doc.documentElementBackendNodeId;
if (
before &&
(snapshotMismatch ||
(fresh && !currentFrame) ||
(after && !sameDocument(before, after)) ||
cdp.getAttachmentId?.(tabId) !== before.attachmentId)
) {
invalid.add(frame.frameId);
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "document-changed",
});
} else if (before && !after) {
invalid.add(frame.frameId);
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "identity-unverified",
});
} else if (
before &&
after &&
sameDocument(before, after) &&
(!doc || doc.documentElementBackendNodeId === before.documentElementBackendNodeId)
)
identities.set(frame.frameId, after);
else
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "identity-unverified",
});
}
},
signal,
);
const allFrames = batches.flatMap((batch) => batch.frames);
const rootFrameId = graph?.rootFrameId ?? allFrames[0]?.frameId ?? "root";
if (invalid.has(rootFrameId))
throw new Error(
"observation document identity changed or could not be verified; observe again",
);
// A descendant cannot retain geometry measured through an invalid ancestor.
const childFrames = new Map<string, string[]>();
for (const frame of allFrames)
if (frame.parentFrameId) {
const children = childFrames.get(frame.parentFrameId);
if (children) children.push(frame.frameId);
else childFrames.set(frame.parentFrameId, [frame.frameId]);
}
const queue = [...invalid];
for (let i = 0; i < queue.length; i++)
for (const child of childFrames.get(queue[i]) ?? []) {
if (!invalid.has(child)) {
invalid.add(child);
queue.push(child);
}
}
const frames = allFrames.filter((frame) => !invalid.has(frame.frameId));
const decoded = new Map(
batches
.flatMap((batch) => batch.documents)
.filter((doc) => !invalid.has(doc.frame.frameId))
.map((doc) => [doc.frame.frameId, doc]),
);
const ax = batches
.flatMap((batch) => batch.ax)
.filter((batch) => !invalid.has(batch.frame.frameId));
const documents = await buildFrameDocuments(
{ rootFrameId, frames },
ax,
{
nodes: [],
rootFrameId,
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.domNodes])),
},
signal,
(frame) =>
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
}),
);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const facts: ObservationFacts<T>["documents"][number][] = [];
for (const document of documents) {
const doc = decoded.get(document.frameId);
const index = doc?.index ?? (await buildDocumentIndex([], signal));
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
const { domNodes, axNodes, contextScopeId: _scope, ...frame } = document;
facts.push({
frame,
identity: identities.get(document.frameId),
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
});
}
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
}
/** Existing semantic consumers get a narrow view; no raw snapshot or identity
* metadata can leak through recording's allowlisted output DTO. */
export function semanticCapture<T extends FrameOwnedAxNode>(
facts: ObservationFacts<T>,
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
const documents = facts.documents.map((doc) => ({
...doc.frame,
contextScopeId: doc.frame.frameId,
domNodes: doc.domNodes.filter(
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
),
axNodes: doc.axNodes,
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
}));
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
return {
documents,
captured: {
nodes: root?.domNodes ?? [],
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
},
};
}
+11 -715
View File
@@ -1,332 +1,21 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics // Browser-side hover probes and overlay exclusion fallback. Static collection
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches // lives in capture-coordinator; snapshot decoding has no browser dependency.
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import type { CapturedNode, CapturedSurfaceProbe } from "./facts";
export type {
CapturedNode,
CapturedSurfaceProbe,
} from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
signal?: AbortSignal;
}
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
interface LayoutMetricsReply {
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// use their own captureViewModel call and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
}
interface RuntimeEvaluateReply { interface RuntimeEvaluateReply {
result?: { result?: {
value?: unknown; value?: unknown;
deepSerializedValue?: DeepSerializedValue;
}; };
} }
@@ -345,44 +34,6 @@ interface CdpDomNode {
shadowRoots?: CdpDomNode[]; shadowRoots?: CdpDomNode[];
} }
interface ParseDocumentResult {
nodes: CapturedNode[];
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: GeometryProjection | null;
scrollX: number;
scrollY: number;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function devicePixelRatio(metrics: LayoutMetricsReply): number {
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
if (!layoutW || !cssW) return 1;
const dpr = layoutW / cssW;
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
return dpr >= 1 ? dpr : 1;
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void { function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return; if (!node) return;
if (typeof node.backendNodeId === "number") { if (typeof node.backendNodeId === "number") {
@@ -791,358 +442,3 @@ export async function collectOverlayExcludedBackendIds(
} }
return excluded; return excluded;
} }
/**
* Parse one DOMSnapshot document into CapturedNode[].
*
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
dpr: number,
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [], excludedBackendNodeIds: new Set() };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
// is injected into the page and its open shadow root
// is inlined by DOMSnapshot as descendants of the host. Its fixed
// full-viewport click-blocker would otherwise dominate occlusion detection
// and hide the real page, so mark the host + its whole subtree excluded.
// parentIndex always references an earlier node, so one forward pass suffices.
const excluded = new Array<boolean>(count).fill(false);
const excludedBackendNodeIds = new Set<number>();
for (let n = 0; n < count; n++) {
const parentIdx = dn.parentIndex?.[n] ?? -1;
const inherited = parentIdx >= 0 && excluded[parentIdx];
let isOverlayHost = inherited;
if (!isOverlayHost) {
const pairs = dn.attributes?.[n] ?? [];
const attrNames: string[] = [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrNames.push(str(strings, pairs[a]));
}
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
}
if (!isOverlayHost) continue;
excluded[n] = true;
const backendNodeId = dn.backendNodeId[n];
if (backendNodeId !== undefined) {
excludedBackendNodeIds.add(backendNodeId);
}
}
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
}
}
}
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
w: b[2] / dpr,
h: b[3] / dpr,
};
const bounds = context.projection
? projectRectToViewport(localRect, context.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
if (tag.startsWith("#")) continue;
const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
cursor,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes, excludedBackendNodeIds };
}
interface ParseFrameDocumentsResult {
iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
excludedBackendNodeIds: Set<number>;
}
function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) {
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
const childDoc = documents[childDocIndex];
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
const projection =
parentContext.projection && iframeNode?.localRect
? childFrameProjection(parentContext.projection, iframeNode.localRect)
: null;
const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings),
ownerFrameBackendNodeId: iframeBackendId,
projection,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments(
documents,
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
nextVisited,
);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
}
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
export async function captureViewModel(
cdp: CdpRunner,
tabId: number,
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
throwIfAborted(options.signal);
let metrics: LayoutMetricsReply = {};
try {
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal);
const dpr = devicePixelRatio(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const viewport: Viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
};
const scrollX = vpSrc.pageX ?? 0;
const scrollY = vpSrc.pageY ?? 0;
await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal);
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwIfAborted(options.signal);
const strings = snap.strings ?? [];
const documents = snap.documents ?? [];
const doc0 = documents[0];
if (!doc0?.nodes?.backendNodeId) {
return {
nodes: [],
viewport,
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
};
}
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
projection: {
sourceClips: [],
edges: [],
topViewport: viewport,
},
scrollX,
scrollY,
};
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
throwIfAborted(options.signal);
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
}
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
if (!frameNodes.has(frameId)) {
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
}
}
return {
nodes,
viewport,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
excludedBackendNodeIds,
};
}
@@ -0,0 +1,65 @@
import type { CdpFrame } from "@/browser-driver/frame-graph";
import type { CdpRunner } from "../shared";
import { sendToCdpTarget } from "../shared";
import { type DocumentIdentity, isCaptureAbort, throwCaptureAborted } from "./facts";
/** Fresh, frame-scoped identity read. An isolated world supplies the owning
* document without querying each node or installing a page observer. */
export async function readDocumentIdentity(
cdp: CdpRunner,
frame: CdpFrame,
signal?: AbortSignal,
): Promise<DocumentIdentity | undefined> {
const attachmentId = cdp.getAttachmentId?.(frame.target.tabId);
if (!attachmentId || !frame.loaderId) return undefined;
let objectId: string | undefined;
const send = <T>(method: string, params: object) => {
throwCaptureAborted(signal);
return sendToCdpTarget<T>(cdp, frame.target, method, params);
};
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: frame.frameId,
worldName: "bsk-document-identity",
});
const element = await send<{ result?: { objectId?: string } }>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
});
objectId = element.result?.objectId;
if (!objectId) return undefined;
const reply = await send<{ node?: { backendNodeId?: number } }>("DOM.describeNode", {
objectId,
depth: 0,
});
const backendNodeId = reply.node?.backendNodeId;
if (!backendNodeId || cdp.getAttachmentId?.(frame.target.tabId) !== attachmentId)
return undefined;
return {
attachmentId,
target: frame.target,
frameId: frame.frameId,
loaderId: frame.loaderId,
documentElementBackendNodeId: backendNodeId,
};
} catch (error) {
if (isCaptureAbort(error)) throw error;
return undefined;
} finally {
if (objectId)
await sendToCdpTarget(cdp, frame.target, "Runtime.releaseObject", { objectId }).catch(
() => {},
);
}
}
export function sameDocument(a: DocumentIdentity, b: DocumentIdentity): boolean {
return (
a.attachmentId === b.attachmentId &&
a.target.tabId === b.target.tabId &&
a.target.sessionId === b.target.sessionId &&
a.frameId === b.frameId &&
a.loaderId === b.loaderId &&
a.documentElementBackendNodeId === b.documentElementBackendNodeId
);
}
+221
View File
@@ -0,0 +1,221 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameOwnedAxNode } from "./frame-document";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DocumentIdentity {
attachmentId: string;
target: CdpTarget;
frameId: string;
loaderId: string;
documentElementBackendNodeId: number;
}
/** Snapshot client/offset rectangles retain their protocol units. In particular,
* transformed bounds cannot be combined with unscaled client offsets. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
readonly clientSpace: "unscaled-client-offset-css";
bounds?: number[];
clientRect?: number[];
offsetRect?: number[];
scrollRect?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface AncestorState {
complete: boolean;
overlay: boolean;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly children: ReadonlyMap<number, readonly number[]>;
readonly ancestry: ReadonlyMap<number, AncestorState>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DecodedDocument {
nodes: DecodedNode[];
}
export interface CaptureIssue {
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms";
reason:
| "capture-unavailable"
| "document-changed"
| "identity-unverified"
| "frame-ownership-unresolved"
| "geometry-unavailable";
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly frame: CdpFrame;
readonly identity?: DocumentIdentity;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export function throwCaptureAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
export function isCaptureAbort(error: unknown): boolean {
return error instanceof Error && error.name === "AbortError";
}
/** Called only at bounded block boundaries, not once per node. */
export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
throwCaptureAborted(signal);
await new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
});
throwCaptureAborted(signal);
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves ancestry evidence even when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const nodes = new Map<number, T>();
const children = new Map<number, number[]>();
const ancestry = new Map<number, AncestorState>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = input[i];
nodes.set(node.backendNodeId, node);
if (node.parentBackendNodeId !== null) {
const siblings = children.get(node.parentBackendNodeId);
if (siblings) siblings.push(node.backendNodeId);
else children.set(node.parentBackendNodeId, [node.backendNodeId]);
}
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (ancestry.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let state: AncestorState = { complete: true, overlay: false };
while (current && !ancestry.has(current.backendNodeId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (visiting.has(current.backendNodeId)) {
// No member of a malformed cycle supplies complete ancestry evidence.
state = {
complete: false,
overlay: path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))),
};
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentMissing) state = { complete: false, overlay: false };
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
if (!current) state = { complete: false, overlay: false };
}
if (current && ancestry.has(current.backendNodeId))
state = ancestry.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const item = path[i];
state = {
complete: state.complete,
overlay: state.overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)),
};
ancestry.set(item.backendNodeId, state);
if (state.overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, children, ancestry, excludedBackendNodeIds };
}
@@ -0,0 +1,154 @@
import type { CdpRunner } from "../shared";
import type { CapturedNode } from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!["input", "textarea", "select"].includes(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (
state.sensitive ||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
export async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<boolean> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
);
if (!hasControls) return true;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// have separate enrichment calls and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
return true;
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
return false;
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
}
@@ -1,181 +0,0 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveFrameProjection } from "../frame-geometry";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection | null,
): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: projection
? (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})()
: null,
}));
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
});
let projection: GeometryProjection | null = null;
try {
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
frameId: frame.frameId,
err,
});
}
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
if (frame.ownerBackendNodeId !== undefined) {
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
}
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
const trees = await Promise.all(
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<CapturedFrameDocument<T>[]> {
const graph = await discoverFrameGraph(cdp, tabId);
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
}
+73 -42
View File
@@ -4,7 +4,8 @@ import {
type CdpTarget, type CdpTarget,
cdpTargetKey, cdpTargetKey,
} from "@/browser-driver/frame-graph"; } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture"; import type { CapturedNode } from "./facts";
import { captureCheckpoint } from "./facts";
export interface FrameOwnedAxNode { export interface FrameOwnedAxNode {
nodeId: string; nodeId: string;
@@ -23,6 +24,16 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string; contextScopeId: string;
axNodes: T[]; axNodes: T[];
domNodes: CapturedNode[]; domNodes: CapturedNode[];
excludedBackendNodeIds?: ReadonlySet<number>;
}
/** Explicit snapshot document data used to join DOM and AX ownership. */
export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
frameParentIds?: ReadonlyMap<string, string>;
} }
interface Ownership { interface Ownership {
@@ -47,17 +58,13 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>( function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: CapturedViewModel, captured: FrameDomInput,
): CdpFrame[] { ): CdpFrame[] {
const frames = new Map<string, CdpFrame>(); const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) { for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame); if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
} }
for (const frameId of captured.frameNodes?.keys() ?? []) {
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => { return [...frames.values()].map((frame) => {
const ownerBackendNodeId = const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId); frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
@@ -70,11 +77,14 @@ function frameList<T extends FrameOwnedAxNode>(
}); });
} }
export function buildFrameDocuments<T extends FrameOwnedAxNode>( export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: CapturedViewModel, captured: FrameDomInput,
): FrameDocument<T>[] { signal?: AbortSignal,
unresolved?: (frame: CdpFrame) => void,
): Promise<FrameDocument<T>[]> {
let work = 0;
const frames = frameList(graph, batches, captured); const frames = frameList(graph, batches, captured);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
@@ -83,54 +93,74 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>(); const backendOwner = new Map<string, string>();
for (const frame of frames) { for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) { for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId); backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
} }
} }
const candidates = new Map<string, OwnedCandidate<T>>(); const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) { for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node])); const nodeById = new Map<string, T>();
for (const node of batch.nodes) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
nodeById.set(node.nodeId, node);
}
let reported = false;
const ownershipByNodeId = new Map<string, Ownership>(); const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>(); const resolveOwnership = async (node: T): Promise<Ownership> => {
const resolveOwnership = (node: T): Ownership => { const path: T[] = [];
const cached = ownershipByNodeId.get(node.nodeId); const visiting = new Set<string>();
if (cached) return cached; let current: T | undefined = node;
if (node.frameId && frameById.has(node.frameId)) { let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
const ownership = { frameId: node.frameId, strength: 4 }; while (current) {
ownershipByNodeId.set(node.nodeId, ownership); if (work++ % 256 === 0) await captureCheckpoint(signal);
return ownership; const cached = ownershipByNodeId.get(current.nodeId);
} if (cached) {
if (typeof node.backendDOMNodeId === "number") { ownership = cached;
const frameId = backendOwner.get( break;
targetBackendKey(batch.frame.target, node.backendDOMNodeId), }
); if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId);
if (current.frameId) {
const frame = frameById.get(current.frameId);
// An explicit contradiction must not fall through to a guessed owner.
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
if (!reported) {
unresolved?.(batch.frame);
reported = true;
}
ownership = { frameId: "", strength: 0 };
break;
}
ownership = { frameId: frame.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
const frameId =
typeof current.backendDOMNodeId === "number"
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
: undefined;
if (frameId) { if (frameId) {
const ownership = { frameId, strength: 3 }; ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership); ownershipByNodeId.set(current.nodeId, ownership);
return ownership; break;
} }
path.push(current);
current = current.parentId ? nodeById.get(current.parentId) : undefined;
} }
if (node.parentId && !resolving.has(node.nodeId)) { for (let i = path.length - 1; i >= 0; i--) {
const parent = nodeById.get(node.parentId); if (work++ % 256 === 0) await captureCheckpoint(signal);
if (parent) { ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
resolving.add(node.nodeId); ownershipByNodeId.set(path[i].nodeId, ownership);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
} }
const ownership = { frameId: batch.frame.frameId, strength: 1 }; if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
ownershipByNodeId.set(node.nodeId, ownership);
return ownership; return ownership;
}; };
for (const node of batch.nodes) { for (const node of batch.nodes) {
const ownership = resolveOwnership(node); if (work++ % 256 === 0) await captureCheckpoint(signal);
const ownership = await resolveOwnership(node);
if (!ownership.frameId) continue;
const key = targetNodeKey(batch.frame.target, node.nodeId); const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key); const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) { if (!existing || ownership.strength > existing.ownership.strength) {
@@ -144,6 +174,7 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
); );
const axNodesByFrame = new Map<string, T[]>(); const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) { for (const candidate of candidates.values()) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const { node, target, ownership } = candidate; const { node, target, ownership } = candidate;
const parentFrameId = node.parentId const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId)) ? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
+184
View File
@@ -0,0 +1,184 @@
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
projectSnapshotRect,
type SnapshotProjection,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import type { CapturedNode } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
captureCheckpoint,
type DocumentIndex,
isCaptureAbort,
type NodeFacts,
} from "./facts";
import {
decodeDocument,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
export interface NormalizedDocument {
index: DocumentIndex;
frame: CdpFrame;
domNodes: CapturedNode[];
documentElementBackendNodeId?: number;
}
/** One target's snapshot. Source ownership is checked before interpreting any
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
export async function normalizeSnapshot(
snapshot: SnapshotReply,
target: CdpTarget,
frames: readonly CdpFrame[],
geometry: GeometryContext,
issues: CaptureIssue[],
signal?: AbortSignal,
): Promise<NormalizedDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const sources = new Map<string, SnapshotDocument>();
for (const doc of raw) {
const id = snapshotFrameId(doc, strings);
if (!id || !frameById.has(id) || sources.has(id)) {
issues.push({
target,
frameId: id,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
continue;
}
sources.set(id, doc);
}
let viewport: Viewport = { width: 0, height: 0 };
let scrollX = 0,
scrollY = 0;
try {
const measured = cssViewport(await geometry.layoutMetrics(target));
viewport = { width: measured.width, height: measured.height };
scrollX = measured.scrollX;
scrollY = measured.scrollY;
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
for (const frame of frames) {
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
else {
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame);
else children.set(frame.parentFrameId, [frame]);
}
}
const projections = new Map<string, SnapshotProjection | null>();
const result: NormalizedDocument[] = [];
let targetProjection: GeometryProjection | null = null;
const targetRoot = pending[0];
if (target.sessionId && targetRoot) {
try {
targetProjection = await geometry.targetProjection(targetRoot.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
for (let cursor = 0; cursor < pending.length; cursor++) {
if (cursor % 256 === 0) await captureCheckpoint(signal);
const frame = pending[cursor];
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const parent = frame.parentFrameId ? frameById.get(frame.parentFrameId) : undefined;
let projection: SnapshotProjection | null = null;
if (!parent)
projection = { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } };
else {
const parentProjection = projections.get(parent.frameId)?.geometry;
if (parentProjection && frame.ownerBackendNodeId !== undefined) {
const edge = parentProjection.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: parentProjection.sourceClips;
try {
projection = await geometry.snapshotProjection(
source,
frame.ownerBackendNodeId,
clips,
viewport,
);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
}
projections.set(frame.frameId, projection);
if (!projection || (target.sessionId && !targetProjection))
issues.push({
target,
frameId: frame.frameId,
stage: "geometry",
reason: "geometry-unavailable",
});
const decoded = await decodeDocument(doc, strings, signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = decoded.nodes[i];
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
x: doc.scrollOffsetX ?? (parent ? 0 : scrollX),
y: doc.scrollOffsetY ?? (parent ? 0 : scrollY),
});
const local = input?.rect;
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
if (rect && target.sessionId)
rect = targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
targetProjection,
)
: null;
const styles = node.layout?.styles;
const visibility = styles?.visibility || "visible";
const opacity = styles?.opacity || "1";
nodes.push({
...node,
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
!!local &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
const domNodes = nodes.filter((node) => !node.tag.startsWith("#"));
result.push({
index,
frame,
domNodes,
documentElementBackendNodeId: nodes.find(
(node) =>
node.nodeType === 1 &&
node.parentBackendNodeId !== null &&
index.nodes.get(node.parentBackendNodeId)?.nodeType === 9,
)?.backendNodeId,
});
}
for (const frameId of sources.keys()) {
if (!projections.has(frameId))
issues.push({ target, frameId, stage: "ownership", reason: "frame-ownership-unresolved" });
}
return result;
}
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture"; import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; import type {
FrameDocument as CapturedFrameDocument,
FrameOwnedAxNode as FrameAxNode,
} from "./frame-document";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */ /** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame { export interface CaptureVomFrame {
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
dom, dom,
sourceOrder: index, sourceOrder: index,
excluded: excluded:
document.frameId === rootFrameId && document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true, (document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
}; };
nodes.set(id, node); nodes.set(id, node);
frameNodeIds.push(id); frameNodeIds.push(id);
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
ax, ax,
sourceOrder: document.domNodes.length + index, sourceOrder: document.domNodes.length + index,
excluded: excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" && typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true, (document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
}); });
frameNodeIds.push(id); frameNodeIds.push(id);
} }
+230
View File
@@ -0,0 +1,230 @@
import type { CapturedNode } from "./facts";
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"display",
"overflow-x",
"overflow-y",
"transform",
"zoom",
"clip-path",
"mask-image",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
export interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
nodeType?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
clientRects?: number[][];
offsetRects?: number[][];
scrollRects?: number[][];
};
}
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
export interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
): Promise<DecodedDocument> {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [] };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
layoutByNode.set(dl!.nodeIndex![i], i);
}
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
if (existing) existing.push(text);
else nodeTextContent.set(parentIdx, [text]);
}
}
}
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
clientSpace: "unscaled-client-offset-css" as const,
bounds: dl?.bounds?.[li],
clientRect: dl?.clientRects?.[li],
offsetRect: dl?.offsetRects?.[li],
scrollRect: dl?.scrollRects?.[li],
styles,
};
const textContent = nodeTextContent.get(n)?.join(" ");
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(parentIdx >= 0 && dn.backendNodeId[parentIdx] === undefined
? { parentMissing: true }
: {}),
tag,
attrs,
layout,
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
position: styles.position || "static",
pointerEvents: styles["pointer-events"] || "auto",
cursor: styles.cursor || "auto",
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes };
}
@@ -1,119 +0,0 @@
# Visual observation design fixtures
These are synthetic, site-independent inputs for the visual observation architecture in
[the design](../../../../docs/design/visual-observation.md). They contain no original page
data, account identifiers, external fonts or production selectors. They are not a production
discovery implementation.
Use [the acceptance checklist](../../../../docs/design/visual-observation-acceptance.md)
for each PR's steps and success criteria. No separate report is required.
Historical probe success does not certify future production modules.
## Browser contract probe
Requires Node.js with built-in WebSocket support (Node 22+) and an installed Chrome. No extra
packages, user profile, extension, or login are required.
From the repository root:
```sh
node apps/extension/test-fixtures/visual-observation/probe.mjs /tmp/bsk-visual-evidence
```
On platforms other than macOS, set `CHROME_PATH` to the installed Chrome executable. The probe
starts its own headless Chrome with an ephemeral profile and a loopback fixture server. It
removes the profile and stops Chrome/server on completion or failure. `report.json`, raw CDP
replies and PNGs are written to the explicitly supplied directory; do not commit them.
The probe verifies CSS exposure versus visibility, snapshot units, DPR raster dimensions,
CSS zoom, frame content origins, child scrolling, nested same-target geometry, a confirmed
OOPIF target, and navigation/document.open identity behavior. It records page-scale emulation
separately; that is **not** browser UI zoom. It does not certify production VOM behavior, arbitrary
occlusion, exact clip-path geometry, or all browser versions.
## Minimal end-to-end reproduction
```sh
python3 -m http.server 4177 --bind 127.0.0.1 \
--directory apps/extension/test-fixtures/visual-observation
bsk session start --width 1440 --height 1100
bsk navigate http://127.0.0.1:4177/table.html --session <session>
bsk observe --session <session>
bsk screenshot --session <session> --out /tmp/bsk-table.png
bsk session stop <session>
```
Always stop the session and the local server. Use a foreground Agent Window and confirm the
painted grid is visible before interpreting the observation. `load`, AX toolbar appearance,
and background screenshots alone do not prove a canvas has drawn.
`table.html` embeds `grid.html` through `localhost` from a `127.0.0.1` parent. The grid has DOM
toolbar controls and one unlabeled Canvas with three synthetic columns and six painted rows.
Main baseline: `Page action`, `Insert record`, and `Filter` are referenceable; the painted
`Canvas column A/B/C` and `Canvas row 1..6` have no textual observation representation or visual
ref, despite an untruncated result. This is a **known gap**, not a golden expectation that future
implementations must preserve.
Future acceptance: retain the DOM controls and add a screenshot-only visual ref for the visible
grid, with correct frame projection. Do not invent AX cell text. Validate the corresponding
screenshot, then move/remove/resize and re-observe according to the freshness contract.
## Discovery policy cases
At a 1440×1100 window and without scrolling:
| Canvas ID | Expected candidate in the new design | Reason |
|---|---|---|
| plain | yes | No name required |
| named | yes | Name does not establish complete pixel semantics |
| aria-hidden | yes | Accessibility exclusion is not visual exclusion |
| inert | yes | Inertness is not visual hiding |
| pointer-none | yes | Screenshot observation does not require pointer interaction |
| visibility-override | yes | Child restores computed visibility |
| opacity-parent | no | Ancestor makes subtree transparent |
| display-none | no | No layout box |
| visibility-hidden | no | Computed visibility suppresses this element |
| zero | no | Empty geometry |
| hidden-overridden | yes | CSS display overrides normal hidden presentation |
| partial-clip | yes, 60×30 crop | Supported overflow intersection |
| full-clip | no | Fully outside ancestor clip |
| axis-clip | yes, x clipped only | overflow-x:clip and overflow-y:visible remain independent |
| border-clip | yes, clipped to client box | Ancestor border box is not its scrollport |
| border-canvas | yes, border-box region | Observation and live screenshot resolver must use the same box kind |
| fallback | yes | AX fallback is not full pixel coverage |
| stack-a / stack-b | one output entry after selection | Same document/parent/exact crop; not proof of a logical object |
`checkVisibility()` is used in the probe only for CSS suppression evidence; it does not establish
viewport intersection, ancestor overflow clipping, or absence of occlusion. `frame.html` supplies
known local coordinates, bordered owners, nested frames and scrollable content for independent
geometry verification. The fixture IDs must never enter production discovery code.
## Renderer operation counts
After installing the repository's normal dependencies:
```sh
node apps/extension/test-fixtures/visual-observation/measure-render.mjs /tmp/bsk-render-evidence
```
This instruments a temporary copy of the current renderer, runs the installed Vitest, and writes
the source hash and loop counters to `render-counts.json`. It checks instrumentation sites and
fails explicitly when source structure changes; update the diagnostic deliberately then. It never
modifies production sources and never asserts that today's quadratic costs must persist.
The wide fixture contains N headings followed by N equally named buttons in a common context.
The deep fixture contains 2,000/4,000 transparent wrappers. Counters cover render-stack visits,
ordered DOM context candidates, and same-container sibling visits. They are not a complete CPU
profile or an assertion about the complexity of all VOM stages.
Use these existing-renderer counts only as a before/after regression baseline. Improving the
historical algorithm is not a prerequisite for Canvas work. New Canvas stages need their own
workload models and measurements; do not copy this renderer's algorithms or use these counts
as evidence that visual discovery, clipping, selection, or screenshot execution is efficient.
## Review boundary
Commit only these synthetic fixtures, diagnostic tools, and reviewed design/evidence summaries.
Keep local browser profiles, original internal-site screenshots/HTML/observations, generated
captures, temporary source instrumentation and build outputs outside the changeset.
@@ -1,25 +0,0 @@
<!doctype html>
<html lang="en">
<meta charset="utf-8">
<title>Frame geometry contract</title>
<style>
body { margin: 0; width: 600px; height: 900px; }
canvas { position: absolute; left: 17px; top: 23px; width: 120px; height: 40px; }
iframe { position: absolute; left: 30px; top: 90px; width: 180px; height: 100px; border: 7px solid #555; }
</style>
<canvas id="frame-canvas" width="120" height="40"></canvas>
<script>
const context = document.querySelector("canvas").getContext("2d");
context.fillStyle = "#15803d";
context.fillRect(0, 0, 120, 40);
context.fillStyle = "white";
context.fillText("FRAME PIXELS", 4, 24);
if (new URLSearchParams(location.search).has("nested")) {
const child = document.createElement("iframe");
child.title = "nested same-target frame";
child.src = "/frame.html";
document.body.append(child);
}
window.fixtureReady = true;
</script>
</html>
@@ -1,26 +0,0 @@
<!doctype html>
<html lang="en">
<meta charset="utf-8">
<title>Canvas grid</title>
<style>
body { margin: 0; font: 16px sans-serif; }
nav { height: 40px; }
canvas { display: block; width: 780px; height: 360px; }
</style>
<nav><button type="button">Insert record</button> <button type="button">Filter</button></nav>
<canvas width="1560" height="720"></canvas>
<script>
const context = document.querySelector("canvas").getContext("2d");
context.scale(2, 2);
context.fillStyle = "white";
context.fillRect(0, 0, 780, 360);
context.strokeStyle = "#777";
for (let x = 0; x <= 780; x += 260) { context.beginPath(); context.moveTo(x, 0); context.lineTo(x, 350); context.stroke(); }
for (let y = 0; y <= 350; y += 50) { context.beginPath(); context.moveTo(0, y); context.lineTo(780, y); context.stroke(); }
context.fillStyle = "#111";
context.font = "16px sans-serif";
["Canvas column A", "Canvas column B", "Canvas column C"].forEach((label, i) => context.fillText(label, i * 260 + 12, 30));
for (let row = 1; row <= 6; row++) context.fillText(`Canvas row ${row}`, 12, row * 50 + 30);
window.fixtureReady = true;
</script>
</html>
@@ -1,56 +0,0 @@
<!doctype html>
<html lang="en">
<meta charset="utf-8">
<title>Visual observation contract fixture</title>
<style>
body { margin: 0; min-height: 1800px; font: 14px sans-serif; }
#cases { display: grid; grid-template-columns: repeat(4, 180px); gap: 12px; padding: 20px; }
.case { width: 160px; height: 70px; }
canvas { width: 120px; height: 40px; }
.clip { width: 60px; height: 30px; overflow: hidden; }
.axis-clip { width: 60px; height: 20px; overflow-x: clip; overflow-y: visible; }
#same, #cross { position: absolute; top: 600px; width: 300px; height: 220px; border: 10px solid #555; }
#same { left: 40px; }
#cross { left: 400px; }
</style>
<h1>Canvas contract</h1>
<button type="button" id="control">Ordinary DOM control</button>
<div id="cases">
<div class="case"><canvas id="plain"></canvas></div>
<div class="case"><canvas id="named" aria-label="Named chart"></canvas></div>
<div class="case" aria-hidden="true"><canvas id="aria-hidden"></canvas></div>
<div class="case" inert><canvas id="inert"></canvas></div>
<div class="case"><canvas id="pointer-none" style="pointer-events:none"></canvas></div>
<div class="case" style="visibility:hidden"><canvas id="visibility-override" style="visibility:visible"></canvas></div>
<div class="case" style="opacity:0"><canvas id="opacity-parent"></canvas></div>
<div class="case"><canvas id="display-none" style="display:none"></canvas></div>
<div class="case"><canvas id="visibility-hidden" style="visibility:hidden"></canvas></div>
<div class="case"><canvas id="zero" style="width:0"></canvas></div>
<div class="case"><canvas id="hidden-overridden" hidden style="display:block"></canvas></div>
<div class="case"><div class="clip"><canvas id="partial-clip"></canvas></div></div>
<div class="case"><div class="clip"><canvas id="full-clip" style="margin-left:80px"></canvas></div></div>
<div class="case"><canvas id="fallback">Accessible fallback is not pixel content</canvas></div>
<div class="case"><div class="axis-clip"><canvas id="axis-clip"></canvas></div></div>
<div class="case"><div id="border-clip-owner" style="width:60px;height:30px;border:5px solid black;padding:4px;overflow:hidden"><canvas id="border-clip"></canvas></div></div>
<div class="case"><canvas id="border-canvas" style="border:5px solid black"></canvas></div>
<div class="case" style="position:relative"><canvas id="stack-a"></canvas><canvas id="stack-b" style="position:absolute;left:0;top:0"></canvas></div>
</div>
<iframe id="same" title="same target"></iframe>
<iframe id="cross" title="cross site"></iframe>
<script>
for (const canvas of document.querySelectorAll("canvas")) {
canvas.width = 120;
canvas.height = 40;
const context = canvas.getContext("2d");
context.fillStyle = "#1d4ed8";
context.fillRect(0, 0, 120, 40);
context.fillStyle = "white";
context.fillText("PIXEL TABLE 42", 5, 24);
}
if (!new URLSearchParams(location.search).has("noFrames")) {
document.querySelector("#same").src = "/frame.html?nested=1";
document.querySelector("#cross").src = `${location.protocol}//localhost:${location.port}/frame.html`;
}
window.fixtureReady = true;
</script>
</html>
@@ -1,86 +0,0 @@
// Diagnostic, not a golden test requiring the current quadratic cost to persist.
// Instrument a temporary copy; never modify the production renderer.
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../..");
const output = process.argv[2];
if (!output || !path.isAbsolute(output))
throw new Error("Pass an absolute evidence output directory");
const require = createRequire(path.join(root, "apps/extension/package.json"));
const vitestDirectory = path.dirname(require.resolve("vitest/package.json"));
const temporary = await mkdtemp(path.join(os.tmpdir(), "bsk-render-measure-"));
await mkdir(output, { recursive: true });
try {
const source = await readFile(path.join(root, "packages/vom/src/render.ts"), "utf8");
let instrumented = source;
for (const [needle, field] of [
["const { node, depth } = stack.pop() as { node: VomNode; depth: number };", "renderVisits"],
["for (const candidate of orderedCandidates) {", "contextCandidates"],
["for (const sibling of siblings) {", "siblingVisits"],
]) {
assert.equal(instrumented.split(needle).length, 2, `Update measurement site: ${field}`);
instrumented = instrumented.replace(needle, `${needle} designCounts.${field}++;`);
}
instrumented = `export const designCounts = {renderVisits:0,contextCandidates:0,siblingVisits:0};\n${instrumented}`;
await writeFile(path.join(temporary, "render.ts"), instrumented);
for (const file of ["layers.ts", "types.ts"])
await copyFile(path.join(root, "packages/vom/src", file), path.join(temporary, file));
const reportPath = path.join(output, "render-counts.json");
await writeFile(
path.join(temporary, "measure.test.ts"),
`
import {writeFileSync} from 'node:fs';
import {test,expect} from ${JSON.stringify(path.join(vitestDirectory, "dist/index.js"))};
import {renderVom,designCounts} from './render';
test('measure operation counts',()=>{
const rows=[];
const make=(id,extra={})=>({id,parentId:1,backendNodeId:id,tag:'div',role:'generic',rect:{x:0,y:0,w:100,h:20},paintOrder:0,position:'static',pointerEvents:'auto',contextScopeId:'root',...extra});
const run=(shape,n,nodes)=>{
Object.assign(designCounts,{renderVisits:0,contextCandidates:0,siblingVisits:0});
const result=renderVom({viewport:{width:1440,height:1000},nodes},{maxTokens:Infinity});
rows.push({shape,n,counts:{...designCounts},refs:result.refs.length});
};
for(const n of [100,200,400]){
const nodes=[make(1,{parentId:null,role:'RootWebArea',name:'Doc'})];
for(let i=0;i<n;i++)nodes.push(make(i+2,{role:'heading',name:'Region '+i,domParentId:1,domAncestorIds:[1]}));
for(let i=0;i<n;i++)nodes.push(make(n+i+2,{tag:'button',role:'button',name:'View',domParentId:1,domAncestorIds:[1]}));
run('wide',n,nodes);
}
for(const n of [2000,4000]){
const nodes=[make(1,{parentId:null,role:'RootWebArea',name:'Doc'})];
for(let i=2;i<=n;i++)nodes.push(make(i,{parentId:i-1}));
nodes.push(make(n+1,{parentId:n,role:'button',tag:'button',name:'Unique control'}));
run('deep',n,nodes);
}
expect(rows.at(-1).refs).toBe(1);
writeFileSync(${JSON.stringify(reportPath)},JSON.stringify({sourceSha256:${JSON.stringify(createHash("sha256").update(source).digest("hex"))},rows},null,2));
});
`,
);
const config = path.join(temporary, "vitest.config.mjs");
await writeFile(
config,
`export default {test:{environment:'node',include:[${JSON.stringify(path.join(temporary, "measure.test.ts"))}]}};`,
);
const result = spawnSync(
process.execPath,
[path.join(vitestDirectory, "vitest.mjs"), "run", "--config", config],
{ cwd: root, encoding: "utf8", timeout: 60000 },
);
await writeFile(
path.join(output, "render-measure.log"),
`${result.stdout ?? ""}\n${result.stderr ?? ""}`,
);
if (result.error) throw result.error;
assert.equal(result.status, 0, `Measurement failed; see ${output}/render-measure.log`);
process.stdout.write(`${reportPath}\n`);
} finally {
await rm(temporary, { recursive: true, force: true });
}
@@ -1,533 +0,0 @@
// Standalone browser contract probe. No extension, credentials or user profile.
// Raw replies and screenshots are written only to the caller-supplied directory.
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { once } from "node:events";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import http from "node:http";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
const fixtureDirectory = path.dirname(fileURLToPath(import.meta.url));
const output = process.argv[2];
if (!output || !path.isAbsolute(output))
throw new Error("Pass an absolute evidence output directory");
const chromePath =
process.env.CHROME_PATH ?? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const styles = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"overflow-x",
"overflow-y",
"display",
];
const checks = [];
function check(name, condition) {
checks.push({ name, passed: Boolean(condition) });
assert.ok(condition, name);
}
class Cdp {
nextId = 0;
pending = new Map();
events = [];
counts = {};
async connect(url) {
this.socket = new WebSocket(url);
this.socket.addEventListener("message", ({ data }) => {
const reply = JSON.parse(data);
if (!reply.id) {
this.events.push(reply);
return;
}
const task = this.pending.get(reply.id);
if (!task) return;
this.pending.delete(reply.id);
clearTimeout(task.timer);
if (reply.error) task.reject(new Error(JSON.stringify(reply.error)));
else task.resolve(reply.result);
});
await new Promise((resolve, reject) => {
this.socket.addEventListener("open", resolve, { once: true });
this.socket.addEventListener("error", reject, { once: true });
});
}
send(method, params = {}, sessionId) {
const id = ++this.nextId;
this.counts[method] = (this.counts[method] ?? 0) + 1;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, 10000);
this.pending.set(id, { resolve, reject, timer });
this.socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
});
}
close() {
for (const task of this.pending.values()) {
clearTimeout(task.timer);
task.reject(new Error("CDP closed"));
}
this.pending.clear();
this.socket?.close();
}
}
function decodedCanvases(snapshot) {
return snapshot.documents.flatMap((document) => {
const layoutByNode = new Map(document.layout.nodeIndex.map((node, index) => [node, index]));
return document.nodes.nodeName.flatMap((name, index) => {
if (snapshot.strings[name] !== "CANVAS") return [];
const attrs = document.nodes.attributes[index];
const attributes = {};
for (let i = 0; i < attrs.length; i += 2)
attributes[snapshot.strings[attrs[i]]] = snapshot.strings[attrs[i + 1]];
const layoutIndex = layoutByNode.get(index) ?? -1;
return [
{
id: attributes.id,
backendNodeId: document.nodes.backendNodeId[index],
frameId: snapshot.strings[document.frameId],
attributes,
bounds: layoutIndex < 0 ? null : document.layout.bounds[layoutIndex],
styles:
layoutIndex < 0
? null
: Object.fromEntries(
styles.map((style, i) => [
style,
snapshot.strings[document.layout.styles[layoutIndex][i]],
]),
),
},
];
});
});
}
await mkdir(output, { recursive: true });
const profile = await mkdtemp(path.join(os.tmpdir(), "bsk-visual-probe-"));
const server = http.createServer(async (request, response) => {
const name = new URL(request.url, "http://fixture").pathname;
if (!["/index.html", "/frame.html"].includes(name)) {
response.writeHead(404).end();
return;
}
try {
response.setHeader("Content-Type", "text/html; charset=utf-8");
response.end(await readFile(path.join(fixtureDirectory, name.slice(1))));
} catch {
response.writeHead(500).end();
}
});
const cdp = new Cdp();
let chrome;
let report = {};
try {
server.listen(0, "127.0.0.1");
await once(server, "listening");
const port = server.address().port;
chrome = spawn(
chromePath,
[
"--headless=new",
`--user-data-dir=${profile}`,
"--remote-debugging-port=0",
"--no-first-run",
"--no-default-browser-check",
"--site-per-process",
"--window-size=1440,1100",
"about:blank",
],
{ stdio: "ignore" },
);
chrome.on("error", (error) => {
report.launchError = error.message;
});
let activePort;
for (let attempt = 0; attempt < 100; attempt++) {
try {
activePort = (await readFile(path.join(profile, "DevToolsActivePort"), "utf8"))
.trim()
.split("\n");
break;
} catch {
await pause(100);
}
}
if (!activePort)
throw new Error(report.launchError ?? "Chrome did not expose DevToolsActivePort");
await cdp.connect(`ws://127.0.0.1:${activePort[0]}${activePort[1]}`);
report.browser = await cdp.send("Browser.getVersion");
const { targetId } = await cdp.send("Target.createTarget", { url: "about:blank" });
const { sessionId: root } = await cdp.send("Target.attachToTarget", { targetId, flatten: true });
const send = (method, params = {}) => cdp.send(method, params, root);
const evaluate = async (expression, session = root, contextId) => {
const result = await cdp.send(
"Runtime.evaluate",
{ expression, returnByValue: true, awaitPromise: true, ...(contextId ? { contextId } : {}) },
session,
);
if (result.exceptionDetails) throw new Error(JSON.stringify(result.exceptionDetails));
return result.result.value;
};
await send("Page.enable");
await send("Runtime.enable");
async function navigate(url) {
await send("Page.navigate", { url });
for (let i = 0; i < 100; i++) {
if (await evaluate("document.readyState === 'complete' && window.fixtureReady === true"))
return;
await pause(50);
}
throw new Error("Fixture did not become ready");
}
const url = `http://127.0.0.1:${port}/index.html`;
await navigate(`${url}?noFrames=1`);
async function capture(label) {
const metrics = await send("Page.getLayoutMetrics");
const snapshot = await send("DOMSnapshot.captureSnapshot", {
computedStyles: styles,
includePaintOrder: true,
includeDOMRects: true,
});
const ax = await send("Accessibility.getFullAXTree");
const runtime = await evaluate(`Array.from(document.querySelectorAll('canvas'), el => {
const r=el.getBoundingClientRect(), s=getComputedStyle(el);
return {id:el.id,rect:{x:r.x,y:r.y,w:r.width,h:r.height},visibility:s.visibility,display:s.display,
checkVisibility:el.checkVisibility({opacityProperty:true,visibilityProperty:true}),
dpr:devicePixelRatio,scrollX,scrollY}; })`);
const result = { metrics, snapshot, ax, runtime, canvases: decodedCanvases(snapshot) };
await writeFile(path.join(output, `${label}.json`), JSON.stringify(result, null, 2));
return result;
}
const baseline = await capture("root-baseline");
const rootImage = await send("Page.captureScreenshot", { format: "png" });
await writeFile(path.join(output, "root.png"), Buffer.from(rootImage.data, "base64"));
const runtime = Object.fromEntries(baseline.runtime.map((node) => [node.id, node]));
for (const id of [
"plain",
"named",
"aria-hidden",
"inert",
"pointer-none",
"visibility-override",
"hidden-overridden",
"fallback",
])
check(`CSS visible: ${id}`, runtime[id].checkVisibility);
for (const id of ["opacity-parent", "display-none", "visibility-hidden"])
check(`CSS suppressed: ${id}`, !runtime[id].checkVisibility);
check("painted text is absent from AX", !JSON.stringify(baseline.ax).includes("PIXEL TABLE 42"));
report.css = baseline.runtime;
const bordered = baseline.canvases.find((node) => node.id === "border-canvas");
report.canvasBox = await send("DOM.getBoxModel", { backendNodeId: bordered.backendNodeId });
check(
"Canvas snapshot uses border box",
bordered.bounds[2] === 130 &&
report.canvasBox.model.border[2] - report.canvasBox.model.border[0] === 130 &&
report.canvasBox.model.content[2] - report.canvasBox.model.content[0] === 120,
);
const rootDocument = baseline.snapshot.documents[0];
const ownerNode = rootDocument.nodes.attributes.findIndex((attrs) =>
attrs.some(
(value, index) => index % 2 === 1 && baseline.snapshot.strings[value] === "border-clip-owner",
),
);
const ownerLayout = rootDocument.layout.nodeIndex.indexOf(ownerNode);
report.clipClientBox = {
snapshot: rootDocument.layout.clientRects[ownerLayout],
runtime: await evaluate(
"(() => {const e=document.getElementById('border-clip-owner');return [e.clientLeft,e.clientTop,e.clientWidth,e.clientHeight]})()",
),
};
check(
"snapshot client rect contains local client offsets and size",
JSON.stringify(report.clipClientBox.snapshot) ===
JSON.stringify(report.clipClientBox.runtime) &&
JSON.stringify(report.clipClientBox.runtime) === "[5,5,68,38]",
);
report.clips = await evaluate(`['partial-clip','full-clip','axis-clip'].map(id=>{
const el=document.getElementById(id),p=el.parentElement,s=getComputedStyle(p);
const a=el.getBoundingClientRect(),b=p.getBoundingClientRect();
return {id,overflowX:s.overflowX,overflowY:s.overflowY,
child:{x:a.x,y:a.y,w:a.width,h:a.height},parent:{x:b.x,y:b.y,w:b.width,h:b.height}};
})`);
const [partial, full, axis] = report.clips;
check(
"partial overflow crop dimensions",
partial.parent.w === 60 &&
partial.parent.h === 30 &&
partial.child.x === partial.parent.x &&
partial.child.y === partial.parent.y,
);
check(
"fully clipped candidate lies beyond ancestor",
full.child.x >= full.parent.x + full.parent.w,
);
check(
"single-axis clip remains independent in computed style",
axis.overflowX === "clip" && axis.overflowY === "visible" && axis.child.h > axis.parent.h,
);
const samples = [];
for (let i = 0; i < 12; i++)
samples.push(
await evaluate(
"JSON.stringify(document.querySelector('#plain').getBoundingClientRect().toJSON())",
),
);
check("static rect stable over 12 reads", new Set(samples).size === 1);
await evaluate("document.querySelector('#plain').style.transform='translateX(0.5px)'");
report.halfPixelMovement = await evaluate(
"document.querySelector('#plain').getBoundingClientRect().x",
);
await evaluate("document.querySelector('#plain').style.transform=''");
report.raster = [];
for (const dpr of [1, 1.5, 2]) {
await send("Emulation.setDeviceMetricsOverride", {
width: 1000,
height: 900,
deviceScaleFactor: dpr,
mobile: false,
});
const state = await capture(`dpr-${dpr}`);
const image = await send("Page.captureScreenshot", {
format: "png",
clip: { x: 20, y: 100, width: 120, height: 40, scale: 1 },
});
const bytes = Buffer.from(image.data, "base64");
const dimensions = { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
report.raster.push({
dpr,
dimensions,
metrics: state.metrics,
plain: state.canvases.find((node) => node.id === "plain"),
});
check(
`screenshot raster DPR ${dpr}`,
dimensions.width === 120 * dpr && dimensions.height === 40 * dpr,
);
}
await send("Emulation.clearDeviceMetricsOverride");
await evaluate("document.body.style.zoom='1.25'");
const cssZoom = await capture("css-zoom-1.25");
check(
"CSS zoom is reflected in snapshot geometry",
cssZoom.canvases.find((n) => n.id === "plain").bounds[2] === 150,
);
report.cssZoom = {
runtime: cssZoom.runtime[0],
snapshot: cssZoom.canvases.find((n) => n.id === "plain"),
};
await evaluate("document.body.style.zoom=''");
await send("Emulation.setPageScaleFactor", { pageScaleFactor: 1.5 });
report.pageScale = (await capture("page-scale-1.5")).metrics;
await send("Emulation.setPageScaleFactor", { pageScaleFactor: 1 });
await evaluate("scrollTo(0,100)");
const scrolled = await capture("scroll-100");
check(
"snapshot document bounds stable under scroll",
JSON.stringify(scrolled.canvases.find((n) => n.id === "plain").bounds) ===
JSON.stringify(baseline.canvases.find((n) => n.id === "plain").bounds),
);
check(
"runtime viewport y follows scroll",
scrolled.runtime[0].rect.y === baseline.runtime[0].rect.y - 100,
);
await navigate(url);
await pause(200);
const targets = await cdp.send("Target.getTargets");
const iframeTarget = targets.targetInfos.find(
(target) => target.type === "iframe" && target.url.startsWith(`http://localhost:${port}/`),
);
check("cross-site frame is an actual OOPIF target", iframeTarget);
const { sessionId: child } = await cdp.send("Target.attachToTarget", {
targetId: iframeTarget.targetId,
flatten: true,
});
report.frames = {};
for (const [name, session] of [
["root", root],
["oopif", child],
]) {
const tree = await cdp.send("Page.getFrameTree", {}, session);
const snapshot = await cdp.send(
"DOMSnapshot.captureSnapshot",
{ computedStyles: styles, includeDOMRects: true },
session,
);
const frameRows = [];
const stack = [tree.frameTree];
while (stack.length) {
const node = stack.pop();
const world = await cdp.send(
"Page.createIsolatedWorld",
{ frameId: node.frame.id, worldName: "bsk-design-probe" },
session,
);
const local = await evaluate(
"({url:location.pathname,innerWidth,innerHeight,scrollX,scrollY,canvas:document.querySelector('canvas').getBoundingClientRect().toJSON()})",
session,
world.executionContextId,
);
let owner;
if (node.frame.parentId) {
// All owners in this fixture live in the root target; child content can be OOPIF.
const address = await send("DOM.getFrameOwner", { frameId: node.frame.id });
owner = await send("DOM.getBoxModel", { backendNodeId: address.backendNodeId });
}
frameRows.push({ frame: node.frame, local, owner });
stack.push(...(node.childFrames ?? []));
}
report.frames[name] = { tree, snapshot, canvases: decodedCanvases(snapshot), frameRows };
}
check(
"same-target snapshot includes nested documents",
report.frames.root.snapshot.documents.length === 3,
);
check("oopif snapshot isolated from root", report.frames.oopif.snapshot.documents.length === 1);
const sameRow = report.frames.root.frameRows.find(
(row) => row.owner && row.local.innerWidth === 300,
);
const nestedRow = report.frames.root.frameRows.find(
(row) => row.owner && row.local.innerWidth === 180,
);
check(
"same frame content origin includes border",
sameRow.owner.model.content[0] === 50 && sameRow.owner.model.content[1] === 610,
);
check(
"nested owner quad is already target-root relative",
nestedRow.owner.model.content[0] === 87 && nestedRow.owner.model.content[1] === 707,
);
const crossRow = report.frames.oopif.frameRows[0];
check(
"OOPIF owner content origin",
crossRow.owner.model.content[0] === 410 && crossRow.owner.model.content[1] === 610,
);
const sameFrame = sameRow.frame.id;
const world = await send("Page.createIsolatedWorld", {
frameId: sameFrame,
worldName: "bsk-design-probe",
});
await evaluate("scrollTo(0,10)", root, world.executionContextId);
const frameScroll = await send("DOMSnapshot.captureSnapshot", {
computedStyles: styles,
includeDOMRects: true,
});
report.frameScroll = {
documents: frameScroll.documents.map((d) => ({
frameId: frameScroll.strings[d.frameId],
scrollOffsetY: d.scrollOffsetY,
})),
canvases: decodedCanvases(frameScroll),
local: await evaluate(
"document.querySelector('canvas').getBoundingClientRect().toJSON()",
root,
world.executionContextId,
),
};
check("frame scroll is local", report.frameScroll.local.y === 13);
check(
"frame snapshot stays document-relative after child scroll",
report.frameScroll.canvases.find((n) => n.frameId === sameFrame).bounds[1] === 23,
);
await evaluate("scrollTo(0,0)", root, world.executionContextId);
await evaluate(
"document.querySelector('#same').style.transformOrigin='0 0';document.querySelector('#same').style.transform='scale(1.25)'",
);
const sameOwner = await send("DOM.getFrameOwner", { frameId: sameFrame });
report.transformedOwner = await send("DOM.getBoxModel", {
backendNodeId: sameOwner.backendNodeId,
});
check(
"scaled frame owner content width",
report.transformedOwner.model.content[2] - report.transformedOwner.model.content[0] === 375,
);
await evaluate("document.querySelector('#same').style.transform=''");
const screenshot = await send("Page.captureScreenshot", { format: "png" });
await writeFile(path.join(output, "frames.png"), Buffer.from(screenshot.data, "base64"));
const before = (await send("Page.getFrameTree")).frameTree.frame;
const oldDocument = (await send("DOM.getDocument")).root.backendNodeId;
await evaluate("history.replaceState(null,'','#same-document')");
const sameDocument = (await send("Page.getFrameTree")).frameTree.frame;
const sameDocumentBackend = (await send("DOM.getDocument")).root.backendNodeId;
await navigate(`${url}?noFrames=1&navigation=2`);
const after = (await send("Page.getFrameTree")).frameTree.frame;
const newDocument = (await send("DOM.getDocument")).root.backendNodeId;
report.identity = { before, sameDocument, after, oldDocument, sameDocumentBackend, newDocument };
check("same-document navigation preserves loader", before.loaderId === sameDocument.loaderId);
check("cross-document navigation changes loader", before.loaderId !== after.loaderId);
check("frame id survives document replacement", before.id === after.id);
check(
"document backend identity changes on navigation",
oldDocument !== newDocument && oldDocument === sameDocumentBackend,
);
const documentTree = await send("DOM.getDocument");
report.identity.htmlBeforeOpen = documentTree.root.children.find(
(node) => node.nodeType === 1,
).backendNodeId;
const anchorNode = await send("DOM.querySelector", {
nodeId: documentTree.root.nodeId,
selector: "canvas",
});
const anchor = await send("DOM.describeNode", { nodeId: anchorNode.nodeId });
await evaluate(
"document.open(); document.write('<!doctype html><title>Replacement</title><canvas></canvas>'); document.close()",
);
report.identity.documentOpen = (await send("Page.getFrameTree")).frameTree.frame;
const afterOpenTree = await send("DOM.getDocument");
report.identity.documentOpenBackend = afterOpenTree.root.backendNodeId;
report.identity.htmlAfterOpen = afterOpenTree.root.children.find(
(node) => node.nodeType === 1,
).backendNodeId;
check(
"document.open replaces documentElement identity",
report.identity.htmlBeforeOpen !== report.identity.htmlAfterOpen,
);
try {
const object = await send("DOM.resolveNode", { backendNodeId: anchor.node.backendNodeId });
const connected = await send("Runtime.callFunctionOn", {
objectId: object.object.objectId,
functionDeclaration: "function(){return this.isConnected}",
returnByValue: true,
});
report.identity.oldAnchorConnected = connected.result.value;
} catch {
report.identity.oldAnchorConnected = false;
}
check(
"document.open invalidates old anchor even if document identifiers survive",
report.identity.oldAnchorConnected === false,
);
report.counts = cdp.counts;
report.checks = checks;
process.stdout.write(
`${JSON.stringify({ browser: report.browser.product, checks: checks.length, output })}\n`,
);
} catch (error) {
report.error = error.stack;
report.checks = checks;
process.exitCode = 1;
process.stderr.write(`${error.stack}\n`);
} finally {
await writeFile(path.join(output, "report.json"), JSON.stringify(report, null, 2));
cdp.close();
if (chrome && chrome.exitCode === null) {
const exited = once(chrome, "exit");
chrome.kill("SIGTERM");
await Promise.race([exited, pause(2000)]);
if (chrome.exitCode === null) {
chrome.kill("SIGKILL");
await exited;
}
}
await new Promise((resolve) => server.close(resolve));
await rm(profile, { recursive: true, force: true });
}
@@ -1,15 +0,0 @@
<!doctype html>
<html lang="en">
<meta charset="utf-8">
<title>Embedded Canvas table reproduction</title>
<style>
body { font: 16px sans-serif; }
iframe { display: block; width: 800px; height: 440px; border: 8px solid #ccc; }
</style>
<h1>Embedded table</h1>
<button type="button">Page action</button>
<iframe title="Table application"></iframe>
<script>
document.querySelector("iframe").src = `http://localhost:${location.port}/grid.html`;
</script>
</html>
@@ -1,184 +0,0 @@
# Visual observation:验收清单
新 agent 先读 [架构与 PR 划分](visual-observation.md),再按本清单验收当前 PR;无需聊天记录,也无需另写报告、逐项编号或填写模板。历史结果见 [PR 0 证据](visual-observation-evidence.md),不能代替本次执行。
## 怎么开始
- 查看分支、HEAD 和工作区改动,确认当前做哪个 PR。PR 0 基于 main `3c5f838`;后续模块对照其前置基线,最终回归对照 main。注意未跟踪文件也属于改动。
- 当前仅有设计、合成 fixture 和诊断脚本;未来模块尚未实现。只验收当前阶段及受影响的已有功能,不要求提前完成后续 PR。
- 测试放在模块附近,调用实际生产代码。未接线模块可用人工输入和 CDP stub;实现 PR 写明运行命令即可,不强制新建指定目录、命名或诊断 API。
- 浏览器验收前核实 CLI 版本,并重新 build、确认扩展 reload。PATH 中的 bsk 曾经比仓库版本旧,不能只凭版本号认定运行了目标代码。
- 不 commit/push。截图、原始页面内容和一次性实验放临时目录;只把长期需要的设计、fixture 和测试留在仓库。
## 现有验证入口
从仓库根目录运行,命令成功退出且相关断言通过才算成功:
```sh
pnpm vom:test
pnpm --filter @browser-skill/extension test src/tools/__tests__/frame-geometry.test.ts src/tools/__tests__/element-geometry.test.ts src/tools/__tests__/observation.test.ts src/tools/__tests__/snapshot-ref.test.ts src/tools/__tests__/interaction.test.ts src/tools/vom src/session-manager/__tests__/ref-store.test.ts
pnpm exec biome check apps/extension/test-fixtures/visual-observation
VISUAL_EVIDENCE="$(mktemp -d /tmp/bsk-visual-acceptance.XXXXXX)"
node apps/extension/test-fixtures/visual-observation/probe.mjs "$VISUAL_EVIDENCE/browser-contract"
node apps/extension/test-fixtures/visual-observation/measure-render.mjs "$VISUAL_EVIDENCE/performance"
```
probe 需要 Node 22+ 和 Chrome,配置见 [fixture 说明](../../apps/extension/test-fixtures/visual-observation/README.md)。它验证浏览器事实,不代表后续生产模块已经通过。measure-render 用于旧 renderer 对照,不要求修复历史复杂度。
实现 PR 还需运行所属模块的新测试、`pnpm --filter @browser-skill/extension compile`、`pnpm ext:test` 和 `pnpm ext:build`。仅修改文档时检查链接和内容即可,不必重跑浏览器。已有失败应说明,不在范围外顺手修复。
## 每个 PR 怎么算完成
### PR 0:设计与复现
只有文档、合成 fixture 和诊断变动;架构、边界及后续 PR 的完成条件明确。现有测试通过;probe 的 checks 非空且全部通过(本版 39 项);measure-render 产生五组 wide/deep 结果。
按下方浏览器步骤,在 main 重现“表格已绘制,但 observe 没有视觉入口”。合成页面和原始 iWiki 分别确认。原始页面未加载、无权限或缺入口时说明阻塞,不能算已复现。入库内容不含内部页面地址、标识或截图。
### PR 1:几何与身份
数值误差:纯函数输入允许 1e-6 CSS px;真实浏览器每条边允许 0.25 CSS px。坐标类型不能混传;动态 frame 归属需运行时检查。
| 怎么测 | 什么算成功 |
|---|---|
| top document bounds=(30,50,120,40),document scroll=(10,20),viewport=800×600 | top border region=(20,30,120,40);不受 backing store 尺寸影响 |
| 使用 frame.html:local Canvas=(17,23,120,40),same owner content origin=(50,610);nested origin=(87,707);OOPIF origin=(410,610) | top origins 分别=(67,633)、(104,730)、(427,633);不能重复给 nested 加父偏移;owner border 不能代替 content |
| child scrollY=10;same frame owner origin 不变 | Canvas top y=623,local y=13;snapshot document y=23;scroll 只扣一次 |
| same iframe 以自身左上角为原点 scale(1.25),原 border origin=(40,600),border=10 | owner content origin=(52.5,612.5),Canvas top=(73.75,641.25),尺寸150×50 |
| Canvas content 120×40,border 每边5;overflow owner CSS width60/height30/border5/padding4 | Canvas region 宽130高50,live 验证同 box;owner clientRect=(5,5,68,38),不能使用78×48 border范围作为clip |
| DPR=1/1.5/2,CSS Canvas120×40;UI zoom与CSS zoom分别处理 | DPR不直接除掉 CSS rect;raster 预测与实际图片分别记录;不以layout/css宽度比作为万能DPR |
| 分别构造缺失 owner、失效 frame、旋转、透视、mask、clip-path、活动pinch、变换的overflow祖先 | 前两项 geometry-unavailable,后六类 geometry-unsupported;无伪造精确 region ref、无按border猜偏移 |
| 同一操作并发请求同一frame 20次,再开启第二次操作 | 第一次每个metrics来源、owner quad、frame投影最多构建/请求一次;第二次重新获取;异常promise不泄漏到其他操作 |
### PR 2:采集与共享事实
| 怎么测 | 什么算成功 |
|---|---|
| target A/B 各含 backendNodeId=7,frame/document不同 | 两个lookup结果独立;AX join、overlay排除和父子查找不串target |
| 4个frame分属2个target,其中一个target有3份document;均成功采集 | 每target恰好一次DOMSnapshot;每frame恰好一次AX批次;失败重试不混入成功请求数;无按frame重复snapshot |
| 采集中模拟frame detach或loader/documentElement更换 | 丢弃受影响document的混合事实并标记原因;其他完整document可保留;不得把新AX接到旧DOM |
| 输入10,000深链、10,000兄弟、孤儿及A→B→A环 | 无栈溢出/无限循环;孤儿/环有明确不完整状态;不生成完整祖先证据;context推导每有效节点至多一次 |
| opacity0祖先;visibility:hidden祖先+visible子;aria-hidden/inert祖先 | 分别得到视觉透明、子可见、语义/交互状态独立;不折叠为同一个excluded |
| 一个frame采集失败,其他frame有语义节点;全部成功但无Canvas | 前者partial且保留其他语义;后者empty而非unavailable;错误原因可追踪到frame及阶段 |
| 注入CDP调用记录和hover探针;纯计算中途取消 | 所有静态DOM/AX捕获先于hover输入;取消后下一工作块不继续处理,未执行的输入/截图为0;取消不变成普通空结果 |
### PR 3:视觉发现
| 怎么测 | 什么算成功 |
|---|---|
| index.html?noFrames=1 的plain、named、aria-hidden、inert、pointer-none、visibility-override、hidden-overridden、fallback | 八个均为候选;无label不影响发现,有AX/fallback不抑制候选 |
| opacity-parent、display-none、visibility-hidden、zero、full-clip | 五个均无候选,原因分别能追踪到视觉/几何规则;不能把aria-hidden列为原因 |
| partial-clip、axis-clip、border-clip、border-canvas | crop尺寸分别60×30、60×40、64×34、130×50;边界来自client box/同种Canvas box |
| 同frame中删除语义parent,再保持DOM/几何不变 | 候选仍存在、地址不变;语义context可变为null;frame owner本身无法解析时不是该用例 |
| 连续两次输入同一只读Facts;对Facts启用深冻结 | 候选内容一致,无输入mutation,无CDP调用;去重前stack-a/b仍是两个候选 |
| 普通候选部分越出viewport;完全越界;overlay-owned Canvas;unsupported clip | 分别交集区域、无候选、排除、明确unsupported状态;不把未知状态伪装成empty |
### PR 4:选择与预算
预算按每个输出行的 `ceil(line.length/4)` 求和(UTF-16 length),包括头部和截断提示。H 为完整协议头成本,V 为固定行 `[visual] Canvas regions present; use screenshot` 的成本。
| 怎么测 | 什么算成功 |
|---|---|
| 同context:无名1000×800候选A,有名10×10候选B,K=1 | 选A;名称不能压倒区域面积 |
| 同document/parent且crop完全相同的A/B;另一个parent的C;crop右边相差0.01的D;相邻tile E | A/B仅输出一个代表;C/D/E独立保留;不输出layers=N作为业务解释 |
| 面积相同的候选随机打乱20次,地址/位置保持不变 | 排序相同;tie-break顺序固定为frameId、top y、left x、backendNodeId(同一观察内);去重代表取该顺序最小者;字符串按代码单元字典序、数值按数值升序 |
| 50,001个不同key,最后一个为最大Canvas,K=1 | 最后最大者仍被选中;dedup key峰值≤50,000;标记dedup-degraded;未因容量耗尽停扫描 |
| 100,000候选、K=1/32/512;另测K=0 | 选中数量≤K、heap峰值≤K、exact-key之外的模糊pair比较为0;K=0不产生ref候选;结果与独立全排序小规模oracle一致 |
| 带blocking layer的纯输入,候选context在L1、背景、未知各一个 | L1优先,背景不标为物理不可见,未知不伪装L1;同层再按面积及固定 tie-break 排序;不修改Facts |
| 预算H-1 | 新observe规划返回insufficient-budget,工具映射invalid_params并给minimum=H;无新文本/ref提交;snapshot旧路径不改变 |
| 有候选,预算H、H+V-1、H+V | 前两个输出合法协议头、无悬空ref、truncated=true;H+V能保留固定视觉提示;全部账本≤预算 |
| 没有显式maxTokens,候选足够多 | 视觉附加部分cost≤2048;最终视觉条目≤512;不是对整个旧语义输出强加2048 |
| 标签含200个Unicode code point、引号、换行、代理对emoji | 标签先截至160 code point再JSON escape,按最终行核算;无断开的代理对,无换行注入额外条目;写不下的行不产生ref |
| finite预算取H+V、64、128、256、2048(小于H时拒绝),混合长短条目 | 每次总账本≤budget;输出决定可重复;已知omitted才给精确数,候选/去重组/上游未知数量不混加 |
### PR 5:类型化 ref
| 怎么测 | 什么算成功 |
|---|---|
| 注册DOM和visual-region两个判别目标 | kind决定操作资格;类型测试不能构造visual+interact矛盾状态;不额外接受调用者capability数组 |
| 同名ref置于两个session/tab;输入@e1与e1 | canonical形式可解析;不同session/tab不串目标;未知目标为既有not_found类错误 |
| 对visual ref调用click/fill/hover/select/get_html/人工帮助DOM定位入口 | 全部按主设计拒绝;CDP输入、DOM HTML读取、DOM高亮等该工具副作用次数为0;不能偷偷降为anchor操作 |
| loader变化、target重attach、documentElement替换;另测history.replaceState不改身份 | 前三者拒绝旧目标;最后不单凭URL变化拒绝;实际anchor仍需连接 |
| document.open保留loader/document backend但替换html和Canvas | 旧ref失效;只比较loader的实现必须被此测试击败 |
| 第二次observe重新绑定e1;第一次ref字符串再次输入 | 明确验证“最新观察映射”而非声称能识别旧字符串;新映射仅来自成功提交;旧DOM工具回归通过 |
### PR 6:截图执行
首次 clipScale 为 `min(1, 2048/(max(w,h)*r), sqrt(4000000/(w*h*r*r)))`;r 来自实际 metrics 适配,最终仍检查真实 PNG 尺寸。
| 怎么测 | 什么算成功 |
|---|---|
| visual ref解析成功并截图;DOM ref另测 | visual路径scrollIntoView次数0;DOM路径保持BASE策略;visual实时box种类与观察一致 |
| 同anchor四边分别移动0.249、0.25、0.251 CSS px;分别改尺寸/裁剪策略 | 前两种仅数值误差可接受,0.251拒绝;document/裁剪策略变化不受容差豁免;不靠old∩live返回小片段 |
| 保持身份/几何,重绘Canvas颜色;另移除Canvas | 重绘允许取得当前图像;移除拒绝;不承诺像素快照或把颜色改变误报为stale |
| css4096×2048,rasterEstimate2;css4000×4000,rasterEstimate1 | 首次clipScale分别0.25、0.5(不放大,取edge/area/1最小值);实际成功尺寸每边≤2048、面积≤4,000,000 |
| mock第一次图片3000×3000,第二次2000×2000;再测两次均3000×3000 | 第一组重试一次并成功,第二组恰好两次后pixel-budget-exceeded;不能无限重试或返回超限图 |
| 空data、非法PNG头、0尺寸、负/非有限几何 | 明确失败,不能猜测width/height;此项不要求将PNG头检查宣称为完整解码校验 |
| capture前取消;overlay抑制后CDP失败;采集时出现dialog | 取消前不再capture;成功/失败/取消均恢复overlay;dialog机制保持既有契约;无额外输入 |
### PR 7:observe 接线
| 怎么测 | 什么算成功 |
|---|---|
| 固定SemanticScene,无视觉候选;普通预算,与BASE replay同一输入 | 语义文本、ref相对顺序、role/name一致;不因为新通道重跑semantic pipeline |
| 同SemanticScene加两个视觉候选,大预算 | 原DOM ref顺序不变,视觉ref在其后;每个输出ref均可在registry找到同目标,registry无未输出新ref |
| DOM文本多到耗尽除预留外的预算 | 固定视觉提示可见(预算允许时);truncated=true;不是DOM stopped后无条件跳过视觉提示 |
| 视觉capture unavailable但语义成功 | 语义结果保留,观察状态能区分empty/unavailable/truncated;debug reason准确且无原始DOM泄漏 |
| snapshot、recording调用、hover probe on/off与BASE对比 | snapshot不新增visual;recording不隐式启用发现;probe默认off,on时仍在静态采集后;相应既有测试通过 |
| blocking layer混合DOM/visual、重复候选、不同预算 | 输出层策略与选择模块的活跃层策略一致;头、摘要、条目、occlusion/omitted行均纳入账本;无悬空ref |
### PR 8:整体验收
复跑上述模块测试、下方性能对照和浏览器场景,确认原始 iWiki 的缺口已解决。检查没有站点特判、重复的几何/预算权威实现或遗留的替代路径;没有顺带扩展到 OCR、逻辑画布聚合或旧算法重写。不能只凭合成页面通过宣布原始问题解决。
## 性能怎么验
在相关模块实现时检查,不拖到最后统一补。旧路径和新增 Canvas 成本分开看:
- **旧路径**:同输入对照 main/前置基线,已有 render/context/sibling 工作量、语义构建次数和 DOM 操作请求不增加;保留 #165 的回归。不要求消除旧算法的历史二次项。
- **无 Canvas**:允许必要检测与共享几何成本,但不构建视觉专用索引、不发视觉专用 CDP 请求、不重复跑语义链路。
- **新增树处理**:deep/wide 节点数取 1k、10k、100k;每节点 context 至多推导一次,每条边每个索引 pass 至多访问一次,pass 数不随规模增加。无递归溢出、每候选完整祖先遍历或全祖先数组复制。
- **选择**:候选数取 1k、10k、100k,K 取 1、32、512;堆峰值不超过 K,去重 key 不超过 50,000,无模糊两两比较。比较计数符合具体算法推导的上界;小规模结果与独立全排序对照一致,容量耗尽后仍能选中最后的大图。
- **请求与取消**:每 target 一次 snapshot、每 frame 一次 AX 批次;单次操作内几何请求共享,不跨操作复用 live cache。几何最大并发 4,新增计算每块最多 256 项并实际让出事件循环,取消后不再启动下一块。
- **接线后耗时**:同机器、浏览器和固定内容,分别测无 Canvas 页与合成表格页。各预热 3 次,再基线/新版本交替各 20 次,比较中位数和 p95;旧阶段与新增成本分开。旧阶段中位数增加超过 max(基线 10%, 2ms) 时复测,两轮均超过且可归因于改动则未通过;噪声无法排除则说明尚未确认。操作数仍须满足前述条件,不能用耗时阈值代替它们。
计数可以由测试插桩取得,不必为验收增加生产接口。性能结果只需说明旧路径是否回退、新增路径的规模与成本,不需要另填表格。
## 浏览器怎么验
### 合成表格(PR 0 复现;PR 7/8 验证修复)
1. 核实 `bsk --version`、`bsk browsers`、`bsk status`;运行 `pnpm ext:build`,确认用户已 reload `apps/extension/dist/chrome-mv3`。
2. 单独启动服务器:`python3 -m http.server 4177 --bind 127.0.0.1 --directory apps/extension/test-fixtures/visual-observation`。
3. `bsk session start --width 1440 --height 1100`,使用返回的 session id 打开下列页面。尖括号替换为实际值,每条命令传同一 session。
```sh
bsk navigate http://127.0.0.1:4177/table.html --session <session> --json
bsk observe --session <session> --json
bsk screenshot --session <session> --out <临时目录>/table.png --json
```
4. 把窗口放前台,实际看到三列六行后再采集 observe。load 完成、AX 工具栏出现或背景占位截图都不能证明表格已经绘制。
5. main 应有 Page action、Insert record、Filter 三个 DOM 控件,truncated=false,但无表格视觉入口。这是 PR 0 的缺口复现。
6. 新实现应保留三个控件,并增加 visual ref。用本次返回的 ref 执行 `bsk screenshot --ref <ref> --session <session> --out <临时目录>/crop.png`,实际查看图片:包含 Canvas column A/B/C 和六行,裁剪对应 780×360 CSS px 的表格区域,不能错到工具栏或 iframe 边框。PNG 满足每边 2048、总像素 4,000,000 的上限。无需生成单元格语义。
### 几何与动态场景
PR 1 验证实际生产几何模块;PR 6 验证截图执行;PR 8 复验完整链路。未接 observe 时可注入人工 facts/ref,但必须调用实际模块,不能以独立 probe 代替实现测试。
- 必测缩放集合:DPR 1/1.5/2 × top/frame;真实 UI zoom 100%/90% × top/frame;CSS zoom 1.25 的 top Canvas;frame scale 1.25 的 same/nested/OOPIF。按 PR 1 的坐标预期核对。OOPIF 需确认实际 iframe target,跨 origin 本身不够。
- UI zoom 要通过浏览器菜单或原生快捷键确认,不能以 CSS zoom 或 pinch emulation 替代。旋转、mask、clip-path、活动 pinch、变换 overflow 按不支持策略降级,不输出猜测的精确 ref。
- 使用 `index.html?noFrames=1` 的 `#plain`,每个实验从 reload 后的新 ref 开始:平移 1px、宽度改为 121px → 旧 ref 截图要求重新观察;移除节点或 document.open → 旧目标失效;仅重绘颜色 → 截图成功且是新颜色。不要在重新 observe 后拿重用的 eN 字符串声称验证了旧 ref。
- visual ref 的 click/fill/hover/select/get_html/DOM 高亮拒绝测试只在合成页或自动测试运行,确认拒绝前没有派发输入、读 HTML 或高亮副作用。
### 原始 iWiki(PR 0 与 PR 8)
入口与就绪说明在本机临时文件 `/private/tmp/bsk-visual-design/private-reproduction.md`。文件丢失或登录失效时向用户取得入口/权限,说明这一项未完成;继续独立的合成验证。
保持只读。前台实际看到表格后采集 observe 和截图:PR 0 确认缺口;PR 8 确认外围 DOM 控件仍在、新 visual ref 能截到实际表格。不要编辑单元格,不要求原页面永久维持历史的 63 个 ref。私有内容不入库。
结束后停止自己的 session、服务器及临时 Chrome,恢复修改过的 zoom/emulation,不关闭其他任务资源。
## 做完怎么回复
直接告诉用户:验了哪个 PR、跑了什么、是否符合上述成功条件;若有失败或未做的检查,说明具体原因。必要时附关键测试输出或截图路径。无需单独报告文件、hash 清单、状态统计或全量日志归档;没运行、被跳过或受阻的项目不要写成通过。
-235
View File
@@ -1,235 +0,0 @@
# Visual observation:PR 0 验证结果与架构决策
日期:2026-09-08。生产基线:main `3c5f838`。分支:`codex/visual-observation-design`。
本次只新增设计、合成 fixture 和诊断工具。没有生产算法变更,没有 commit/push。
总体设计及后续 PR 依赖见 [架构与实施计划](visual-observation.md)。
本文是历史证据;新会话请按 [验收清单](visual-observation-acceptance.md) 执行相关检查并说明结果,不继承本文的通过结论。
## 1. 环境与证据等级
| 环境 | 实际版本/说明 |
|---|---|
| PATH 中的 bsk | 0.1.7,未用于有效页面基线 |
| 仓库已有 target/debug/bsk | 0.2.0;`update --check` 报告发布最新版也是 0.2.0,没有安装更新 |
| daemon / connected extension | 0.2.0,protocol 1.1 |
| 页面基线 Chrome | 151.0.0.0,由 bsk browsers 报告 |
| 生产扩展构建 | `pnpm ext:build`,用户 reload 当前 main 基线构建后采集 |
| 独立协议实验 Chrome | 152.0.7977.76,临时无登录 profile;与用户浏览器不是同一个运行实例 |
| 基线测试 | VOM 47 tests / 2 files;相关扩展 217 tests / 14 files;全部通过 |
页面结果是实际 bsk + reload 扩展的结果。协议实验是独立 Chrome 的真实 CDP 响应。
临时 replay 是将合成响应送入 main adapter 的诊断,不代表另一条真实浏览器完整运行。
不同证据不混用,版本号本身不证明构建 commit。
所有内部页面原文、地址、iframe 标识、截图和用户信息仅保存在本机临时目录,不进入仓库。
本文仅保存匿名化事实,仓库 fixture 完全独立构造。
## 2. 原始问题已确认:表格加载完成后仍有观察缺口
有效就绪条件:前台可见页面已经绘制出网格、列标题和六行表格。不能使用以下替代条件:
- `document.readyState === complete`。
- AX 已有嵌入应用的工具栏。
- 页面出现 iframe。
- 背景截图仍为站点加载 logo。
本次初始后台窗口恰好出现“AX 已有工具栏,图像仍为加载占位”的情况。这些样本已排除。
在前台绘制完成后重新取得匹配的截图和 observe,结果为:
| 事实 | 结果 |
|---|---|
| 实际截图 | 可见列标题、网格与六行表格 |
| main observe | 63 个 ref,含外围和嵌入应用工具栏 |
| truncated | false |
| 表格列内容 | 没有相应文本表示 |
| visual ref | 没有 |
| 直接打开已观察到的嵌入应用 | 发现一个无 aria-label 的 Canvas,CSS 尺寸 1450×835、backing store 2900×1670,DPR 2 |
直接打开嵌入应用用于验证其渲染介质;其宽度变化是独立 tab 布局,不能拿这组坐标当父页面中的 crop。
父页面的截图/VOM 对照与直接嵌入页的 DOM 证据分别记录。
结论:主要能力缺口是视觉内容没有观察入口;不属于这次有效样本的文本预算耗尽。
新架构不能承诺从 Canvas 自动恢复单元格语义;正确结果是保留工具栏,并提供可截图的区域入口。
## 3. 合成最小复现
`table.html` → 跨站点 `grid.html`:一个 iframe、DOM 工具栏、一个无名 Canvas。
Canvas 绘制三列、六行,所有文字与原页面无关。
main 实际输出的关键部分:
```text
@view 1440x923
@e1 button "Page action"
Iframe "Table application"
navigation
@e2 button "Insert record"
@e3 button "Filter"
```
`ref_count=3`,`truncated=false`。截图已人工确认包含 `Canvas column A/B/C` 和六行。
这些像素文字既没有文本表示,也没有 visual ref。该最小复现已覆盖原场景的关键结构,
无需站点权限或生产选择器。
## 4. CSS 与语义/交互分离
真实浏览器实验确认:
- aria-hidden 祖先、inert 祖先、pointer-events:none 不会使 Canvas 自动视觉消失。
- `visibility:hidden` 祖先下的 `visibility:visible` 子 Canvas 可以恢复可见。
- `hidden` 元素被 `display:block` 覆盖时仍可绘制。
- opacity:0 祖先会使子树透明;不能仅检查 Canvas 自己的 opacity。
- display:none 无 layout bounds;visibility:hidden 可以仍有 bounds。
- AX fallback 文本不包含绘制的 `PIXEL TABLE 42`,不能用 fallback 存在作为发现去重条件。
决定:事实层分别保存 computed visibility、透明继承、语义暴露和交互属性。
视觉发现不读取 semantic keep/drop 结果;祖先索引只能传播符合属性语义的状态。
`checkVisibility()` 的实验作用仅限 CSS suppression,不用它证明 overflow、viewport 或遮挡结果。
常规 overflow 由独立几何规则负责;任意遮挡不进入首版精确能力范围。
## 5. 单位、frame 投影与截图
### 5.1 DPR 与截图像素是不同契约
独立 Chrome 上对相同 120×40 CSS clip 的结果:
| Emulated DPR | DOMSnapshot Canvas w×h | Screenshot w×h | layoutViewport/cssLayoutViewport 宽度比 |
|---|---|---|---|
| 1 | 120×40 | 120×40 | 1 |
| 1.5 | 120×40 | 180×60 | 1 |
| 2 | 120×40 | 240×80 | 1 |
因此该比值不能用作通用截图 DPR。原始 DOMSnapshot 单位与截图 raster scale 必须分别适配,
输出 PNG 尺寸是最终预算事实。不能把本组 DPR emulation 结果泛化为所有浏览器缩放组合。
### 5.2 三种缩放分别记账
- CSS `zoom:1.25`:120px 宽 Canvas 的布局 bounds 为 150px,运行时几何同步变化。
- Emulation.setPageScaleFactor(1.5):记录为 page-scale/pinch 类证据,不充当 UI zoom。
- 用户 Chrome 原生 UI 缩放到 90%:界面明确显示 90%,DPR 从 2 变为约 1.8,innerWidth 从 1440 变为 1600,visualViewport.scale 仍为 1;Canvas CSS 宽约 120.0000076,截图仍是 2880px 宽。main VOM 视口为 1600×1025。已恢复 100%。
现有 bsk 的 Meta+- CDP 输入未改变浏览器缩放,所以该无效果尝试不算缩放验证;
有效样本来自原生浏览器 UI 操作。真实 UI zoom 仅验证了上述点,不能宣称完整 zoom 矩阵已通过。
决定:使用 CSS 视口空间作为发现/截图区域公共契约;精确单位 adapter 的实现还必须在 PR 1
跑计划内组合回归,不用一个比例同时解决所有协议字段。
### 5.3 Frame 空间与 main 已知偏移
fixture local Canvas 原点为 (17,23)。
| 场景 | 实测 owner content 原点(target root) | 正确 top Canvas 原点 | main parser replay |
|---|---|---|---|
| 同 target frame,10px border | (50,610) | (67,633) | (57,623) |
| 上述 frame 内 nested,另有 7px border | (87,707) | (104,730) | (87,713) |
| OOPIF,10px border | (410,610) | (427,633) | 本表不混入非完整 replay 的 OOPIF 结果 |
根 target 的 DOMSnapshot 含三份 document(top/same/nested);OOPIF 在单独 target 中只返回自己的 document。
probe 通过实际 `TargetInfo.type=iframe` 验证 OOPIF,而非仅根据 origin 推断进程隔离。
子 frame scrollY=10 后,runtime local y 从 23 变为 13,snapshot document y 仍为 23。
owner scale(1.25) 后 content quad 宽度变为 375,child innerWidth 仍为 300。
决定:frame-local 原始矩形必须先减该 document 的滚动,再经过一次权威投影;同 target 的
owner quad 已是 target-root 坐标,不能重复逐层加偏移。禁止 border-box 替代 content origin。
Box 种类也必须进入契约:带 5px border 的 120px Canvas,其 snapshot/border quad 宽 130,
content quad 宽 120。视觉入口采用 border-box,实时验证同种 box;iframe 投影则采用 content quad。
带 border/padding 的 overflow owner,snapshot.clientRects 对应 `[clientLeft,clientTop,clientWidth,clientHeight]`,
本 fixture 为 `[5,5,68,38]`。裁剪边界应从 client box 派生,不能直接拿 ancestor border bounds。
单轴 `overflow-x:clip;overflow-y:visible` 的 computed style 保持分轴,必须分轴传播。
CSS zoom 1.25 后,上述 owner 的 bounds 变为 `[745,455.625,97,59.5]`,clientRects 仍为
`[5,5,68,38]`,offsetRects 为 `[596,365,78,48]`。直接按 border-width 比例推导 client box
会混入不同空间与整数取整。首版将有 CSS zoom/DOM transform 的 overflow 祖先组合明确降级,
不承诺精确 crop;普通未变换 overflow 及 iframe content-quad 投影保留支持。
## 6. Document 与 ref 时效
真实 CDP 实验:
| 操作 | frameId | loaderId | document backend | anchor |
|---|---|---|---|---|
| history.replaceState | 不变 | 不变 | 不变 | 仍属于当前树 |
| 跨 document 导航 | 不变 | 变化 | 变化 | 旧目标不可继续视为当前内容 |
| document.open/write/close | 不变 | 不变 | 本次实验不变 | 旧 Canvas 失去连接,documentElement 被替换 |
决定:身份取 target attachment epoch + frameId + loaderId + documentElement backend identity;
工具执行时仍检查 anchor.isConnected 及其所属 documentElement。不能仅凭 loaderId 判断有效。
DOMSnapshot 可提供根元素 backend identity,frame graph 需要保留 loaderId;当前 main graph
没有该字段,这是后续数据契约的明确修改点。
12 次静态几何采样完全相同;0.5px 移动可被检测;真实 90% 缩放引入约 1e-5 CSS px 的
数值误差。首版采用最大边误差 0.25 CSS px 作为保守工程阈值,超出要求重新 observe。
这不是大规模统计得到的通用稳定性保证;改变阈值需更新其测试及行为文档。
既有 eN 被下一次观察重用后,单靠旧字符串不能检测它来自上一代。内部 generation 不消除
该协议限制。本次不改变协议,只验证当前存储目标的身份/几何。Canvas 可以在身份与几何
不变时重绘;截图始终表达执行时图像,不承诺观察时像素快照。
## 7. 既有路径回归基线:用于防止负优化
诊断复制当前 renderer 到临时目录,在三个明确循环点加计数;生产文件未改变。
复现命令见 [fixture README](../../apps/extension/test-fixtures/visual-observation/README.md)。
| 输入 | Render stack visits | DOM context candidates | Same-container sibling visits |
|---|---:|---:|---:|
| 100 headings + 100 View buttons | 201 | 10,000 | 15,050 |
| 200 headings + 200 View buttons | 401 | 40,000 | 60,100 |
| 400 headings + 400 View buttons | 801 | 160,000 | 240,200 |
| 2,000 transparent wrappers + control | 2,001 | 0 | 0 |
| 4,000 transparent wrappers + control | 4,001 | 0 | 0 |
显式栈访问线性并通过深树输入;本组宽树上下文计数呈二次增长。这里记录的是现有路径的
特征,供后续同输入比较是否发生回退,不是本次要求修复的问题,也不是 Canvas 新算法的模板。
计数不是完整 CPU profile,更不能替代 Canvas 发现、裁剪、选择和截图各自的性能测试。
处理边界:
1. 既有语义、DOM 操作和截图路径对照 main 不发生性能回退;不把历史算法优化列为本次前置依赖。
2. Canvas 新路径按自己的规模和访问模式论证算法;发现、裁剪、去重、选择和截图分别验收。
3. 精确 crop key 去重的选择来自首版的覆盖等价契约和新增成本分析,与 #165 采用什么算法无关。
4. PR 7/8 分别报告既有路径回归、无 Canvas 页面的额外成本、有 Canvas 页面的新增成本。
新功能有合理的额外成本不等于旧算法负优化;重复执行旧工作则属于需要消除的回退。
## 8. 锁定首版策略与退化
| 决策 | 首版契约 | 理由/验收 |
|---|---|---|
| 去重 | 同 document + 同直接 parent + 四条 crop 边完全相等 | 无模糊聚类;误差导致重复比误合并更可控 |
| 索引容量 | 50,000 keys;耗尽后其余候选继续未去重选择 | 内存有界,不因早期小图耗尽而停止后续发现 |
| K | min(预算推导容量, 512) | 所有行仍逐条核算可变成本 |
| 默认视觉预算 | 未给总 maxTokens 时 2048 estimated tokens | 有显式 maxTokens 时全部纳入同一预算 |
| 摘要预留 | 固定提示行按实际估算器成本预留 | 不用不可验证的固定 token 常量表示某段文本 |
| 几何时效 | 最大边差 > 0.25 CSS px 即重新观察 | 身份和裁剪策略变化独立拒绝 |
| 图像 | max edge 2048、max pixels 4,000,000、最多两次 capture | 成功必须以实际尺寸满足后置条件 |
| 外部几何并发 | 4 | 单操作缓存;操作结束释放 |
| 新增纯计算取消 | 每 256 项形成工作块并让出事件循环 | 不仅是轮询无法及时更新的 signal |
这些数值是可审查的首版资源政策,不是声称已证明用户体验最优的测量结果。
参数可以在对应模块 PR 中经新的证据调整,但必须一并更新契约及测试。
进一步收敛:首版精确区域支持轴向矩形、普通 overflow 链、正向轴对齐平移/缩放与 frame。
旋转、透视、clip-path、mask、活动 pinch-zoom 进入明确的 geometry-unsupported 降级,
不给看似精确的 region ref,提示现有视口截图途径。不会把包围矩形偷换成精确区域。
## 9. 已执行的检查与下一阶段门槛
已执行:
- main VOM tests:47 passed。
- main 相关 extension tests:217 passed,含 frame/element geometry、observation、interaction、ref-store、VOM。
- `pnpm ext:build` 成功,用户 reload 后采集有效页面基线。
- 独立 Chrome browser contract probe:39 个断言通过;可复现命令见 fixture README。
- main 合成 capture replay、显式栈/上下文操作计数。
- 实际 Chrome UI 90% zoom 样本,恢复 100%。
- 原始表格与合成嵌入式表格均人工查看截图,并与未截断 observe 对照。
PR 0 交付的是已明确边界的设计和基线,不是未来功能已经通过:新的 discovery、typed ref、
freshness resolver、pixel planner 和 renderer 接线尚未实现。后续每个 PR 必须验证自己的
生产实现,不能把本次浏览器事实验证替代实现验收。
人工证据不设置默认 CI 对内网站点的访问依赖。合成 probe 可按需执行;页面原文不进仓库。
后续 PR 的职责和验收保持在主设计中,本文负责记录证据、决策及没有作出的保证。
-675
View File
@@ -1,675 +0,0 @@
# BrowserSkill 视觉观察:架构设计、边界约束与 PR 实施计划
> 日期:2026-09-08
> 状态:PR 0 设计验证稿;基线、浏览器实验和参数决策见 [验证结果](visual-observation-evidence.md)。生产实现尚未开始。执行步骤和成功条件见 [验收清单](visual-observation-acceptance.md),无需另写报告。
> 实施基线:从实施时确认的 main 新建分支;本次分析使用本地 main `3c5f838`。
> 参考材料:当前 `fix/vom-canvas-observation` 分支、PR #165、此前的 visual-surface 架构蓝图。旧实现和旧测试是证据来源,不是不可改变的规格。
> 本次范围:设计规格及可复现的合成 fixture/诊断工具;不修改生产代码,不继承旧分支提交。
## 1. 设计结论
本次工程的目标是:
**为 VOM 建立独立的视觉观察通道,共享可靠的采集与几何基础,通过明确的区域身份提供截图能力,并使处理成本随输入规模可预测增长。**
工作方式是先确定整体架构、行为契约、模块依赖、复杂度和验收矩阵,再分 PR 实现。分步实现不等于分步思考架构。任何实现阶段需要改变已定契约时,必须先更新设计及相关验收项,不得用局部特殊分支掩盖契约冲突。
本次不是一次目录整理,也不是通用视觉理解平台重写。设计完整覆盖当前能力的生命周期,但不为尚无需求和证据的未来能力预建实现。
### 1.1 必须成立的五个结果
1. observe 能表达“这里存在需要看图理解的 Canvas 内容”,不依赖该元素是否有可用 AX 语义。
2. 视觉入口的 document、frame、target、节点及坐标归属明确。
3. 视觉 ref 与 DOM 操作 ref 具有明确且不可矛盾的工具契约。
4. 截图解析当前目标并验证时效,返回浏览器最终合成的当前图像及真实尺寸。
5. 采集、索引、发现、选择、输出和截图均有复杂度或资源上限,并从首个实现 PR 开始验证。
### 1.2 不能用来证明成功的指标
- 代码文件拆得更多。
- 旧分支全部测试通过,但没有重新审核测试中的行为假设。
- 能识别某个特定 嵌入式 Canvas 表格 页面,却依赖站点 class、URL 或布局常量。
- 单个简单页面运行很快,但没有最坏输入分析。
- 文本出现 surface 标记,但其截图对应关系、时效和工具限制没有验证。
## 2. 问题本质与现状
### 2.1 main 的基础能力
当前 main 的主要路径为:
```text
DOMSnapshot + 各 frame 的 AX
→ FrameDocument
→ semantic graph:构建、解析、结构整理
→ VomScene
→ VOM 文本及 DOM ref
```
已经存在 frame document、语义图、几何模块、ref store、截图路径和显式栈渲染器。应复用这些职责,不在旁边建立第二套长期运行的采集、frame traversal 或语义树。
缺口是:元素的 DOM/AX 描述不等于 Canvas 内部的文字、图形和布局。补 role、名称或普通可点击 ref,并不能建立完整的视觉观察能力。
### 2.2 必须分别解决的问题
| 问题 | 要回答的具体问题 | 不属于该问题的内容 |
|---|---|---|
| 视觉发现 | 哪些 Canvas 构成可观察候选? | 它是否可点击、输出多少条 |
| 空间定位 | 候选在顶层视口的区域是什么? | 它是否重要、属于哪个业务画布 |
| 输出选择 | 有限预算下展示哪些入口? | 改写几何事实、推断业务实体 |
| ref 身份 | 入口绑定什么目标、允许哪些工具使用? | 像素缩放、内容识别 |
| 时效验证 | 当前目标是否仍符合此次观察契约? | 保证 Canvas 像素未发生重绘 |
| 图像获取 | 如何获取当前合成区域并限制成本? | 重建 Canvas 内部对象 |
### 2.3 当前分支值得保留与需要重审的部分
值得保留:语义/视觉双通道、视觉附录、单节点截图锚点、frame 几何共享、像素预算、相关失败场景和 fixture。
需要重审:
- 将 `aria-hidden`、`inert` 与视觉隐藏合并排除。语义暴露、交互资格、视觉存在是不同维度。
- 将旧区域与实时区域的交集当作时效验证。区域重合不能证明目标仍对应原观察。
- 将几何重合和共同父节点解释成逻辑画布身份。
- 将名称有无作为重要性的绝对优先条件,可能让大量有标签的小图排在无标签主画布之前。
- 视觉附录依赖 DOM 渲染后的剩余预算,可能使视觉通道完全消失。
- 在每个候选上重复祖先遍历,或假设引入空间索引便自动消除了最坏二次复杂度。
PR #165 优化了既有渲染及上下文查询路径。本次将它作为性能回归保护基线,不作为 Canvas 算法模板,也不据此要求重做原有算法。Canvas 的发现、裁剪、选择和截图具有不同的数据规模与访问模式,应分别设计和验证适合自己的算法。
PR 0 已在用户指定页面确认:表格实际绘制后,main 输出外围工具栏而没有表格列内容及视觉入口,且未截断。直接打开嵌入应用确认存在无标签 Canvas。内部页面资料仅留临时目录;入库使用匿名化结论和独立合成 fixture。
## 3. 范围收敛
### 3.1 本次纳入
- Canvas 元素锚定的视觉观察入口。
- 顶层、同 target 子 frame、OOPIF 及嵌套 frame 的身份和坐标归属。
- 已明确支持的视口及 overflow 裁剪。
- 无标签、存在 AX、不可交互但有视觉内容等候选情况。
- 有界输出选择及保守的重复截图入口去重。
- 视觉 ref 的工具契约和 document/几何时效检查。
- 截图区域解析、像素预算、真实输出尺寸验证和有界重试。
- 深层 DOM、密集小图、重叠 Canvas 和多 frame 的性能约束。
### 3.2 本次不纳入
- OCR、图像语义识别、Canvas 内部对象树。
- 画布内部点选、拖拽或内部对象 ref。
- 地图 tile 拼接、跨节点逻辑画布聚合、多 anchor 区域。
- 面向所有 SVG、视频、CSS 背景等介质的通用发现插件系统。
- 任意页面遮挡、透明度、clip-path 和 mask 的精确像素级重建。
- 改写所有现有语义算法、表单、hover probe 或 recording 协议。
- 新的 ref 字符串协议、跨 observe 持久对象 ID、分页接口。
边界外的能力可以提出独立设计,不能通过放宽当前去重阈值或给单节点 target 塞可选字段实现。
## 4. 总体架构
```mermaid
flowchart TD
Browser[浏览器 / CDP] --> Capture[CaptureCoordinator]
Capture --> Batch[RawCaptureBatch]
Batch --> Geometry[GeometryContext]
Batch --> Normalize[Normalize + DocumentIndex]
Geometry --> Normalize
Normalize --> Facts[ObservationFacts]
Facts --> Semantic[现有 SemanticPipeline]
Facts --> Discover[VisualDiscovery]
Discover --> Candidates[VisualCandidates]
Semantic --> SemanticScene[SemanticScene]
Candidates --> Select[VisualSelection]
SemanticScene --> Context[输出上下文 / 活跃层策略]
Context --> Select
SemanticScene --> Renderer[ObservationRenderer + BudgetPlan]
Select --> Renderer
Renderer --> Output[文本 + Typed Ref Descriptors]
Output --> Registry[RefRegistry]
Registry --> Resolve[当前目标及区域解析]
Browser --> Resolve
Resolve --> Pixel[PixelPlanner]
Pixel --> Shot[CaptureExecutor]
Browser --> Shot
```
### 4.1 架构原则
1. 共享事实,分开解释。视觉存在不复用语义节点的保留/排除结论。
2. 语义关系是视觉区域的可选上下文,不是视觉身份的根。
3. 几何只有一套解释和投影实现,观察与工具执行使用不同生命周期的上下文。
4. renderer 只处理数据;浏览器调用止于采集或工具执行边界。
5. 预算选择只影响输出,不回写采集事实。
6. ref 的 kind 决定工具适用性,不能由调用者传入另一份可矛盾能力数组。
7. 模块失败必须可归因;不能将“没有候选”与“采集失败”混为一谈。
8. 所有资源限制都具有可解释的退化行为,不只是一句“超过上限停止”。
## 5. 模块划分与实现位置
以下为目标职责和建议位置,不要求第一个 PR 搬动全部文件。迁移文件与修改算法尽量分开;模块完成切换后删除被替代路径。
```text
apps/extension/src/tools/
observation.ts # 工具入口与结果接线,逐步减去内部实现
observation/
capture-coordinator.ts # CDP 编排及批次生命周期
capture-types.ts # 原始批次、归属、阶段状态
normalize.ts # 原始节点 → 归一化事实
document-index.ts # 按 document 建立共享索引
facts.ts # 下游可读事实契约
geometry/
coordinate-types.ts
frame-context.ts # frame/target 投影权威入口
primitives.ts # 复用/迁入现有数学实现
vom/
capture.ts # 保留或拆出的 DOMSnapshot adapter
frame-document.ts # 复用现有 frame 归属逻辑
semantic-graph/ # 保留语义职责
visual/
types.ts
discovery.ts
selection.ts
coverage-dedup.ts
refs/
target.ts
resolve.ts
screenshot/
resolve-region.ts
pixel-plan.ts
capture.ts
apps/extension/src/session-manager/
ref-store.ts # session 生命周期与注册存储
packages/vom/src/
types.ts # 纯场景及渲染契约
render.ts # 保留显式栈语义遍历
render-budget.ts # 唯一输出预算核算
visual-appendix.ts # 纯数据视觉条目输出
```
不同时保留旧 `geometry.ts` 与新 primitives 的两套数学实现。上述目录可按现有工程规范调整,但职责与依赖不得随意调整。
### 5.1 CaptureCoordinator
输入:tab/target、已验证的 URL、取消信号、采集选项。
输出:带来源、时间范围、document 身份和阶段状态的 RawCaptureBatch。
- 按 target 分组采集;DOMSnapshot 每 target 一次成功采集,重试单独计数且有界。
- AX 按协议支持的 frame/target 组织,数量有预算。
- 原始采集完成后才能运行会改变页面状态的 hover probe,保留已有行为顺序。
- 明确记录无法采集的 frame,不能把失败 frame 的数据借用其他 target 补齐。
- 不判断 Canvas 重要性,不投放 ref。
批次是来源和生命周期边界,不是浏览器原子事务。`Readonly` 或冻结对象不能证明各 CDP 响应来自同一瞬间。遇到可检测的导航/document 更换时应中止或有界重采集,不能拼接不同 document 的数据。
### 5.2 GeometryContext
输入:该次操作的 frame graph、原始 metrics、按需取得的 owner quad。
输出:frame-local/target-local 到 top-viewport 的投影、裁剪及当前节点区域解析。
- 坐标类型同时携带空间及所属 frame/target;仅标记 `frame-local` 不足以防止错 frame 使用。
- 原始 DOMSnapshot 单位、CSS 视口单位、截图像素单位分别适配。
- Canvas 视觉入口采用元素 border-box 的可截图区域;实时解析明确请求同一种 box,不能沿用默认 content-quad 路径再将边框差异误判为移动。iframe 子视口投影仍采用 owner content quad,两者职责不同。
- 普通 overflow 裁剪使用 ancestor client/scrollport box,不使用祖先 border bounds。adapter 保留 includeDOMRects 的 client offsets/size 及其单位;存在缩放时先转换到相同坐标空间,禁止直接相加不同单位。证据不足时明确降级,不猜测 border/padding 常量。
- 一个操作内复用 metrics、owner quad 和 projection promise;不跨 observe 或 tool execution 复用 live geometry。
- 合成变换可以预计算;裁剪边界复杂度须真实计入,不能靠 promise cache 就宣称全部计算线性。
- 不允许通过 iframe border box 加减常量替代有证据支持的 content geometry。
### 5.3 Normalize / ObservationFacts / DocumentIndex
输入:原始批次及几何上下文。
输出:按 document 隔离的只读事实和有界共享索引。
- Raw node 不能含语义不明的 top rect;normalized node 的派生字段由 normalization 创建。
- 节点 key 包含 document/target 所属信息,不能仅用 backendNodeId 跨 target 建表。
- 建立 node lookup、parent/children 等基础索引;派生索引必须有明确消费者和复杂度。
- 用显式栈和 visiting/visited 状态处理任意深度及异常环。
- orphan 保留“祖先信息不完整”状态;可以在已知范围内生成保守候选,但不能伪装成完整裁剪证据。
- 视觉样式、AX 暴露、交互资格分开保存,禁止万能 `excluded` 字段承载全部规则。
- 不给每个节点复制完整祖先数组或完整裁剪链;必要时共享结构或按需解析,计入内存预算。
### 5.4 SemanticPipeline
消费归一化事实,输出现有语义场景。通过窄 adapter 接入新事实契约;迁移完成后删除不再需要的 adapter。
- 不因为视觉发现需求改变 semantic keep/drop、名称恢复或可交互推断规则。
- 不把视觉候选塞回语义树遍历。
- PR #165 的显式栈、上下文索引及行为回归保留。
- 偶然发现的既有语义性能问题只作范围外记录,不构成本次工作项或前置依赖。本次只要求不使原有路径发生性能回退。
### 5.5 VisualDiscovery
输入:ObservationFacts。
输出:候选及发现状态,不含最终 ref,不依赖 token 预算。
候选表达一个 Canvas 节点锚定的截图区域。必须具有可解析地址、支持范围内的几何证据和非空候选区域。
- 无 label 不排除;有 AX 内容也不能证明像素内容已被完整描述。
- `aria-hidden`、`inert`、`pointer-events:none` 不单独构成视觉排除原因。
- 零尺寸、无渲染区域、已证实的完全透明/隐藏、支持范围内的完全裁剪可排除。
- `visibility` 等继承与覆盖行为必须按实际 CSS 语义处理,不能把祖先 hidden 一律不可逆传播。
- `hidden` 等属性不应替代最终渲染事实;存在特殊状态时按实验确认的规则处理。
- agent 自身 overlay 排除应按所属 document/target 传播。
- 常规轴向 overflow 累积在一致的坐标空间计算;变换后不能把 top-space 包围矩形当作精确 local-axis clip。
- 不重建任意遮挡。几何区域是截图候选范围,不保证范围内每个像素均来自该 Canvas。
### 5.6 VisualSelection / CoverageDedup
输入:候选、输出上下文及预算规划所需参数。
输出:选中候选、明确的遗漏/退化状态。
- 默认以区域覆盖量等可解释几何依据排序;label 是命名事实,不作为压倒面积的优先条件。
- 活跃层优先级属于选择策略;被降低优先级不等于物理不可见。
- 排序键预计算;同层按可见面积降序,再按 frameId、top y、left x、backendNodeId 升序决定顺序。字符串按代码单元字典序、数值按数值比较;同一精确去重组取该顺序最小者。不把跨批次 backend id 稳定性作为保证。
- 去重只减少重复截图入口,不产生新的 logical surface。
- 首版仅在同 document、同直接 DOM parent、四条裁剪边数值完全相同的候选之间去重,使用精确 key 的 Map;不使用近似 IoU、尺寸 band 或空间邻居比较。亚像素不同的范围保留独立入口。
- 首版不输出无法证明的 `layers=N` 业务解释。去重成员只是覆盖证据,不是内部对象模型。
- 精确去重索引达到 50,000 个 key 后,剩余候选以未去重形式继续进入有界选择,记录去重退化;不得直接丢掉剩余区域。索引已知 key 仍可命中去重。
- 即使没有去重,top-K 选择和总输出成本也必须有界。
### 5.7 ObservationRenderer / BudgetPlan
输入:SemanticScene、视觉选择结果和输出预算。
输出:文本及与已提交行严格对应的 typed ref descriptors。
- 唯一模块核算总文本预算;摘要、头部、条目和截断提示全部计入。新 observe 的统一账本为每个实际输出行分别 `ceil(line.length / 4)` 后求和,length 使用 JavaScript UTF-16 单位;不混用整段核算与逐行核算。
- 保留语义显式栈和 DOM ref 相对顺序;视觉条目在语义阶段之后提交。
- 有视觉候选时,规划阶段预留简短视觉提示;其余视觉条目使用明确分配的剩余预算。
- 极小预算不保证语义与视觉同时完整,必须产生确定的截断结果。
- 写入行与提交 ref 在逻辑上原子:未写入的条目不能留下可解析 ref。
- 使用既有 token 估算方法时,预算表示“估算 token 上限”,不能宣称精确模型 tokenizer 上限。
- 新 observe 预算规划若发现显式预算不足以容纳最小协议头,返回 insufficient-budget,由工具层映射为既有 invalid_params,附最低估算成本;不生成半截协议头或新 ref。snapshot 的旧预算语义不在本次顺带改变。这是 observe 极小预算的明确兼容性例外。逐行统一账本也可能改变临界预算的截断位置;足够预算下无 Canvas 的语义文本及顺序保持不变。
- 首版固定视觉提示为 `[visual] Canvas regions present; use screenshot`,成本由同一估算器计算。有候选但条目不足时仍尝试保留提示;连提示也容不下则只返回合法的截断语义输出,truncated=true。未知计数不伪装成精确遗漏数。
- 输出标签最多 160 个 Unicode code point,再经 JSON escaping 与整行成本核算;排序不依赖标签长度。原始采集输入量 B 仍单独计入,不因输出裁剪便宣称采集内存有界。
- `snapshot` 继续使用既定静态语义输出;新视觉入口首先仅接入 `observe`。不要顺带改变 recording/其他调用方默认选项。
### 5.8 RefRegistry / Resolve
保存 session/tab 隔离的目标描述。DOM 与视觉目标通过判别联合区分;kind 为权威,外部展示能力可从 kind 派生。
```ts
// 概念契约,具体类型名与仓库适配时确定。
type RefTarget =
| { kind: "dom"; address: NodeAddress }
| {
kind: "visual-region";
anchor: NodeAddress;
observed: ObservedGeometry;
observationId: ObservationId;
};
// NodeAddress 必须能表示 tab、CDP target、frame、document 身份及 backend node。
// 坐标值同时保留所属空间及 owner,不使用无归属 Rect 穿越模块边界。
```
工具适用性:
| 消费入口 | DOM ref | Visual ref |
|---|---|---|
| screenshot | 维持既有 DOM 截图策略 | 使用视觉区域策略 |
| click/fill/hover/select 等 | 维持既有检查 | 拒绝,且拒绝发生在派发输入前 |
| get_html | 维持既有行为 | 拒绝;不隐式降级为 anchor 的 HTML |
| 人工帮助定位 | 维持既有行为 | 若当前入口必须解释为 DOM 目标则拒绝;未来支持视觉区域需另立契约 |
所有 ref 消费者都必须通过类型化 resolver,不能依赖调用者记得传可选 capability 参数。
现有 `eN` 重用存在明确限制:如果新的 observe 已将 `e1` 重新绑定,单凭客户端传来的旧字符串 `e1` 无法判断它来自上一代。内部 generation 不会自动解决该问题。观察身份采用 target attach epoch + frameId + loaderId + documentElement backend identity,并在执行时验证 anchor 仍连接到匹配的 documentElement;loaderId 单独不足以覆盖 document.open。
本次保留当前“最新观察映射”的外部语义;只承诺能检测当前已存条目的 document/节点/几何失效,不承诺识别所有跨代旧字符串。彻底解决需独立协议设计。
### 5.9 ScreenshotExecution
分成三部分:
1. `resolve-region`:按目标种类解析当前身份与几何、验证 document 和区域变化。
2. `pixel-plan`:纯数值函数,输出 clip scale 等计划。
3. `capture`:overlay suppression、CDP 截图、真实尺寸读取、有界重试、取消及 dialog 集成。
- Visual ref 不自动滚动,不通过旧、新区域交集掩盖位移。
- 首版对超过既定容差的位移、尺寸或裁剪变化返回“需要重新观察”;容差在 PR 0 由测量误差证据锁定。
- 截图返回执行时浏览器最终合成图,不是观察时像素快照。
- document 身份验证不能替代几何验证;几何相同也不能证明 Canvas 内容未重绘。
- 像素规划采用明确 metrics 适配值;真实 PNG 尺寸是输出预算后置条件的依据。
- 尺寸头校验不宣称完整 PNG 内容验证;缺失/不合法数据与预算超限分开归因。
- 成功返回必须满足像素上限;有界重试仍失败则返回失败,不能猜测尺寸或无界重拍。
## 6. 算法与资源预算
### 6.0 两类独立的性能责任
**既有路径:防止回退。** 在相同输入、选项和环境下对照 main,检查原有语义处理、DOM ref、普通交互与截图的工作量、内存和实际耗时。不得因接入 Canvas 而重复运行语义解析、重建已有索引、增加原有节点的上下文查询,或撤销 #165 的优化。本次不要求把既有算法统一改成某个复杂度。
**新增路径:独立设计。** 围绕 Canvas 数量、frame 数量、裁剪复杂度、输出预算及截图像素量分析新成本;不照搬旧 renderer 的算法,也不以“旧算法使用了索引/显式栈”代替新问题的算法论证。下文复杂度与资源要求约束新增或本次实际改动的职责。
性能验收分三组,分别报告,不能互相替代:
| 验收组 | 对照方式 | 必须说明的结果 |
|---|---|---|
| 既有语义/操作路径 | 同一输入在 main 与新分支运行,隔离新增视觉工作 | 既有阶段未引入额外遍历、重复请求或可归因的性能回退 |
| 无 Canvas 页面 | 实际完整 observe 对照 | 仅保留必要的候选检测开销;确认无候选后不建视觉索引、不执行视觉几何查询或截图 |
| 有 Canvas 页面 | 单独测量视觉各阶段,再测完整 observe | 新成本与 C/F/P/K/像素等规模相符,资源有界;不把合理的新功能成本误记为旧算法退化 |
实际耗时需要多次同环境对照并结合操作数判断,不能仅凭一个容易波动的耗时值下结论。新增字段采集或共享模块调整的额外开销也必须归因并报告,不能藏在“原有成本”中。
### 6.1 规模定义
| 符号 | 含义 |
|---|---|
| B | 原始协议数据及字符串总量 |
| N | 采集 DOM 节点数 |
| A | AX 节点数 |
| F / T | frame 数 / CDP target 数 |
| C | 视觉候选数 |
| K | 最大选中输出条目数 |
| Q | 去重实际候选比较次数 |
| P | 几何计算量,如顶点处理、裁剪操作总数 |
| H | 语义上下文查询实际处理的数据总量 |
| L | 最终输出文本长度 |
不能忽略 B、P、H 后笼统宣称整个管线 `O(N)`。原始数据解码至少与输入量相关,复杂裁剪和语义上下文的总成本需要显式计算。
### 6.2 模块性能契约
| 阶段 | 复杂度/上限目标 | 禁止的实现 |
|---|---|---|
| 原始解码 | 与 B 线性相关 | 为每个节点重新解码共享列或字符串 |
| 文档索引 | `O(N + A + F)`,不含协议字节解码 | 每个候选全文扫描 |
| 可合成祖先状态 | `O(N)` 建立,候选常数次查询 | 每 Canvas 完整祖先 walk |
| Frame 几何 | 变换/证据按 frame 复用,额外成本计入 P | 每 node 查询 metrics/quad |
| 发现 | 索引后 `O(C)`,额外几何成本计入 P | 发现函数内重新查 CDP |
| top-K | `O(C log(K+1) + K log(K+1))` | 无需要地重复全量排序 |
| 去重 | 精确 key Map,摊销 O(C),索引至多 50,000 个 key;首版 Q=0 | 引入模糊匹配和同 bucket 两两比较 |
| 新增视觉渲染与既有渲染接线 | 新增工作按选中条目及 L 核算;既有 H 对照 main 不回退 | 为视觉条目重新执行语义遍历或扫描全部节点 |
| 截图 | 调用和重试次数有界,成功图像像素数有界 | 无限重试、猜测输出尺寸 |
K 从预算允许的最小条目成本推导,并受异常安全上限约束;实际每行仍核算成本。面积优先 top-K 不应被描述为严格求解“可变长度条目预算最优装包”,本次不实现此类优化问题。
### 6.3 首版资源参数及锁定规则
以下是 PR 0 锁定的首版策略参数。它们是有明确退化语义的工程预算,不是性能最优值的统计结论;调整须更新对应契约测试:
- Visual 截图最大边:2048 px;最大面积:4,000,000 pixels。
- 单次视觉截图最多 2 次 capture 尝试。
- Frame 几何外部请求最大并发:4。
- K 异常安全上限 512;正常数量由预算与条目成本决定。
- 几何变化阈值:top-viewport CSS px 的四条边最大差值超过 0.25 即要求重新 observe;document、anchor、裁剪策略改变不受该容差豁免。
- 没有显式 maxTokens 时,视觉通道估算 token 预算 2048;有显式预算时,优先为固定摘要行按当前估算器计算实际预留量,条目用剩余预算,全部计入同一总预算。
- 精确覆盖去重索引最多 50,000 个 key;耗尽后继续未去重 top-K,标记 dedup-degraded;不得停止扫描剩余候选。
- 新增纯计算阶段每 256 个节点/候选形成可取消工作块;需要实际让出事件循环,而不是只读一个尚无机会更新的 AbortSignal。
每个限额均须有单位、适用阶段、计数口径和耗尽行为。数字集中在所属模块策略配置中,不散落为多个含义相近的常量。若 PR 0 未锁定参数及退化契约,对应实现 PR 不得以随意常量合入。
### 6.4 性能测试方法
- 节点访问、祖先推导、候选比较、CDP 请求数、最大并发及中间集合大小均做可断言计数。
- 对新增树处理使用深树/宽树测试,证明无调用栈溢出或重复全量扫描;既有 renderer 的对应输入用于 main 前后回归对照,不要求本次消除其历史复杂度。
- 规模翻倍时检查操作数增长是否符合分析,避免只看一次墙钟时间。
- 精确去重索引至少测试:全部重合、亚像素差异、同 crop 不同 parent、全部互不相交、容量耗尽;未来若引入空间索引,必须另增退化测试。
- 大量有名小图与无名主画布并存,验证排序不会因标签策略丢失主要区域。
- 保留 #165 回归;为新增模块增加性能契约,而不是声称旧测试覆盖整个链路。
- 真实耗时基准用于观察额外成本和实际收益,需记录环境;不使用耗时阈值代替算法验证。按验收清单的性能步骤 预热 3 次、交替采样 20 次,分别报告旧阶段和新增成本;旧阶段中位数增加超过 max(基线 10%, 2ms) 时独立复测,两轮均超过且可归因于改动则 FAIL,无法排除噪声则 BLOCKED。
## 7. 失败、降级与一致性
内部状态至少区分:
```text
capture-unavailable
document-changed
frame-ownership-unresolved
geometry-unavailable
geometry-unsupported
outside-supported-clip
discovery-partial
selection-truncated
dedup-degraded
target-detached
geometry-changed
image-invalid
pixel-budget-exceeded
```
这些是内部分类,不要求逐项扩展外部 RPC code。外部沿用既有错误分类并给出准确 message/必要 detail;不可把所有时效或几何失败叫权限问题。
原则:
- 视觉分支失败不应使已获得的语义结果无故丢失。
- 没有候选、没有预算、没有采集成功必须可区分。
- exact omitted count 只在可计算时输出;原始候选数与去重后的区域数不能相加冒充精确数量。
- 捕获批次不保证页面静止;可检测的身份变化必须处理,不可检测的像素重绘不作虚假保证。
- 取消在阶段边界和有界工作块之间检查,不能在大型纯计算循环结束后才响应。
## 8. 兼容性与影响面约束
1. 从 main 新建实现分支;不将当前分支整体 cherry-pick 作为迁移方式。
2. 没有视觉候选的页面,语义文本、DOM ref 相对顺序和操作行为维持原有契约。
例外为上述 observe 显式预算小于最小协议头的 invalid_params 行为及统一账本导致的临界预算截断位置变化;足够预算不改变旧语义。
3. 有视觉候选的 observe 允许新增视觉文本,并按新预算规划改变截断位置;这是明确行为变化。
4. `snapshot`、recording、hover probe 默认行为不随视觉通道接入而隐式改变。
5. 修正已证明错误的坐标允许改变输出位置,不以错误 golden 为兼容目标。
6. 几何共享改动必须覆盖已有 DOM 点击/截图路径回归;不以 Canvas 测试代替。
7. CLI/协议外形优先保持;确需新增外部字段、错误子码或参数时,必须在对应 PR 明确列出全部消费者。
8. 不进行无关格式化、目录大搬迁、依赖升级或语义启发式修改。
9. 不长期保留两套 capture decoder、metrics adapter、frame resolver、token accounting 或 pixel planner。
10. 不把站点特征、fixture 标签、CSS class 或 URL 放入生产决策。
## 9. 设计验证清单与关闭方式
| 待验证项 | 证据 | 必须形成的决定 |
|---|---|---|
| 嵌入式 Canvas 表格 原始失败 | main 的观察输出、对应截图及结构证据 | 失败归属、最小复现和成功标准 |
| DOMSnapshot / metrics 单位 | DPR、zoom、scroll、frame 矩阵 | 原始单位适配和转换入口(实验结论见证据文档) |
| frame / target 坐标 | 同 target、OOPIF、nested、带边框/变换样例 | content quad 来源、坐标归属及支持范围 |
| document 身份 | 导航、同 frame document 替换、target 重建 | 使用哪些现有标识/事件,何时失效 |
| CSS 视觉规则 | visibility 覆盖、opacity、hidden、单轴 overflow | 可合成状态和不支持情况 |
| 时效容差 | 静止页面测量噪声、可控移动/缩放/裁剪变化 | 容差单位、比较规则和失败行为 |
| 预算 | 调用方 max_tokens 使用方式、密集画布样例 | 提示预留、K、限额与截断规则 |
| 去重 | 重合、近重合、不同父节点、相邻 tile | 首版等价规则和 Q 上限 |
验证结果与最终决定见 [PR 0 证据及决策](visual-observation-evidence.md)。首版精确几何支持收敛到已验证的轴向矩形、正向轴对齐平移/缩放、普通 frame/overflow 链。旋转、透视、clip-path、mask 和活动 pinch-zoom 不承诺精确区域 ref:返回 geometry-unsupported 状态并提示使用现有视口截图。后续扩展必须独立验证。浏览器 UI zoom 与 CSS zoom 分开验证,不能用 pinch emulation 代替 UI zoom。
进一步限定:CSS zoom/DOM transform 作用于 overflow 裁剪祖先的组合,首版也进入 geometry-unsupported。
实验表明 snapshot.bounds 已缩放,而 clientRects/offsetRects 仍有未缩放和整数取整语义,不能靠一个宽度比准确恢复所有边。
普通 Canvas 的 CSS zoom bounds 与 iframe content quad 缩放仍可使用;这不意味着缩放后的任意 DOM scrollport 都已获得精确支持。
Facts 必须保留 client box、transform/zoom 及 clip-path/mask 等支持性证据,discovery 返回状态而不是另写近似投影。
## 10. PR 划分与依赖
PR 0 是设计与证据入口;PR 1—6 建立模块;PR 7 接入公开 observe;PR 8 完成跨模块验收和迁移收尾。编号是顺序计划,不是实际 GitHub PR 编号。
```text
PR 0 设计规格与实验
└─ PR 1 几何与身份契约
└─ PR 2 采集批次与共享事实
└─ PR 3 视觉发现
└─ PR 4 选择、去重与预算
PR 1 + PR 2 ──→ PR 5 类型化 ref
PR 1 + PR 5 ──→ PR 6 视觉截图执行
PR 2 + PR 3 + PR 4 + PR 5 + PR 6
└─ PR 7 observe 组合与接线
└─ PR 8 整体验收与迁移收尾
```
独立模块可以使用人工输入测试,不必等待真实上游接线。实现可依赖前序已合入 main 的 PR,避免多个大型长期分支互相漂移。阶段未接线的模块必须有紧邻的消费计划,不保留无期限死代码或对用户暴露半成品。
### PR 0:规格收敛、最小复现与基线
完成:
- 审核现有分支 fixture,区分正确规格与需要推翻的预期。
- 记录 嵌入式 Canvas 表格 原始失败及 main 基线,建立独立于站点的最小 fixture。
- 完成第 9 节实验,锁定核心数据类型、支持范围、容差和资源参数。
- 固定 #165 的相关回归结果和操作数基线;不夸大旧测试覆盖范围。
- 将行为契约映射到后续每个模块的测试清单。
不做:生产算法替换、全目录迁移、正式视觉通道接入。
验收:核心身份、坐标、裁剪、预算和失效规则不存在未指定行为;未支持情况有明确策略。若仍有未知项,先调整范围而不是让后续 PR 猜测。
### PR 1:统一几何与节点身份契约
完成:
- 引入有 owner 的坐标和地址类型,区分 frame/target/top/CSS/image 空间。
- 统一 metrics 解释、frame projection 和 owner geometry 查询。
- 共享单次操作内的 promise cache;保证采集和执行使用独立上下文。
- 给既有 DOM 几何入口做窄适配;移除已被替代的投影路径。
不做:Canvas 发现、排序、ref 外部行为变化。
验收:scroll、border、同 target、OOPIF、nested、支持范围内的 transform/DPR/zoom 矩阵通过;已有 DOM 点击与截图回归通过;请求计数和几何成本可解释。
### PR 2:采集批次、归一化事实和共享索引
完成:
- 将原始解码、采集编排、normalization 明确分开。
- 按 target 去重采集,记录 document 身份、时间范围和阶段失败。
- 建立 ObservationFacts、document 隔离 lookup/children 索引和正确的祖先状态结构。
- 现有 SemanticPipeline 通过 adapter 消费新事实;保持 hover 在静态采集后发生。
- 删除不再使用的可变中间态和重复解码/归属推断路径。
不做:变更语义保留规则、视觉输出和截图政策。
验收:语义回归及 frame ownership 通过;deep/wide/orphan/cycle 有明确结果;解码、索引、采集数量符合预算;无每节点完整祖先复制。
### PR 3:纯数据视觉候选发现
完成:
- 实现 VisualCandidate 及 discoverVisualCandidates。
- 落实无标签、有 AX、aria-hidden/inert/pointer-events、透明/隐藏、裁剪规则。
- 候选关联 document/anchor/geometry,语义上下文只作为可选信息。
- 返回发现失败或不完整状态,避免与无候选混淆。
不做:ref 分配、截图、token 排序、逻辑画布合并。
验收:每条发现规则都有正反 fixture;输入同一份事实输出确定;索引后的候选发现不重复扫 DOM 或走完整祖先链;不增加 CDP 调用。
### PR 4:有界选择、保守去重与输出预算计划
完成:
- 明确几何优先排序及确定性 tie-breaker,预计算比较键。
- 实现有界 top-K,K 受文本预算和安全限额约束。
- 实现精确覆盖 key 去重及索引容量耗尽后“继续未去重选择”的退化行为;首版 Q=0,不进行模糊两两比较。
- 建立纯 BudgetPlan 和视觉摘要/条目格式,统一预算核算契约。
- 明确候选数、输出数、遗漏数及去重退化信息的口径。
不做:多 anchor target、业务画布推断、生产 observe 接线。
验收:密集小图、无名主画布、全部重合、同 bucket 不等价、极小预算均有确定结果;比较次数、堆大小、输出成本有界。
### PR 5:类型化 ref 注册与工具解析
完成:
- RefStore 保存判别联合,绑定 document 及 observation 信息。
- kind 派生工具适用性,删除可独立构造的矛盾 capability 状态。
- 审核并迁移截图、交互、get_html、人工帮助等全部 ref 消费者。
- 保持 eN 外部语义,文档明确跨代字符串识别限制。
不做:引入新版 ref 协议、自动把视觉 ref 当 DOM anchor 使用。
验收:跨 session/tab 隔离;document 失效;全部工具矩阵通过;拒绝在输入派发前发生;旧 DOM ref 行为保持。
### PR 6:视觉截图的区域解析、像素规划与执行
完成:
- 分离 resolve-region、pixel-plan、capture executor。
- 实现 visual 不滚动、身份验证、几何容差及重新观察错误。
- 实现真实输出尺寸后置条件、有界重试和准确失败归因。
- 复用 overlay suppression、取消和 dialog 机制。
不做:Canvas 内部截图、像素冻结、多节点 union、内部点交互。
验收:移动/缩放/裁剪变化、节点移除、document 替换、DPR/zoom、无效图像、超预算和取消路径通过;成功图像符合像素预算;现有 DOM 截图回归通过。
### PR 7:观察结果组合与 observe 接线
完成:
- 将 Facts 的语义/视觉两路结果交给统一输出规划和 renderer。
- 保留 #165 显式栈和 DOM ref 相对顺序,追加视觉条目。
- 仅对已写入条目提交 ref,失败/截断状态正确传递。
- 视觉提示参与预先预算,避免 DOM 耗尽预算后完全无提示。
- 仅在 observe 默认启用新视觉通道;必要的 CLI/skill 使用说明同步更新。
不做:顺带修改 snapshot、recording 或 hover 默认策略。
验收:从人工 facts 到文本/ref 的集成测试、真实单 Canvas observe → screenshot 闭环通过;无 Canvas 页面回归通过;极小预算无悬空 ref。
### PR 8:跨模块验收、性能证据与收尾
完成:
- 在真实浏览器运行完整 fixture 矩阵和 嵌入式 Canvas 表格 原始场景,保存可复现结果。
- 验证整个管线的请求数量、集合规模、比较次数、深树安全和真实耗时。
- 检查新模块是否存在重复解释、长期 adapter、无消费者代码或语义影响扩散。
- 收尾剩余迁移文档、调试信息说明、支持范围和已知限制。
不做:把此前未测试的性能正确性全部拖到本 PR,也不接纳新的功能范围。
验收:第 11 节全部通过;若发现某模块契约错误,修正在该模块并补对应验收,不在最终接线处加站点特判。
## 11. 总体验收矩阵
本节是范围索引,具体怎么做、什么算成功见 [验收清单](visual-observation-acceptance.md)。新会话按当前 PR 执行并直接说明结果即可,无需报告模板或逐项登记。修改契约时同步清单,避免两套标准。
| 维度 | 必测场景 | 主要责任 |
|---|---|---|
| 发现 | 无名、有 AX、aria-hidden、inert、pointer-events:none | PR 3 |
| 视觉状态 | 零尺寸、透明祖先、visibility 覆盖、支持范围内 hidden 状态 | PR 0/2/3 |
| 裁剪 | viewport、单轴/嵌套 overflow、支持范围内变换 | PR 1/2/3 |
| frame | 同 target、OOPIF、nested、border、缺失 owner | PR 1/2 |
| 身份 | target 冲突、节点移除、导航、document 替换 | PR 2/5/6 |
| 输出 | DOM 顺序、视觉提示、modal 策略、极小预算、遗漏口径 | PR 4/7 |
| ref | 工具矩阵、跨 tab/session、无悬空 ref | PR 5/7 |
| 截图 | 几何变化、DPR/zoom、实际像素、重试、overlay | PR 6 |
| 算法 | 深树、宽树、高密度、全部重合、索引退化 | 各模块 PR |
| 回归 | #165、普通 DOM 点击/截图、snapshot、recording、hover | PR 1/2/7/8 |
| 产品结果 | main 原始失败与新链路的 嵌入式 Canvas 表格 场景对照 | PR 0/8 |
## 12. 相比旧蓝图的明确调整
| 旧方向 | 本文调整 | 原因 |
|---|---|---|
| 全链路 transaction 先行 | 建立来源明确的 capture batch;不承诺原子快照 | 生命周期模型不能代替一致性证据 |
| candidate → stack → logical surface 必经分层 | candidate + 输出覆盖去重;logical surface 不进入本次 | 限制身份模型及截图复杂度 |
| 统一祖先 excluded | 视觉、语义、交互状态分开 | 避免加速错误规则 |
| 各阶段一律兼容 adapter | 仅消费者迁移需要时使用,并随切换删除 | 避免临时抽象堆积 |
| 性能作为后续优化阶段 | 每个模块设计/实现均有复杂度与计数验收 | 防止功能完成后再补算法 |
| renderer 最后尝试附录 | 先规划预算,再分阶段输出 | 避免视觉入口系统性消失 |
| observed rect 与 live rect 求交 | 身份和几何分别验证 | 避免成功返回错误截图片段 |
| generation 暗示 freshness 已解决 | 明确现有 eN 重用限制 | 防止无法实现的旧字符串识别承诺 |
| 当前测试全部视为兼容基准 | 先审核期望,再保留正确回归 | 测试不能替代行为规格 |
## 13. 完成定义与设计变更纪律
工程完成必须同时满足:
- 五条核心结果全部成立,真实 嵌入式 Canvas 表格 场景及独立 fixture 均有证据。
- 每个模块只有一个权威实现,依赖方向与本文一致。
- 坐标和节点地址跨边界时身份明确,失败可归因。
- 视觉发现不依赖语义树保留结果,不混淆视觉与交互属性。
- 性能上限及退化可测试;复杂度没有遗漏协议输入量和几何工作量。
- 现有语义、DOM 操作及 #165 成果受到明确回归保护。
- 当前支持范围、近似/拒绝行为和 ref freshness 限制已记录。
- 不存在为了通过原始站点而增加的生产特判。
每个实现 PR 的描述必须写明:解决哪个独立问题、改变哪个契约、哪些行为允许变化、验证哪些正确性与性能要求、删除哪条旧路径。若没有删除旧路径,应说明保留原因和明确的后续消费/移除 PR。
新的案例若属于已定规则范围,应修复负责该规则的模块;若要求改变目标身份、几何支持或对外协议,应先形成设计变更。不得在 observation 接线、renderer 或截图执行末端不断补跨层例外。
## 14. 参考与证据来源
- 本地 main:`3c5f838`;当前分析分支:`fix/vom-canvas-observation`。
- PR #165 本地合并提交:`5ed8287`;实现提交:`641d934`。涉及 renderer 上下文索引、显式栈等优化。
- 主要代码范围:`apps/extension/src/tools/observation.ts`、`tools/vom/capture.ts`、`tools/vom/frame-capture.ts`、`tools/vom/semantic-graph/`、`tools/frame-geometry.ts`、`session-manager/ref-store.ts`、`packages/vom/src/render.ts`。
- 当前分支参考:`tools/vom/rendered-surfaces/`、`tools/visual-screenshot-plan.ts` 及相关 fixture/test。
- 原 visual-surface 架构蓝图作为历史参考,不是本设计的执行规格。
- 浏览器语义参考:[aria-hidden](https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Reference/Attributes/aria-hidden)、[inert](https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Global_attributes/inert)、[visibility](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Properties/visibility)。本次 CDP 单位、frame 和身份实验结论见验证结果文档;支持范围之外的行为不作推断。
本文中的路径为代码职责定位,最终目录命名可适配工程规范;行为契约、问题边界、复杂度预算和验收要求不能因此省略。
@@ -47,7 +47,6 @@
"client": { "client": {
"platform": "web", "platform": "web",
"inject": [ "inject": [
"@deepseek-ai/dsh-client-runtime",
"@deepseek-ai/dsh-client-ui-tool", "@deepseek-ai/dsh-client-ui-tool",
"@deepseek-ai/dsh-client-ui-layout" "@deepseek-ai/dsh-client-ui-layout"
], ],
@@ -69,7 +68,6 @@
"peerDependencies": { "peerDependencies": {
"@deepseek-ai/cordis": "^4.0.1", "@deepseek-ai/cordis": "^4.0.1",
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.6", "@deepseek-ai/dsh-attachment": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6", "@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6", "@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6", "@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
@@ -82,9 +80,6 @@
"@deepseek-ai/dsh-attachment": { "@deepseek-ai/dsh-attachment": {
"optional": true "optional": true
}, },
"@deepseek-ai/dsh-client-runtime": {
"optional": true
},
"@deepseek-ai/dsh-client-ui-attachment": { "@deepseek-ai/dsh-client-ui-attachment": {
"optional": true "optional": true
}, },
@@ -7,6 +7,7 @@
*/ */
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment"; import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
// Development-only types for the injected services; newer hosts no longer ship this package.
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client"; import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope. // Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
import type {} from "@deepseek-ai/dsh-client-ui-layout/client"; import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
@@ -25,7 +25,6 @@ const CLIENT_EXTERNALS: readonly string[] = [
"@deepseek-ai/dsh-client-ui-primitives", "@deepseek-ai/dsh-client-ui-primitives",
"@deepseek-ai/dsh-client-ui-attachment", "@deepseek-ai/dsh-client-ui-attachment",
"@deepseek-ai/dsh-client-schema-form", "@deepseek-ai/dsh-client-schema-form",
"@deepseek-ai/dsh-client-runtime/client",
]; ];
/** Wire/type layers with no shared runtime identity: safe to inline. */ /** Wire/type layers with no shared runtime identity: safe to inline. */