mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(observation): preserve snapshot document provenance and URL metadata
This commit is contained in:
@@ -3353,6 +3353,66 @@ describe("handleSnapshot", () => {
|
||||
};
|
||||
}
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("preserves external-link hints without graph metadata (snapshot URL: %s)", async (hasSnapshotUrl) => {
|
||||
const snapshot = {
|
||||
strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"],
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
...(hasSnapshotUrl ? { documentURL: 4 } : {}),
|
||||
nodes: {
|
||||
backendNodeId: [1, 2],
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [0, 1],
|
||||
attributes: [[], [2, 3]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["link"],
|
||||
},
|
||||
{
|
||||
nodeId: "link",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "link" },
|
||||
name: { type: "computedString", value: "Docs" },
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, snapshot, VP_METRICS);
|
||||
const cdp: CdpRunner = {
|
||||
...deps.cdp,
|
||||
getFrameGraph: async () => {
|
||||
throw new Error("graph unavailable");
|
||||
},
|
||||
};
|
||||
const result = await captureVomObservation(
|
||||
cdp,
|
||||
4,
|
||||
hasSnapshotUrl ? undefined : "https://page.test/start",
|
||||
);
|
||||
expect(result.text).toContain("outside.test");
|
||||
expect(result.text).toContain("Docs");
|
||||
expect(
|
||||
deps.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("exposes the frame-aware observation result for recording", async () => {
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
|
||||
@@ -894,7 +894,7 @@ export async function captureVomObservation(
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal);
|
||||
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
|
||||
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const semanticGraph = buildSemanticGraph({
|
||||
|
||||
@@ -437,7 +437,8 @@ describe("OOPIF capture", () => {
|
||||
const reply = async (_target: unknown, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "DOMSnapshot.captureSnapshot")
|
||||
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
|
||||
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
|
||||
@@ -589,6 +590,7 @@ function siblingCaptureFixture(
|
||||
}
|
||||
});
|
||||
return {
|
||||
snapshot,
|
||||
cdp: { send: send as CdpRunner["send"] },
|
||||
send,
|
||||
peak: () => peak,
|
||||
@@ -600,6 +602,201 @@ function siblingCaptureFixture(
|
||||
};
|
||||
}
|
||||
|
||||
describe("snapshot document provenance", () => {
|
||||
it.each([
|
||||
"new-child",
|
||||
"new-root",
|
||||
"stale-owner",
|
||||
])("retains current snapshot evidence with %s graph data", async (mode) => {
|
||||
const f = siblingCaptureFixture();
|
||||
const cdp: CdpRunner = {
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: mode === "new-root" ? "old-root" : "frame-0",
|
||||
frames: [
|
||||
{ frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } },
|
||||
...(mode === "stale-owner"
|
||||
? [
|
||||
{
|
||||
frameId: "frame-1",
|
||||
parentFrameId: "frame-0",
|
||||
ownerBackendNodeId: 999,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
}),
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
expect(facts.rootFrameId).toBe("frame-0");
|
||||
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({
|
||||
frame: { ownerBackendNodeId: 100 },
|
||||
domNodes: [
|
||||
expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }),
|
||||
expect.anything(),
|
||||
],
|
||||
});
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
|
||||
).not.toBeNull();
|
||||
expect(f.measured()).not.toContain(999);
|
||||
expect(
|
||||
f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
f.send.mock.calls.some(
|
||||
([, method, params]) =>
|
||||
method === "Accessibility.getFullAXTree" &&
|
||||
(params as { frameId?: string }).frameId === "frame-7",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => {
|
||||
const f = siblingCaptureFixture();
|
||||
f.snapshot.strings.push("https://current.test/page", "https://child.test/page");
|
||||
Object.assign(f.snapshot.documents[0], { documentURL: 2 });
|
||||
Object.assign(f.snapshot.documents[1], { documentURL: 3 });
|
||||
const facts = await captureObservationFacts(
|
||||
{
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: "frame-0",
|
||||
frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }],
|
||||
}),
|
||||
},
|
||||
4,
|
||||
undefined,
|
||||
"https://caller.test/",
|
||||
);
|
||||
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe(
|
||||
"https://current.test/page",
|
||||
);
|
||||
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe(
|
||||
"https://child.test/page",
|
||||
);
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts a migrated snapshot claim without merging AX from the old target", async () => {
|
||||
const f = siblingCaptureFixture();
|
||||
const cdp: CdpRunner = {
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: "frame-0",
|
||||
frames: [
|
||||
{ frameId: "frame-0", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "frame-1",
|
||||
parentFrameId: "frame-0",
|
||||
ownerBackendNodeId: 100,
|
||||
target: { tabId: 4, sessionId: "old" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
|
||||
(method === "Accessibility.getFullAXTree"
|
||||
? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] }
|
||||
: {}) as T,
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1");
|
||||
expect(current?.frame.target).toEqual({ tabId: 4 });
|
||||
expect(current?.domNodes.length).toBeGreaterThan(0);
|
||||
expect(current?.axNodes).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
0, 1,
|
||||
])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => {
|
||||
const f = siblingCaptureFixture();
|
||||
f.snapshot.documents.push(f.snapshot.documents[index]);
|
||||
if (index === 0) {
|
||||
await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow(
|
||||
"root document ownership is ambiguous",
|
||||
);
|
||||
} else {
|
||||
const facts = await captureObservationFacts(f.cdp, 4);
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length,
|
||||
).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("retains the caller URL in the root AX-only fallback", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
getFrameGraph: async () => {
|
||||
throw new Error("no graph");
|
||||
},
|
||||
send: async <T>(_tabId: number, method: string): Promise<T> => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot");
|
||||
return (
|
||||
method === "Accessibility.getFullAXTree"
|
||||
? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] }
|
||||
: {}
|
||||
) as T;
|
||||
},
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/");
|
||||
expect(facts.documents[0].frame.url).toBe("https://page.test/");
|
||||
expect(facts.documents[0].axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("retains disconnected document data without inventing top-level geometry", async () => {
|
||||
const f = siblingCaptureFixture();
|
||||
f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] };
|
||||
const facts = await captureObservationFacts(f.cdp, 4);
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0],
|
||||
).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } });
|
||||
expect(f.measured()).toEqual([]);
|
||||
expect(facts.issues).toContainEqual(
|
||||
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("isolates competing target claims independent of batch order %s", async (reverse) => {
|
||||
const f = siblingCaptureFixture();
|
||||
const remote = {
|
||||
frameId: "frame-1",
|
||||
parentFrameId: "frame-0",
|
||||
ownerBackendNodeId: 100,
|
||||
target: { tabId: 4, sessionId: "remote" },
|
||||
};
|
||||
const main = { frameId: "frame-0", target: { tabId: 4 } };
|
||||
const cdp: CdpRunner = {
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: "frame-0",
|
||||
frames: reverse ? [remote, main] : [main, remote],
|
||||
}),
|
||||
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
|
||||
(method === "DOMSnapshot.captureSnapshot"
|
||||
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
|
||||
: method === "Page.getLayoutMetrics"
|
||||
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
|
||||
: {}) as T,
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
|
||||
).not.toBeNull();
|
||||
expect(facts.issues).toContainEqual(
|
||||
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sibling frame measurement scheduling", () => {
|
||||
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
|
||||
const pending = new Map<number, () => void>();
|
||||
|
||||
@@ -159,10 +159,12 @@ describe("buildFrameDocuments", () => {
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
frame: {
|
||||
frameId,
|
||||
target: { tabId: 4 },
|
||||
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
|
||||
},
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
|
||||
import { REQUESTED_STYLES, type SnapshotReply, snapshotFrameId } from "./snapshot";
|
||||
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
|
||||
|
||||
interface TargetBatch<T extends FrameOwnedAxNode> {
|
||||
target: CdpTarget;
|
||||
@@ -31,6 +31,8 @@ interface TargetBatch<T extends FrameOwnedAxNode> {
|
||||
documents: NormalizedDocument[];
|
||||
ax: FrameAxBatch<T>[];
|
||||
fallbackExcluded: Set<number>;
|
||||
snapshotFrameIds?: Set<string>;
|
||||
rootFrameId?: string;
|
||||
}
|
||||
|
||||
/** A fixed worker pool scoped to one capture; each worker issues at most one
|
||||
@@ -62,36 +64,11 @@ async function collectTargets<T>(
|
||||
if (failed) throw failure;
|
||||
}
|
||||
|
||||
/** When topology discovery is unavailable, snapshot documents still carry
|
||||
* explicit target/frame provenance. Only explicit contentDocumentIndex edges
|
||||
* establish ancestry; a foreign frame is never renamed to fit a request. */
|
||||
function snapshotFrames(snapshot: SnapshotReply, target: CdpTarget): CdpFrame[] {
|
||||
const raw = snapshot.documents ?? [];
|
||||
const strings = snapshot.strings ?? [];
|
||||
const frames = raw.map((doc) => {
|
||||
const frameId = snapshotFrameId(doc, strings);
|
||||
return frameId ? ({ frameId, target } as CdpFrame) : undefined;
|
||||
});
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const edges = raw[i].nodes?.contentDocumentIndex;
|
||||
const parent = frames[i];
|
||||
if (!parent || !edges) continue;
|
||||
for (let e = 0; e < edges.index.length; e++) {
|
||||
const child = frames[edges.value[e]];
|
||||
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
||||
if (child && owner !== undefined && child !== parent) {
|
||||
child.parentFrameId = parent.frameId;
|
||||
child.ownerBackendNodeId = owner;
|
||||
}
|
||||
}
|
||||
}
|
||||
return frames.filter((frame): frame is CdpFrame => !!frame);
|
||||
}
|
||||
|
||||
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
pageUrl?: string,
|
||||
): Promise<ObservationFacts<T>> {
|
||||
const startedAt = Date.now();
|
||||
throwCaptureAborted(signal);
|
||||
@@ -104,6 +81,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
throwCaptureAborted(signal);
|
||||
const geometry = new GeometryContext(cdp, tabId, graph, signal);
|
||||
const issues: CaptureIssue[] = [];
|
||||
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
|
||||
const groups = new Map<string, TargetBatch<T>>();
|
||||
for (const frame of graph?.frames ?? []) {
|
||||
const key = cdpTargetKey(frame.target);
|
||||
@@ -118,9 +96,13 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
};
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.frames.push(frame);
|
||||
group.frames.push(
|
||||
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
|
||||
? { ...frame, url: pageUrl }
|
||||
: frame,
|
||||
);
|
||||
}
|
||||
if (!groups.size)
|
||||
if (!groups.has(cdpTargetKey({ tabId })))
|
||||
groups.set(cdpTargetKey({ tabId }), {
|
||||
target: { tabId },
|
||||
frames: [],
|
||||
@@ -146,14 +128,22 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwCaptureAborted(signal);
|
||||
if (!graph) batch.frames = snapshotFrames(snapshot, batch.target);
|
||||
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
|
||||
batch.snapshotFrameIds = observed.ids;
|
||||
batch.rootFrameId = observed.rootFrameId;
|
||||
// Keep graph-only frames for AX fallback, never in preference to observed membership.
|
||||
batch.frames = [
|
||||
...observed.frames,
|
||||
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
|
||||
];
|
||||
batch.documents = await normalizeSnapshot(
|
||||
snapshot,
|
||||
batch.target,
|
||||
batch.frames,
|
||||
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
|
||||
geometry,
|
||||
issues,
|
||||
signal,
|
||||
observed.rootFrameId,
|
||||
);
|
||||
const formsAvailable = await enrichFormControlStates(
|
||||
scoped,
|
||||
@@ -174,7 +164,13 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
|
||||
}
|
||||
if (!batch.frames.length && !graph)
|
||||
batch.frames = [{ frameId: "root", target: batch.target }];
|
||||
batch.frames = [
|
||||
{
|
||||
frameId: "root",
|
||||
target: batch.target,
|
||||
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
|
||||
},
|
||||
];
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "Accessibility.enable", {});
|
||||
@@ -206,12 +202,68 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
signal,
|
||||
);
|
||||
|
||||
const frames = batches.flatMap((batch) => batch.frames);
|
||||
const rootFrameId = graph?.rootFrameId ?? frames[0]?.frameId ?? "root";
|
||||
const decoded = new Map(
|
||||
batches.flatMap((batch) => batch.documents).map((doc) => [doc.frame.frameId, doc]),
|
||||
// A snapshot claim outranks an old graph hint. Conflicting actual claims
|
||||
// cannot be resolved by completion order or by overwriting a frameId map.
|
||||
const claims = new Map<string, string>();
|
||||
const invalid = new Set<string>();
|
||||
for (const batch of batches) {
|
||||
const targetKey = cdpTargetKey(batch.target);
|
||||
for (const id of batch.snapshotFrameIds ?? []) {
|
||||
const previous = claims.get(id);
|
||||
if (previous !== undefined && previous !== targetKey) invalid.add(id);
|
||||
claims.set(id, targetKey);
|
||||
}
|
||||
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
|
||||
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
|
||||
}
|
||||
const currentFrames = batches.flatMap((batch) =>
|
||||
batch.frames.filter(
|
||||
(frame) =>
|
||||
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
|
||||
),
|
||||
);
|
||||
const ax = batches.flatMap((batch) => batch.ax);
|
||||
const children = new Map<string, string[]>();
|
||||
for (const frame of currentFrames) {
|
||||
if (!frame.parentFrameId) continue;
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame.frameId);
|
||||
else children.set(frame.parentFrameId, [frame.frameId]);
|
||||
}
|
||||
const blocked = [...invalid];
|
||||
for (let i = 0; i < blocked.length; i++) {
|
||||
for (const child of children.get(blocked[i]) ?? []) {
|
||||
if (!invalid.has(child)) {
|
||||
invalid.add(child);
|
||||
blocked.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const frame of currentFrames) {
|
||||
if (invalid.has(frame.frameId))
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
}
|
||||
const rootBatch = batches.find((batch) => !batch.target.sessionId);
|
||||
const rootFrameId =
|
||||
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
|
||||
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
|
||||
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const belongs = (frame: CdpFrame) => {
|
||||
const current = frameById.get(frame.frameId);
|
||||
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
|
||||
};
|
||||
const decoded = new Map(
|
||||
batches
|
||||
.flatMap((batch) => batch.documents)
|
||||
.filter((doc) => belongs(doc.frame))
|
||||
.map((doc) => [doc.frame.frameId, doc]),
|
||||
);
|
||||
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId, frames },
|
||||
ax,
|
||||
|
||||
@@ -32,8 +32,6 @@ export interface FrameDomInput {
|
||||
nodes: CapturedNode[];
|
||||
rootFrameId?: string;
|
||||
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
|
||||
frameParentIds?: ReadonlyMap<string, string>;
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
@@ -58,23 +56,13 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: FrameDomInput,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
return [...frames.values()];
|
||||
}
|
||||
|
||||
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
@@ -85,7 +73,7 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
unresolved?: (frame: CdpFrame) => void,
|
||||
): Promise<FrameDocument<T>[]> {
|
||||
let work = 0;
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frames = frameList(graph, batches);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import {
|
||||
type FrameProjectionState,
|
||||
@@ -39,6 +39,7 @@ export async function normalizeSnapshot(
|
||||
geometry: GeometryContext,
|
||||
issues: CaptureIssue[],
|
||||
signal?: AbortSignal,
|
||||
rootFrameId?: string,
|
||||
): Promise<NormalizedDocument[]> {
|
||||
const strings = snapshot.strings ?? [];
|
||||
const raw = snapshot.documents ?? [];
|
||||
@@ -82,28 +83,47 @@ export async function normalizeSnapshot(
|
||||
const projections = new Map<string, FrameProjectionState | null>();
|
||||
const result: NormalizedDocument[] = [];
|
||||
let targetProjection: GeometryProjection | null = null;
|
||||
const targetRoot = pending[0];
|
||||
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
|
||||
if (target.sessionId && targetRoot) {
|
||||
try {
|
||||
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
||||
const known = await geometry.frame(targetRoot.frameId);
|
||||
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
|
||||
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
}
|
||||
for (let cursor = 0; cursor < pending.length; cursor++) {
|
||||
const visited = new Set<string>();
|
||||
const remaining = frames[Symbol.iterator]();
|
||||
for (let cursor = 0; visited.size < frames.length; cursor++) {
|
||||
if (cursor >= pending.length) {
|
||||
let next = remaining.next();
|
||||
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
|
||||
if (next.done) break;
|
||||
pending.push(next.value);
|
||||
}
|
||||
if (cursor % 256 === 0) await captureCheckpoint(signal);
|
||||
const frame = pending[cursor];
|
||||
if (visited.has(frame.frameId)) continue;
|
||||
visited.add(frame.frameId);
|
||||
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
|
||||
const doc = sources.get(frame.frameId);
|
||||
if (!doc) continue;
|
||||
const source = { target, frameId: frame.frameId };
|
||||
const parent = frame.parentFrameId ? frameById.get(frame.parentFrameId) : undefined;
|
||||
const state: FrameProjectionState | null = !parent
|
||||
? {
|
||||
status: "available",
|
||||
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
|
||||
}
|
||||
: (projections.get(frame.frameId) ?? null);
|
||||
const state: FrameProjectionState | null =
|
||||
frame.frameId === rootFrameId
|
||||
? {
|
||||
status: "available",
|
||||
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
|
||||
}
|
||||
: (projections.get(frame.frameId) ?? null);
|
||||
if (!state && frame.frameId !== rootFrameId)
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
projections.set(frame.frameId, state);
|
||||
const projection = state?.status === "available" ? state.projection : null;
|
||||
if (!projection || (target.sessionId && !targetProjection))
|
||||
@@ -173,8 +193,8 @@ export async function normalizeSnapshot(
|
||||
if (i % 256 === 0) await captureCheckpoint(signal);
|
||||
const node = decoded.nodes[i];
|
||||
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
|
||||
x: doc.scrollOffsetX ?? (parent ? 0 : scrollX),
|
||||
y: doc.scrollOffsetY ?? (parent ? 0 : scrollY),
|
||||
x: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
|
||||
y: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
|
||||
});
|
||||
const local = input?.rect;
|
||||
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
|
||||
@@ -209,9 +229,5 @@ export async function normalizeSnapshot(
|
||||
domNodes,
|
||||
});
|
||||
}
|
||||
for (const frameId of sources.keys()) {
|
||||
if (!projections.has(frameId))
|
||||
issues.push({ target, frameId, stage: "ownership", reason: "frame-ownership-unresolved" });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode } from "./facts";
|
||||
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
|
||||
export const REQUESTED_STYLES = [
|
||||
@@ -22,6 +23,7 @@ interface RareBooleanData {
|
||||
|
||||
export interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
documentURL?: number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
@@ -211,3 +213,68 @@ export async function decodeDocument(
|
||||
}
|
||||
return { nodes };
|
||||
}
|
||||
|
||||
/** Snapshot membership and same-target edges are current response evidence.
|
||||
* The earlier graph only supplies compatible metadata and cross-target edges. */
|
||||
export function describeSnapshotFrames(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
hintById: ReadonlyMap<string, CdpFrame>,
|
||||
pageUrl?: string,
|
||||
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
|
||||
const raw = snapshot.documents ?? [];
|
||||
const strings = snapshot.strings ?? [];
|
||||
const ids = new Set<string>();
|
||||
const invalid = new Set<string>();
|
||||
const frames = raw.map((doc) => {
|
||||
const frameId = snapshotFrameId(doc, strings);
|
||||
if (!frameId) return undefined;
|
||||
if (ids.has(frameId)) invalid.add(frameId);
|
||||
ids.add(frameId);
|
||||
const hint = hintById.get(frameId);
|
||||
const compatible =
|
||||
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
|
||||
const url =
|
||||
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
|
||||
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
|
||||
});
|
||||
const root = frames[0];
|
||||
if (root) {
|
||||
const hint = hintById.get(root.frameId);
|
||||
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
|
||||
if (
|
||||
target.sessionId &&
|
||||
hint &&
|
||||
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
|
||||
parent &&
|
||||
cdpTargetKey(parent.target) !== cdpTargetKey(target)
|
||||
) {
|
||||
root.parentFrameId = parent.frameId;
|
||||
root.ownerBackendNodeId = hint.ownerBackendNodeId;
|
||||
}
|
||||
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
|
||||
}
|
||||
const owned = new Set<string>();
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const edges = raw[i].nodes?.contentDocumentIndex;
|
||||
const parent = frames[i];
|
||||
if (!parent || !edges) continue;
|
||||
for (let e = 0; e < edges.index.length; e++) {
|
||||
const child = frames[edges.value[e]];
|
||||
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
||||
if (!child || owner === undefined) continue;
|
||||
if (child === root || child === parent || owned.has(child.frameId)) {
|
||||
invalid.add(child.frameId);
|
||||
continue;
|
||||
}
|
||||
owned.add(child.frameId);
|
||||
child.parentFrameId = parent.frameId;
|
||||
child.ownerBackendNodeId = owner;
|
||||
}
|
||||
}
|
||||
return {
|
||||
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
|
||||
ids,
|
||||
rootFrameId: root?.frameId,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user