mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(observation): preserve snapshot document provenance and URL metadata
This commit is contained in:
@@ -3323,6 +3323,66 @@ describe("handleSnapshot", () => {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
])("preserves external-link hints without graph metadata (snapshot URL: %s)", async (hasSnapshotUrl) => {
|
||||||
|
const snapshot = {
|
||||||
|
strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"],
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
frameId: "main",
|
||||||
|
...(hasSnapshotUrl ? { documentURL: 4 } : {}),
|
||||||
|
nodes: {
|
||||||
|
backendNodeId: [1, 2],
|
||||||
|
parentIndex: [-1, 0],
|
||||||
|
nodeName: [0, 1],
|
||||||
|
attributes: [[], [2, 3]],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[10, 20, 100, 40],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
backendDOMNodeId: 1,
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
childIds: ["link"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "link",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 2,
|
||||||
|
role: { type: "role", value: "link" },
|
||||||
|
name: { type: "computedString", value: "Docs" },
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const deps = makeOverlayDeps(ax, snapshot, VP_METRICS);
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
...deps.cdp,
|
||||||
|
getFrameGraph: async () => {
|
||||||
|
throw new Error("graph unavailable");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const result = await captureVomObservation(
|
||||||
|
cdp,
|
||||||
|
4,
|
||||||
|
hasSnapshotUrl ? undefined : "https://page.test/start",
|
||||||
|
);
|
||||||
|
expect(result.text).toContain("outside.test");
|
||||||
|
expect(result.text).toContain("Docs");
|
||||||
|
expect(
|
||||||
|
deps.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"),
|
||||||
|
).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("exposes the frame-aware observation result for recording", async () => {
|
it("exposes the frame-aware observation result for recording", async () => {
|
||||||
const ax: CdpAxNode[] = [
|
const ax: CdpAxNode[] = [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -894,7 +894,7 @@ export async function captureVomObservation(
|
|||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal);
|
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
|
||||||
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
|
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
|
||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
const semanticGraph = buildSemanticGraph({
|
const semanticGraph = buildSemanticGraph({
|
||||||
|
|||||||
@@ -437,7 +437,8 @@ describe("OOPIF capture", () => {
|
|||||||
const reply = async (_target: unknown, method: string) => {
|
const reply = async (_target: unknown, method: string) => {
|
||||||
if (method === "Page.getLayoutMetrics")
|
if (method === "Page.getLayoutMetrics")
|
||||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
|
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
|
||||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
if (method === "DOMSnapshot.captureSnapshot")
|
||||||
|
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
|
||||||
if (method === "DOM.getBoxModel")
|
if (method === "DOM.getBoxModel")
|
||||||
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
|
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
|
||||||
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
|
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
|
||||||
@@ -589,6 +590,7 @@ function siblingCaptureFixture(
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
|
snapshot,
|
||||||
cdp: { send: send as CdpRunner["send"] },
|
cdp: { send: send as CdpRunner["send"] },
|
||||||
send,
|
send,
|
||||||
peak: () => peak,
|
peak: () => peak,
|
||||||
@@ -600,6 +602,201 @@ function siblingCaptureFixture(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
describe("snapshot document provenance", () => {
|
||||||
|
it.each([
|
||||||
|
"new-child",
|
||||||
|
"new-root",
|
||||||
|
"stale-owner",
|
||||||
|
])("retains current snapshot evidence with %s graph data", async (mode) => {
|
||||||
|
const f = siblingCaptureFixture();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
...f.cdp,
|
||||||
|
getFrameGraph: async () => ({
|
||||||
|
rootFrameId: mode === "new-root" ? "old-root" : "frame-0",
|
||||||
|
frames: [
|
||||||
|
{ frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } },
|
||||||
|
...(mode === "stale-owner"
|
||||||
|
? [
|
||||||
|
{
|
||||||
|
frameId: "frame-1",
|
||||||
|
parentFrameId: "frame-0",
|
||||||
|
ownerBackendNodeId: 999,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
]
|
||||||
|
: []),
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const facts = await captureObservationFacts(cdp, 4);
|
||||||
|
expect(facts.rootFrameId).toBe("frame-0");
|
||||||
|
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({
|
||||||
|
frame: { ownerBackendNodeId: 100 },
|
||||||
|
domNodes: [
|
||||||
|
expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }),
|
||||||
|
expect.anything(),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
|
||||||
|
).not.toBeNull();
|
||||||
|
expect(f.measured()).not.toContain(999);
|
||||||
|
expect(
|
||||||
|
f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"),
|
||||||
|
).toHaveLength(1);
|
||||||
|
expect(
|
||||||
|
f.send.mock.calls.some(
|
||||||
|
([, method, params]) =>
|
||||||
|
method === "Accessibility.getFullAXTree" &&
|
||||||
|
(params as { frameId?: string }).frameId === "frame-7",
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => {
|
||||||
|
const f = siblingCaptureFixture();
|
||||||
|
f.snapshot.strings.push("https://current.test/page", "https://child.test/page");
|
||||||
|
Object.assign(f.snapshot.documents[0], { documentURL: 2 });
|
||||||
|
Object.assign(f.snapshot.documents[1], { documentURL: 3 });
|
||||||
|
const facts = await captureObservationFacts(
|
||||||
|
{
|
||||||
|
...f.cdp,
|
||||||
|
getFrameGraph: async () => ({
|
||||||
|
rootFrameId: "frame-0",
|
||||||
|
frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
4,
|
||||||
|
undefined,
|
||||||
|
"https://caller.test/",
|
||||||
|
);
|
||||||
|
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe(
|
||||||
|
"https://current.test/page",
|
||||||
|
);
|
||||||
|
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe(
|
||||||
|
"https://child.test/page",
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url,
|
||||||
|
).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts a migrated snapshot claim without merging AX from the old target", async () => {
|
||||||
|
const f = siblingCaptureFixture();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
...f.cdp,
|
||||||
|
getFrameGraph: async () => ({
|
||||||
|
rootFrameId: "frame-0",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "frame-0", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "frame-1",
|
||||||
|
parentFrameId: "frame-0",
|
||||||
|
ownerBackendNodeId: 100,
|
||||||
|
target: { tabId: 4, sessionId: "old" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
|
||||||
|
(method === "Accessibility.getFullAXTree"
|
||||||
|
? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] }
|
||||||
|
: {}) as T,
|
||||||
|
};
|
||||||
|
const facts = await captureObservationFacts(cdp, 4);
|
||||||
|
const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1");
|
||||||
|
expect(current?.frame.target).toEqual({ tabId: 4 });
|
||||||
|
expect(current?.domNodes.length).toBeGreaterThan(0);
|
||||||
|
expect(current?.axNodes).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
0, 1,
|
||||||
|
])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => {
|
||||||
|
const f = siblingCaptureFixture();
|
||||||
|
f.snapshot.documents.push(f.snapshot.documents[index]);
|
||||||
|
if (index === 0) {
|
||||||
|
await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow(
|
||||||
|
"root document ownership is ambiguous",
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const facts = await captureObservationFacts(f.cdp, 4);
|
||||||
|
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
|
||||||
|
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
|
||||||
|
expect(
|
||||||
|
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length,
|
||||||
|
).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retains the caller URL in the root AX-only fallback", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
getFrameGraph: async () => {
|
||||||
|
throw new Error("no graph");
|
||||||
|
},
|
||||||
|
send: async <T>(_tabId: number, method: string): Promise<T> => {
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot");
|
||||||
|
return (
|
||||||
|
method === "Accessibility.getFullAXTree"
|
||||||
|
? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] }
|
||||||
|
: {}
|
||||||
|
) as T;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/");
|
||||||
|
expect(facts.documents[0].frame.url).toBe("https://page.test/");
|
||||||
|
expect(facts.documents[0].axNodes).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retains disconnected document data without inventing top-level geometry", async () => {
|
||||||
|
const f = siblingCaptureFixture();
|
||||||
|
f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] };
|
||||||
|
const facts = await captureObservationFacts(f.cdp, 4);
|
||||||
|
expect(
|
||||||
|
facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0],
|
||||||
|
).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } });
|
||||||
|
expect(f.measured()).toEqual([]);
|
||||||
|
expect(facts.issues).toContainEqual(
|
||||||
|
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
])("isolates competing target claims independent of batch order %s", async (reverse) => {
|
||||||
|
const f = siblingCaptureFixture();
|
||||||
|
const remote = {
|
||||||
|
frameId: "frame-1",
|
||||||
|
parentFrameId: "frame-0",
|
||||||
|
ownerBackendNodeId: 100,
|
||||||
|
target: { tabId: 4, sessionId: "remote" },
|
||||||
|
};
|
||||||
|
const main = { frameId: "frame-0", target: { tabId: 4 } };
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
...f.cdp,
|
||||||
|
getFrameGraph: async () => ({
|
||||||
|
rootFrameId: "frame-0",
|
||||||
|
frames: reverse ? [remote, main] : [main, remote],
|
||||||
|
}),
|
||||||
|
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
|
||||||
|
(method === "DOMSnapshot.captureSnapshot"
|
||||||
|
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
|
||||||
|
: method === "Page.getLayoutMetrics"
|
||||||
|
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
|
||||||
|
: {}) as T,
|
||||||
|
};
|
||||||
|
const facts = await captureObservationFacts(cdp, 4);
|
||||||
|
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
|
||||||
|
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
|
||||||
|
expect(
|
||||||
|
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
|
||||||
|
).not.toBeNull();
|
||||||
|
expect(facts.issues).toContainEqual(
|
||||||
|
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("sibling frame measurement scheduling", () => {
|
describe("sibling frame measurement scheduling", () => {
|
||||||
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
|
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
|
||||||
const pending = new Map<number, () => void>();
|
const pending = new Map<number, () => void>();
|
||||||
|
|||||||
@@ -159,10 +159,12 @@ describe("buildFrameDocuments", () => {
|
|||||||
["child", [domNode("child", 9)]],
|
["child", [domNode("child", 9)]],
|
||||||
]);
|
]);
|
||||||
const capture = captured(frameNodes);
|
const capture = captured(frameNodes);
|
||||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
|
||||||
capture.frameParentIds = new Map([["child", "left"]]);
|
|
||||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||||
frame: { frameId, target: { tabId: 4 } },
|
frame: {
|
||||||
|
frameId,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
|
||||||
|
},
|
||||||
nodes: [],
|
nodes: [],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import {
|
|||||||
type FrameOwnedAxNode,
|
type FrameOwnedAxNode,
|
||||||
} from "./frame-document";
|
} from "./frame-document";
|
||||||
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
|
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
|
||||||
import { REQUESTED_STYLES, type SnapshotReply, snapshotFrameId } from "./snapshot";
|
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
|
||||||
|
|
||||||
interface TargetBatch<T extends FrameOwnedAxNode> {
|
interface TargetBatch<T extends FrameOwnedAxNode> {
|
||||||
target: CdpTarget;
|
target: CdpTarget;
|
||||||
@@ -31,6 +31,8 @@ interface TargetBatch<T extends FrameOwnedAxNode> {
|
|||||||
documents: NormalizedDocument[];
|
documents: NormalizedDocument[];
|
||||||
ax: FrameAxBatch<T>[];
|
ax: FrameAxBatch<T>[];
|
||||||
fallbackExcluded: Set<number>;
|
fallbackExcluded: Set<number>;
|
||||||
|
snapshotFrameIds?: Set<string>;
|
||||||
|
rootFrameId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A fixed worker pool scoped to one capture; each worker issues at most one
|
/** A fixed worker pool scoped to one capture; each worker issues at most one
|
||||||
@@ -62,36 +64,11 @@ async function collectTargets<T>(
|
|||||||
if (failed) throw failure;
|
if (failed) throw failure;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** When topology discovery is unavailable, snapshot documents still carry
|
|
||||||
* explicit target/frame provenance. Only explicit contentDocumentIndex edges
|
|
||||||
* establish ancestry; a foreign frame is never renamed to fit a request. */
|
|
||||||
function snapshotFrames(snapshot: SnapshotReply, target: CdpTarget): CdpFrame[] {
|
|
||||||
const raw = snapshot.documents ?? [];
|
|
||||||
const strings = snapshot.strings ?? [];
|
|
||||||
const frames = raw.map((doc) => {
|
|
||||||
const frameId = snapshotFrameId(doc, strings);
|
|
||||||
return frameId ? ({ frameId, target } as CdpFrame) : undefined;
|
|
||||||
});
|
|
||||||
for (let i = 0; i < raw.length; i++) {
|
|
||||||
const edges = raw[i].nodes?.contentDocumentIndex;
|
|
||||||
const parent = frames[i];
|
|
||||||
if (!parent || !edges) continue;
|
|
||||||
for (let e = 0; e < edges.index.length; e++) {
|
|
||||||
const child = frames[edges.value[e]];
|
|
||||||
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
|
||||||
if (child && owner !== undefined && child !== parent) {
|
|
||||||
child.parentFrameId = parent.frameId;
|
|
||||||
child.ownerBackendNodeId = owner;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return frames.filter((frame): frame is CdpFrame => !!frame);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
|
pageUrl?: string,
|
||||||
): Promise<ObservationFacts<T>> {
|
): Promise<ObservationFacts<T>> {
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
throwCaptureAborted(signal);
|
throwCaptureAborted(signal);
|
||||||
@@ -104,6 +81,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
|||||||
throwCaptureAborted(signal);
|
throwCaptureAborted(signal);
|
||||||
const geometry = new GeometryContext(cdp, tabId, graph, signal);
|
const geometry = new GeometryContext(cdp, tabId, graph, signal);
|
||||||
const issues: CaptureIssue[] = [];
|
const issues: CaptureIssue[] = [];
|
||||||
|
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
|
||||||
const groups = new Map<string, TargetBatch<T>>();
|
const groups = new Map<string, TargetBatch<T>>();
|
||||||
for (const frame of graph?.frames ?? []) {
|
for (const frame of graph?.frames ?? []) {
|
||||||
const key = cdpTargetKey(frame.target);
|
const key = cdpTargetKey(frame.target);
|
||||||
@@ -118,9 +96,13 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
|||||||
};
|
};
|
||||||
groups.set(key, group);
|
groups.set(key, group);
|
||||||
}
|
}
|
||||||
group.frames.push(frame);
|
group.frames.push(
|
||||||
|
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
|
||||||
|
? { ...frame, url: pageUrl }
|
||||||
|
: frame,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if (!groups.size)
|
if (!groups.has(cdpTargetKey({ tabId })))
|
||||||
groups.set(cdpTargetKey({ tabId }), {
|
groups.set(cdpTargetKey({ tabId }), {
|
||||||
target: { tabId },
|
target: { tabId },
|
||||||
frames: [],
|
frames: [],
|
||||||
@@ -146,14 +128,22 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
|||||||
includeDOMRects: true,
|
includeDOMRects: true,
|
||||||
});
|
});
|
||||||
throwCaptureAborted(signal);
|
throwCaptureAborted(signal);
|
||||||
if (!graph) batch.frames = snapshotFrames(snapshot, batch.target);
|
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
|
||||||
|
batch.snapshotFrameIds = observed.ids;
|
||||||
|
batch.rootFrameId = observed.rootFrameId;
|
||||||
|
// Keep graph-only frames for AX fallback, never in preference to observed membership.
|
||||||
|
batch.frames = [
|
||||||
|
...observed.frames,
|
||||||
|
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
|
||||||
|
];
|
||||||
batch.documents = await normalizeSnapshot(
|
batch.documents = await normalizeSnapshot(
|
||||||
snapshot,
|
snapshot,
|
||||||
batch.target,
|
batch.target,
|
||||||
batch.frames,
|
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
|
||||||
geometry,
|
geometry,
|
||||||
issues,
|
issues,
|
||||||
signal,
|
signal,
|
||||||
|
observed.rootFrameId,
|
||||||
);
|
);
|
||||||
const formsAvailable = await enrichFormControlStates(
|
const formsAvailable = await enrichFormControlStates(
|
||||||
scoped,
|
scoped,
|
||||||
@@ -174,7 +164,13 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
|||||||
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
|
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
|
||||||
}
|
}
|
||||||
if (!batch.frames.length && !graph)
|
if (!batch.frames.length && !graph)
|
||||||
batch.frames = [{ frameId: "root", target: batch.target }];
|
batch.frames = [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
target: batch.target,
|
||||||
|
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
|
||||||
|
},
|
||||||
|
];
|
||||||
try {
|
try {
|
||||||
throwCaptureAborted(signal);
|
throwCaptureAborted(signal);
|
||||||
await scoped.send(tabId, "Accessibility.enable", {});
|
await scoped.send(tabId, "Accessibility.enable", {});
|
||||||
@@ -206,12 +202,68 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
|||||||
signal,
|
signal,
|
||||||
);
|
);
|
||||||
|
|
||||||
const frames = batches.flatMap((batch) => batch.frames);
|
// A snapshot claim outranks an old graph hint. Conflicting actual claims
|
||||||
const rootFrameId = graph?.rootFrameId ?? frames[0]?.frameId ?? "root";
|
// cannot be resolved by completion order or by overwriting a frameId map.
|
||||||
const decoded = new Map(
|
const claims = new Map<string, string>();
|
||||||
batches.flatMap((batch) => batch.documents).map((doc) => [doc.frame.frameId, doc]),
|
const invalid = new Set<string>();
|
||||||
|
for (const batch of batches) {
|
||||||
|
const targetKey = cdpTargetKey(batch.target);
|
||||||
|
for (const id of batch.snapshotFrameIds ?? []) {
|
||||||
|
const previous = claims.get(id);
|
||||||
|
if (previous !== undefined && previous !== targetKey) invalid.add(id);
|
||||||
|
claims.set(id, targetKey);
|
||||||
|
}
|
||||||
|
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
|
||||||
|
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
|
||||||
|
}
|
||||||
|
const currentFrames = batches.flatMap((batch) =>
|
||||||
|
batch.frames.filter(
|
||||||
|
(frame) =>
|
||||||
|
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
const ax = batches.flatMap((batch) => batch.ax);
|
const children = new Map<string, string[]>();
|
||||||
|
for (const frame of currentFrames) {
|
||||||
|
if (!frame.parentFrameId) continue;
|
||||||
|
const siblings = children.get(frame.parentFrameId);
|
||||||
|
if (siblings) siblings.push(frame.frameId);
|
||||||
|
else children.set(frame.parentFrameId, [frame.frameId]);
|
||||||
|
}
|
||||||
|
const blocked = [...invalid];
|
||||||
|
for (let i = 0; i < blocked.length; i++) {
|
||||||
|
for (const child of children.get(blocked[i]) ?? []) {
|
||||||
|
if (!invalid.has(child)) {
|
||||||
|
invalid.add(child);
|
||||||
|
blocked.push(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const frame of currentFrames) {
|
||||||
|
if (invalid.has(frame.frameId))
|
||||||
|
issues.push({
|
||||||
|
target: frame.target,
|
||||||
|
frameId: frame.frameId,
|
||||||
|
stage: "ownership",
|
||||||
|
reason: "frame-ownership-unresolved",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const rootBatch = batches.find((batch) => !batch.target.sessionId);
|
||||||
|
const rootFrameId =
|
||||||
|
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
|
||||||
|
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
|
||||||
|
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
|
||||||
|
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
const belongs = (frame: CdpFrame) => {
|
||||||
|
const current = frameById.get(frame.frameId);
|
||||||
|
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
|
||||||
|
};
|
||||||
|
const decoded = new Map(
|
||||||
|
batches
|
||||||
|
.flatMap((batch) => batch.documents)
|
||||||
|
.filter((doc) => belongs(doc.frame))
|
||||||
|
.map((doc) => [doc.frame.frameId, doc]),
|
||||||
|
);
|
||||||
|
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
|
||||||
const documents = await buildFrameDocuments(
|
const documents = await buildFrameDocuments(
|
||||||
{ rootFrameId, frames },
|
{ rootFrameId, frames },
|
||||||
ax,
|
ax,
|
||||||
|
|||||||
@@ -32,8 +32,6 @@ export interface FrameDomInput {
|
|||||||
nodes: CapturedNode[];
|
nodes: CapturedNode[];
|
||||||
rootFrameId?: string;
|
rootFrameId?: string;
|
||||||
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
||||||
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
|
|
||||||
frameParentIds?: ReadonlyMap<string, string>;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
interface Ownership {
|
interface Ownership {
|
||||||
@@ -58,23 +56,13 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
|||||||
function frameList<T extends FrameOwnedAxNode>(
|
function frameList<T extends FrameOwnedAxNode>(
|
||||||
graph: CdpFrameGraph | null,
|
graph: CdpFrameGraph | null,
|
||||||
batches: FrameAxBatch<T>[],
|
batches: FrameAxBatch<T>[],
|
||||||
captured: FrameDomInput,
|
|
||||||
): CdpFrame[] {
|
): CdpFrame[] {
|
||||||
const frames = new Map<string, CdpFrame>();
|
const frames = new Map<string, CdpFrame>();
|
||||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||||
for (const batch of batches) {
|
for (const batch of batches) {
|
||||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||||
}
|
}
|
||||||
return [...frames.values()].map((frame) => {
|
return [...frames.values()];
|
||||||
const ownerBackendNodeId =
|
|
||||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
|
||||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
|
||||||
return {
|
|
||||||
...frame,
|
|
||||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
|
||||||
...(parentFrameId ? { parentFrameId } : {}),
|
|
||||||
};
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||||
@@ -85,7 +73,7 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
|||||||
unresolved?: (frame: CdpFrame) => void,
|
unresolved?: (frame: CdpFrame) => void,
|
||||||
): Promise<FrameDocument<T>[]> {
|
): Promise<FrameDocument<T>[]> {
|
||||||
let work = 0;
|
let work = 0;
|
||||||
const frames = frameList(graph, batches, captured);
|
const frames = frameList(graph, batches);
|
||||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import type { Viewport } from "@browser-skill/vom";
|
import type { Viewport } from "@browser-skill/vom";
|
||||||
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
|
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
import {
|
import {
|
||||||
type FrameProjectionState,
|
type FrameProjectionState,
|
||||||
@@ -39,6 +39,7 @@ export async function normalizeSnapshot(
|
|||||||
geometry: GeometryContext,
|
geometry: GeometryContext,
|
||||||
issues: CaptureIssue[],
|
issues: CaptureIssue[],
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
|
rootFrameId?: string,
|
||||||
): Promise<NormalizedDocument[]> {
|
): Promise<NormalizedDocument[]> {
|
||||||
const strings = snapshot.strings ?? [];
|
const strings = snapshot.strings ?? [];
|
||||||
const raw = snapshot.documents ?? [];
|
const raw = snapshot.documents ?? [];
|
||||||
@@ -82,28 +83,47 @@ export async function normalizeSnapshot(
|
|||||||
const projections = new Map<string, FrameProjectionState | null>();
|
const projections = new Map<string, FrameProjectionState | null>();
|
||||||
const result: NormalizedDocument[] = [];
|
const result: NormalizedDocument[] = [];
|
||||||
let targetProjection: GeometryProjection | null = null;
|
let targetProjection: GeometryProjection | null = null;
|
||||||
const targetRoot = pending[0];
|
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
|
||||||
if (target.sessionId && targetRoot) {
|
if (target.sessionId && targetRoot) {
|
||||||
try {
|
try {
|
||||||
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
const known = await geometry.frame(targetRoot.frameId);
|
||||||
|
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
|
||||||
|
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isCaptureAbort(error)) throw error;
|
if (isCaptureAbort(error)) throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (let cursor = 0; cursor < pending.length; cursor++) {
|
const visited = new Set<string>();
|
||||||
|
const remaining = frames[Symbol.iterator]();
|
||||||
|
for (let cursor = 0; visited.size < frames.length; cursor++) {
|
||||||
|
if (cursor >= pending.length) {
|
||||||
|
let next = remaining.next();
|
||||||
|
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
|
||||||
|
if (next.done) break;
|
||||||
|
pending.push(next.value);
|
||||||
|
}
|
||||||
if (cursor % 256 === 0) await captureCheckpoint(signal);
|
if (cursor % 256 === 0) await captureCheckpoint(signal);
|
||||||
const frame = pending[cursor];
|
const frame = pending[cursor];
|
||||||
|
if (visited.has(frame.frameId)) continue;
|
||||||
|
visited.add(frame.frameId);
|
||||||
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
|
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
|
||||||
const doc = sources.get(frame.frameId);
|
const doc = sources.get(frame.frameId);
|
||||||
if (!doc) continue;
|
if (!doc) continue;
|
||||||
const source = { target, frameId: frame.frameId };
|
const source = { target, frameId: frame.frameId };
|
||||||
const parent = frame.parentFrameId ? frameById.get(frame.parentFrameId) : undefined;
|
const state: FrameProjectionState | null =
|
||||||
const state: FrameProjectionState | null = !parent
|
frame.frameId === rootFrameId
|
||||||
? {
|
? {
|
||||||
status: "available",
|
status: "available",
|
||||||
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
|
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
|
||||||
}
|
}
|
||||||
: (projections.get(frame.frameId) ?? null);
|
: (projections.get(frame.frameId) ?? null);
|
||||||
|
if (!state && frame.frameId !== rootFrameId)
|
||||||
|
issues.push({
|
||||||
|
target,
|
||||||
|
frameId: frame.frameId,
|
||||||
|
stage: "ownership",
|
||||||
|
reason: "frame-ownership-unresolved",
|
||||||
|
});
|
||||||
projections.set(frame.frameId, state);
|
projections.set(frame.frameId, state);
|
||||||
const projection = state?.status === "available" ? state.projection : null;
|
const projection = state?.status === "available" ? state.projection : null;
|
||||||
if (!projection || (target.sessionId && !targetProjection))
|
if (!projection || (target.sessionId && !targetProjection))
|
||||||
@@ -173,8 +193,8 @@ export async function normalizeSnapshot(
|
|||||||
if (i % 256 === 0) await captureCheckpoint(signal);
|
if (i % 256 === 0) await captureCheckpoint(signal);
|
||||||
const node = decoded.nodes[i];
|
const node = decoded.nodes[i];
|
||||||
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
|
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
|
||||||
x: doc.scrollOffsetX ?? (parent ? 0 : scrollX),
|
x: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
|
||||||
y: doc.scrollOffsetY ?? (parent ? 0 : scrollY),
|
y: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
|
||||||
});
|
});
|
||||||
const local = input?.rect;
|
const local = input?.rect;
|
||||||
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
|
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
|
||||||
@@ -209,9 +229,5 @@ export async function normalizeSnapshot(
|
|||||||
domNodes,
|
domNodes,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
for (const frameId of sources.keys()) {
|
|
||||||
if (!projections.has(frameId))
|
|
||||||
issues.push({ target, frameId, stage: "ownership", reason: "frame-ownership-unresolved" });
|
|
||||||
}
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||||
import type { CapturedNode } from "./facts";
|
import type { CapturedNode } from "./facts";
|
||||||
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
|
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
|
||||||
export const REQUESTED_STYLES = [
|
export const REQUESTED_STYLES = [
|
||||||
@@ -22,6 +23,7 @@ interface RareBooleanData {
|
|||||||
|
|
||||||
export interface SnapshotDocument {
|
export interface SnapshotDocument {
|
||||||
frameId?: string | number;
|
frameId?: string | number;
|
||||||
|
documentURL?: number;
|
||||||
scrollOffsetX?: number;
|
scrollOffsetX?: number;
|
||||||
scrollOffsetY?: number;
|
scrollOffsetY?: number;
|
||||||
nodes?: {
|
nodes?: {
|
||||||
@@ -211,3 +213,68 @@ export async function decodeDocument(
|
|||||||
}
|
}
|
||||||
return { nodes };
|
return { nodes };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Snapshot membership and same-target edges are current response evidence.
|
||||||
|
* The earlier graph only supplies compatible metadata and cross-target edges. */
|
||||||
|
export function describeSnapshotFrames(
|
||||||
|
snapshot: SnapshotReply,
|
||||||
|
target: CdpTarget,
|
||||||
|
hintById: ReadonlyMap<string, CdpFrame>,
|
||||||
|
pageUrl?: string,
|
||||||
|
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
|
||||||
|
const raw = snapshot.documents ?? [];
|
||||||
|
const strings = snapshot.strings ?? [];
|
||||||
|
const ids = new Set<string>();
|
||||||
|
const invalid = new Set<string>();
|
||||||
|
const frames = raw.map((doc) => {
|
||||||
|
const frameId = snapshotFrameId(doc, strings);
|
||||||
|
if (!frameId) return undefined;
|
||||||
|
if (ids.has(frameId)) invalid.add(frameId);
|
||||||
|
ids.add(frameId);
|
||||||
|
const hint = hintById.get(frameId);
|
||||||
|
const compatible =
|
||||||
|
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
|
||||||
|
const url =
|
||||||
|
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
|
||||||
|
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
|
||||||
|
});
|
||||||
|
const root = frames[0];
|
||||||
|
if (root) {
|
||||||
|
const hint = hintById.get(root.frameId);
|
||||||
|
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
|
||||||
|
if (
|
||||||
|
target.sessionId &&
|
||||||
|
hint &&
|
||||||
|
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
|
||||||
|
parent &&
|
||||||
|
cdpTargetKey(parent.target) !== cdpTargetKey(target)
|
||||||
|
) {
|
||||||
|
root.parentFrameId = parent.frameId;
|
||||||
|
root.ownerBackendNodeId = hint.ownerBackendNodeId;
|
||||||
|
}
|
||||||
|
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
|
||||||
|
}
|
||||||
|
const owned = new Set<string>();
|
||||||
|
for (let i = 0; i < raw.length; i++) {
|
||||||
|
const edges = raw[i].nodes?.contentDocumentIndex;
|
||||||
|
const parent = frames[i];
|
||||||
|
if (!parent || !edges) continue;
|
||||||
|
for (let e = 0; e < edges.index.length; e++) {
|
||||||
|
const child = frames[edges.value[e]];
|
||||||
|
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
||||||
|
if (!child || owner === undefined) continue;
|
||||||
|
if (child === root || child === parent || owned.has(child.frameId)) {
|
||||||
|
invalid.add(child.frameId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
owned.add(child.frameId);
|
||||||
|
child.parentFrameId = parent.frameId;
|
||||||
|
child.ownerBackendNodeId = owner;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
|
||||||
|
ids,
|
||||||
|
rootFrameId: root?.frameId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user