From 36d0d914197daca62799a90e5874a520fa1479f2 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 8 Sep 2026 21:42:20 +0800 Subject: [PATCH] fix(observation): preserve snapshot document provenance and URL metadata --- .../src/tools/__tests__/observation.test.ts | 60 ++++++ apps/extension/src/tools/observation.ts | 2 +- .../vom/__tests__/capture-coordinator.test.ts | 199 +++++++++++++++++- .../vom/__tests__/frame-document.test.ts | 8 +- .../src/tools/vom/capture-coordinator.ts | 126 +++++++---- .../extension/src/tools/vom/frame-document.ts | 16 +- apps/extension/src/tools/vom/normalize.ts | 50 +++-- apps/extension/src/tools/vom/snapshot.ts | 67 ++++++ 8 files changed, 455 insertions(+), 73 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index da52eb5..182df46 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -3353,6 +3353,66 @@ describe("handleSnapshot", () => { }; } + it.each([ + false, + true, + ])("preserves external-link hints without graph metadata (snapshot URL: %s)", async (hasSnapshotUrl) => { + const snapshot = { + strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"], + documents: [ + { + frameId: "main", + ...(hasSnapshotUrl ? { documentURL: 4 } : {}), + nodes: { + backendNodeId: [1, 2], + parentIndex: [-1, 0], + nodeName: [0, 1], + attributes: [[], [2, 3]], + }, + layout: { + nodeIndex: [0, 1], + bounds: [ + [0, 0, 1000, 800], + [10, 20, 100, 40], + ], + }, + }, + ], + }; + const ax: CdpAxNode[] = [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["link"], + }, + { + nodeId: "link", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "Docs" }, + }, + ]; + const deps = makeOverlayDeps(ax, snapshot, VP_METRICS); + const cdp: CdpRunner = { + ...deps.cdp, + getFrameGraph: async () => { + throw new Error("graph unavailable"); + }, + }; + const result = await captureVomObservation( + cdp, + 4, + hasSnapshotUrl ? undefined : "https://page.test/start", + ); + expect(result.text).toContain("outside.test"); + expect(result.text).toContain("Docs"); + expect( + deps.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"), + ).toHaveLength(1); + }); + it("exposes the frame-aware observation result for recording", async () => { const ax: CdpAxNode[] = [ { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 1c09abe..00e6c42 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -894,7 +894,7 @@ export async function captureVomObservation( throwIfAborted(options.signal, "observation"); await cdp.ensureAttachedToUrl?.(tabId, url); throwIfAborted(options.signal, "observation"); - const facts = await captureObservationFacts(cdp, tabId, options.signal); + const facts = await captureObservationFacts(cdp, tabId, options.signal, url); const { captured, documents: normalizedDocuments } = semanticCapture(facts); throwIfAborted(options.signal, "observation"); const semanticGraph = buildSemanticGraph({ diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index 981019d..bc82868 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -437,7 +437,8 @@ describe("OOPIF capture", () => { const reply = async (_target: unknown, method: string) => { if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } }; - if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "DOMSnapshot.captureSnapshot") + return typeof _target === "number" ? childSnapshot("main", 11) : snapshot; if (method === "DOM.getBoxModel") return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } }; if (method === "DOM.resolveNode") throw new Error("nested owner replaced"); @@ -589,6 +590,7 @@ function siblingCaptureFixture( } }); return { + snapshot, cdp: { send: send as CdpRunner["send"] }, send, peak: () => peak, @@ -600,6 +602,201 @@ function siblingCaptureFixture( }; } +describe("snapshot document provenance", () => { + it.each([ + "new-child", + "new-root", + "stale-owner", + ])("retains current snapshot evidence with %s graph data", async (mode) => { + const f = siblingCaptureFixture(); + const cdp: CdpRunner = { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: mode === "new-root" ? "old-root" : "frame-0", + frames: [ + { frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } }, + ...(mode === "stale-owner" + ? [ + { + frameId: "frame-1", + parentFrameId: "frame-0", + ownerBackendNodeId: 999, + target: { tabId: 4 }, + }, + ] + : []), + ], + }), + }; + const facts = await captureObservationFacts(cdp, 4); + expect(facts.rootFrameId).toBe("frame-0"); + expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({ + frame: { ownerBackendNodeId: 100 }, + domNodes: [ + expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }), + expect.anything(), + ], + }); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect, + ).not.toBeNull(); + expect(f.measured()).not.toContain(999); + expect( + f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"), + ).toHaveLength(1); + expect( + f.send.mock.calls.some( + ([, method, params]) => + method === "Accessibility.getFullAXTree" && + (params as { frameId?: string }).frameId === "frame-7", + ), + ).toBe(true); + }); + + it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => { + const f = siblingCaptureFixture(); + f.snapshot.strings.push("https://current.test/page", "https://child.test/page"); + Object.assign(f.snapshot.documents[0], { documentURL: 2 }); + Object.assign(f.snapshot.documents[1], { documentURL: 3 }); + const facts = await captureObservationFacts( + { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: "frame-0", + frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }], + }), + }, + 4, + undefined, + "https://caller.test/", + ); + expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe( + "https://current.test/page", + ); + expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe( + "https://child.test/page", + ); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url, + ).toBeUndefined(); + }); + + it("accepts a migrated snapshot claim without merging AX from the old target", async () => { + const f = siblingCaptureFixture(); + const cdp: CdpRunner = { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: "frame-0", + frames: [ + { frameId: "frame-0", target: { tabId: 4 } }, + { + frameId: "frame-1", + parentFrameId: "frame-0", + ownerBackendNodeId: 100, + target: { tabId: 4, sessionId: "old" }, + }, + ], + }), + sendToTarget: async (_target: CdpTarget, method: string): Promise => + (method === "Accessibility.getFullAXTree" + ? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] } + : {}) as T, + }; + const facts = await captureObservationFacts(cdp, 4); + const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1"); + expect(current?.frame.target).toEqual({ tabId: 4 }); + expect(current?.domNodes.length).toBeGreaterThan(0); + expect(current?.axNodes).toEqual([]); + }); + + it.each([ + 0, 1, + ])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => { + const f = siblingCaptureFixture(); + f.snapshot.documents.push(f.snapshot.documents[index]); + if (index === 0) { + await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow( + "root document ownership is ambiguous", + ); + } else { + const facts = await captureObservationFacts(f.cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7"); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length, + ).toBeGreaterThan(0); + } + }); + + it("retains the caller URL in the root AX-only fallback", async () => { + const cdp: CdpRunner = { + getFrameGraph: async () => { + throw new Error("no graph"); + }, + send: async (_tabId: number, method: string): Promise => { + if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot"); + return ( + method === "Accessibility.getFullAXTree" + ? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] } + : {} + ) as T; + }, + }; + const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/"); + expect(facts.documents[0].frame.url).toBe("https://page.test/"); + expect(facts.documents[0].axNodes).toHaveLength(1); + }); + + it("retains disconnected document data without inventing top-level geometry", async () => { + const f = siblingCaptureFixture(); + f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] }; + const facts = await captureObservationFacts(f.cdp, 4); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0], + ).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } }); + expect(f.measured()).toEqual([]); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "frame-1", stage: "ownership" }), + ); + }); + + it.each([ + false, + true, + ])("isolates competing target claims independent of batch order %s", async (reverse) => { + const f = siblingCaptureFixture(); + const remote = { + frameId: "frame-1", + parentFrameId: "frame-0", + ownerBackendNodeId: 100, + target: { tabId: 4, sessionId: "remote" }, + }; + const main = { frameId: "frame-0", target: { tabId: 4 } }; + const cdp: CdpRunner = { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: "frame-0", + frames: reverse ? [remote, main] : [main, remote], + }), + sendToTarget: async (_target: CdpTarget, method: string): Promise => + (method === "DOMSnapshot.captureSnapshot" + ? { ...f.snapshot, documents: [f.snapshot.documents[1]] } + : method === "Page.getLayoutMetrics" + ? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } + : {}) as T, + }; + const facts = await captureObservationFacts(cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7"); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect, + ).not.toBeNull(); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "frame-1", stage: "ownership" }), + ); + }); +}); + describe("sibling frame measurement scheduling", () => { it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => { const pending = new Map void>(); diff --git a/apps/extension/src/tools/vom/__tests__/frame-document.test.ts b/apps/extension/src/tools/vom/__tests__/frame-document.test.ts index ce7a39c..b62ef56 100644 --- a/apps/extension/src/tools/vom/__tests__/frame-document.test.ts +++ b/apps/extension/src/tools/vom/__tests__/frame-document.test.ts @@ -159,10 +159,12 @@ describe("buildFrameDocuments", () => { ["child", [domNode("child", 9)]], ]); const capture = captured(frameNodes); - capture.frameOwnerBackendNodeIds = new Map([["child", 7]]); - capture.frameParentIds = new Map([["child", "left"]]); const frames = ["main", "left", "right", "child"].map((frameId) => ({ - frame: { frameId, target: { tabId: 4 } }, + frame: { + frameId, + target: { tabId: 4 }, + ...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}), + }, nodes: [], })); diff --git a/apps/extension/src/tools/vom/capture-coordinator.ts b/apps/extension/src/tools/vom/capture-coordinator.ts index c8315c0..f1fed99 100644 --- a/apps/extension/src/tools/vom/capture-coordinator.ts +++ b/apps/extension/src/tools/vom/capture-coordinator.ts @@ -23,7 +23,7 @@ import { type FrameOwnedAxNode, } from "./frame-document"; import { type NormalizedDocument, normalizeSnapshot } from "./normalize"; -import { REQUESTED_STYLES, type SnapshotReply, snapshotFrameId } from "./snapshot"; +import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot"; interface TargetBatch { target: CdpTarget; @@ -31,6 +31,8 @@ interface TargetBatch { documents: NormalizedDocument[]; ax: FrameAxBatch[]; fallbackExcluded: Set; + snapshotFrameIds?: Set; + rootFrameId?: string; } /** A fixed worker pool scoped to one capture; each worker issues at most one @@ -62,36 +64,11 @@ async function collectTargets( if (failed) throw failure; } -/** When topology discovery is unavailable, snapshot documents still carry - * explicit target/frame provenance. Only explicit contentDocumentIndex edges - * establish ancestry; a foreign frame is never renamed to fit a request. */ -function snapshotFrames(snapshot: SnapshotReply, target: CdpTarget): CdpFrame[] { - const raw = snapshot.documents ?? []; - const strings = snapshot.strings ?? []; - const frames = raw.map((doc) => { - const frameId = snapshotFrameId(doc, strings); - return frameId ? ({ frameId, target } as CdpFrame) : undefined; - }); - for (let i = 0; i < raw.length; i++) { - const edges = raw[i].nodes?.contentDocumentIndex; - const parent = frames[i]; - if (!parent || !edges) continue; - for (let e = 0; e < edges.index.length; e++) { - const child = frames[edges.value[e]]; - const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]]; - if (child && owner !== undefined && child !== parent) { - child.parentFrameId = parent.frameId; - child.ownerBackendNodeId = owner; - } - } - } - return frames.filter((frame): frame is CdpFrame => !!frame); -} - export async function captureObservationFacts( cdp: CdpRunner, tabId: number, signal?: AbortSignal, + pageUrl?: string, ): Promise> { const startedAt = Date.now(); throwCaptureAborted(signal); @@ -104,6 +81,7 @@ export async function captureObservationFacts( throwCaptureAborted(signal); const geometry = new GeometryContext(cdp, tabId, graph, signal); const issues: CaptureIssue[] = []; + const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []); const groups = new Map>(); for (const frame of graph?.frames ?? []) { const key = cdpTargetKey(frame.target); @@ -118,9 +96,13 @@ export async function captureObservationFacts( }; groups.set(key, group); } - group.frames.push(frame); + group.frames.push( + !frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl + ? { ...frame, url: pageUrl } + : frame, + ); } - if (!groups.size) + if (!groups.has(cdpTargetKey({ tabId }))) groups.set(cdpTargetKey({ tabId }), { target: { tabId }, frames: [], @@ -146,14 +128,22 @@ export async function captureObservationFacts( includeDOMRects: true, }); throwCaptureAborted(signal); - if (!graph) batch.frames = snapshotFrames(snapshot, batch.target); + const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl); + batch.snapshotFrameIds = observed.ids; + batch.rootFrameId = observed.rootFrameId; + // Keep graph-only frames for AX fallback, never in preference to observed membership. + batch.frames = [ + ...observed.frames, + ...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)), + ]; batch.documents = await normalizeSnapshot( snapshot, batch.target, - batch.frames, + batch.frames.filter((frame) => observed.ids.has(frame.frameId)), geometry, issues, signal, + observed.rootFrameId, ); const formsAvailable = await enrichFormControlStates( scoped, @@ -174,7 +164,13 @@ export async function captureObservationFacts( batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal); } if (!batch.frames.length && !graph) - batch.frames = [{ frameId: "root", target: batch.target }]; + batch.frames = [ + { + frameId: "root", + target: batch.target, + ...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}), + }, + ]; try { throwCaptureAborted(signal); await scoped.send(tabId, "Accessibility.enable", {}); @@ -206,12 +202,68 @@ export async function captureObservationFacts( signal, ); - const frames = batches.flatMap((batch) => batch.frames); - const rootFrameId = graph?.rootFrameId ?? frames[0]?.frameId ?? "root"; - const decoded = new Map( - batches.flatMap((batch) => batch.documents).map((doc) => [doc.frame.frameId, doc]), + // A snapshot claim outranks an old graph hint. Conflicting actual claims + // cannot be resolved by completion order or by overwriting a frameId map. + const claims = new Map(); + const invalid = new Set(); + for (const batch of batches) { + const targetKey = cdpTargetKey(batch.target); + for (const id of batch.snapshotFrameIds ?? []) { + const previous = claims.get(id); + if (previous !== undefined && previous !== targetKey) invalid.add(id); + claims.set(id, targetKey); + } + const accepted = new Set(batch.frames.map((frame) => frame.frameId)); + for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id); + } + const currentFrames = batches.flatMap((batch) => + batch.frames.filter( + (frame) => + !claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target), + ), ); - const ax = batches.flatMap((batch) => batch.ax); + const children = new Map(); + for (const frame of currentFrames) { + if (!frame.parentFrameId) continue; + const siblings = children.get(frame.parentFrameId); + if (siblings) siblings.push(frame.frameId); + else children.set(frame.parentFrameId, [frame.frameId]); + } + const blocked = [...invalid]; + for (let i = 0; i < blocked.length; i++) { + for (const child of children.get(blocked[i]) ?? []) { + if (!invalid.has(child)) { + invalid.add(child); + blocked.push(child); + } + } + } + for (const frame of currentFrames) { + if (invalid.has(frame.frameId)) + issues.push({ + target: frame.target, + frameId: frame.frameId, + stage: "ownership", + reason: "frame-ownership-unresolved", + }); + } + const rootBatch = batches.find((batch) => !batch.target.sessionId); + const rootFrameId = + rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root"; + if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous"); + const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId)); + const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); + const belongs = (frame: CdpFrame) => { + const current = frameById.get(frame.frameId); + return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target); + }; + const decoded = new Map( + batches + .flatMap((batch) => batch.documents) + .filter((doc) => belongs(doc.frame)) + .map((doc) => [doc.frame.frameId, doc]), + ); + const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame)); const documents = await buildFrameDocuments( { rootFrameId, frames }, ax, diff --git a/apps/extension/src/tools/vom/frame-document.ts b/apps/extension/src/tools/vom/frame-document.ts index 0e72b18..0635ed9 100644 --- a/apps/extension/src/tools/vom/frame-document.ts +++ b/apps/extension/src/tools/vom/frame-document.ts @@ -32,8 +32,6 @@ export interface FrameDomInput { nodes: CapturedNode[]; rootFrameId?: string; frameNodes?: ReadonlyMap; - frameOwnerBackendNodeIds?: ReadonlyMap; - frameParentIds?: ReadonlyMap; } interface Ownership { @@ -58,23 +56,13 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string { function frameList( graph: CdpFrameGraph | null, batches: FrameAxBatch[], - captured: FrameDomInput, ): CdpFrame[] { const frames = new Map(); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); for (const batch of batches) { if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame); } - return [...frames.values()].map((frame) => { - const ownerBackendNodeId = - frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId); - const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId); - return { - ...frame, - ...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}), - ...(parentFrameId ? { parentFrameId } : {}), - }; - }); + return [...frames.values()]; } export async function buildFrameDocuments( @@ -85,7 +73,7 @@ export async function buildFrameDocuments( unresolved?: (frame: CdpFrame) => void, ): Promise[]> { let work = 0; - const frames = frameList(graph, batches, captured); + const frames = frameList(graph, batches); const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; const domNodesForFrame = (frameId: string): CapturedNode[] => diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 98f9249..02b2e0f 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -1,5 +1,5 @@ import type { Viewport } from "@browser-skill/vom"; -import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; +import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; import { type GeometryProjection, projectRectToViewport } from "../geometry"; import { type FrameProjectionState, @@ -39,6 +39,7 @@ export async function normalizeSnapshot( geometry: GeometryContext, issues: CaptureIssue[], signal?: AbortSignal, + rootFrameId?: string, ): Promise { const strings = snapshot.strings ?? []; const raw = snapshot.documents ?? []; @@ -82,28 +83,47 @@ export async function normalizeSnapshot( const projections = new Map(); const result: NormalizedDocument[] = []; let targetProjection: GeometryProjection | null = null; - const targetRoot = pending[0]; + const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined; if (target.sessionId && targetRoot) { try { - targetProjection = await geometry.targetProjection(targetRoot.frameId); + const known = await geometry.frame(targetRoot.frameId); + if (known && cdpTargetKey(known.target) === cdpTargetKey(target)) + targetProjection = await geometry.targetProjection(targetRoot.frameId); } catch (error) { if (isCaptureAbort(error)) throw error; } } - for (let cursor = 0; cursor < pending.length; cursor++) { + const visited = new Set(); + const remaining = frames[Symbol.iterator](); + for (let cursor = 0; visited.size < frames.length; cursor++) { + if (cursor >= pending.length) { + let next = remaining.next(); + while (!next.done && visited.has(next.value.frameId)) next = remaining.next(); + if (next.done) break; + pending.push(next.value); + } if (cursor % 256 === 0) await captureCheckpoint(signal); const frame = pending[cursor]; + if (visited.has(frame.frameId)) continue; + visited.add(frame.frameId); for (const child of children.get(frame.frameId) ?? []) pending.push(child); const doc = sources.get(frame.frameId); if (!doc) continue; const source = { target, frameId: frame.frameId }; - const parent = frame.parentFrameId ? frameById.get(frame.parentFrameId) : undefined; - const state: FrameProjectionState | null = !parent - ? { - status: "available", - projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } }, - } - : (projections.get(frame.frameId) ?? null); + const state: FrameProjectionState | null = + frame.frameId === rootFrameId + ? { + status: "available", + projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } }, + } + : (projections.get(frame.frameId) ?? null); + if (!state && frame.frameId !== rootFrameId) + issues.push({ + target, + frameId: frame.frameId, + stage: "ownership", + reason: "frame-ownership-unresolved", + }); projections.set(frame.frameId, state); const projection = state?.status === "available" ? state.projection : null; if (!projection || (target.sessionId && !targetProjection)) @@ -173,8 +193,8 @@ export async function normalizeSnapshot( if (i % 256 === 0) await captureCheckpoint(signal); const node = decoded.nodes[i]; const input = snapshotViewportRect(node.layout?.bounds ?? [], source, { - x: doc.scrollOffsetX ?? (parent ? 0 : scrollX), - y: doc.scrollOffsetY ?? (parent ? 0 : scrollY), + x: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0), + y: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0), }); const local = input?.rect; let rect = input && projection ? projectSnapshotRect(input, projection) : null; @@ -209,9 +229,5 @@ export async function normalizeSnapshot( domNodes, }); } - for (const frameId of sources.keys()) { - if (!projections.has(frameId)) - issues.push({ target, frameId, stage: "ownership", reason: "frame-ownership-unresolved" }); - } return result; } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 42fc5d4..6ec90b6 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -1,3 +1,4 @@ +import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; import type { CapturedNode } from "./facts"; import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts"; export const REQUESTED_STYLES = [ @@ -22,6 +23,7 @@ interface RareBooleanData { export interface SnapshotDocument { frameId?: string | number; + documentURL?: number; scrollOffsetX?: number; scrollOffsetY?: number; nodes?: { @@ -211,3 +213,68 @@ export async function decodeDocument( } return { nodes }; } + +/** Snapshot membership and same-target edges are current response evidence. + * The earlier graph only supplies compatible metadata and cross-target edges. */ +export function describeSnapshotFrames( + snapshot: SnapshotReply, + target: CdpTarget, + hintById: ReadonlyMap, + pageUrl?: string, +): { frames: CdpFrame[]; ids: Set; rootFrameId?: string } { + const raw = snapshot.documents ?? []; + const strings = snapshot.strings ?? []; + const ids = new Set(); + const invalid = new Set(); + const frames = raw.map((doc) => { + const frameId = snapshotFrameId(doc, strings); + if (!frameId) return undefined; + if (ids.has(frameId)) invalid.add(frameId); + ids.add(frameId); + const hint = hintById.get(frameId); + const compatible = + hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined; + const url = + (doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url; + return { frameId, target, ...(url ? { url } : {}) } as CdpFrame; + }); + const root = frames[0]; + if (root) { + const hint = hintById.get(root.frameId); + const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined; + if ( + target.sessionId && + hint && + cdpTargetKey(hint.target) === cdpTargetKey(target) && + parent && + cdpTargetKey(parent.target) !== cdpTargetKey(target) + ) { + root.parentFrameId = parent.frameId; + root.ownerBackendNodeId = hint.ownerBackendNodeId; + } + if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl; + } + const owned = new Set(); + for (let i = 0; i < raw.length; i++) { + const edges = raw[i].nodes?.contentDocumentIndex; + const parent = frames[i]; + if (!parent || !edges) continue; + for (let e = 0; e < edges.index.length; e++) { + const child = frames[edges.value[e]]; + const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]]; + if (!child || owner === undefined) continue; + if (child === root || child === parent || owned.has(child.frameId)) { + invalid.add(child.frameId); + continue; + } + owned.add(child.frameId); + child.parentFrameId = parent.frameId; + child.ownerBackendNodeId = owner; + } + } + return { + frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)), + ids, + rootFrameId: root?.frameId, + }; +}