From 8d7c0aa66c6d535e8ffce60cf810b84065b525cc Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 18 Aug 2026 14:36:45 +0800 Subject: [PATCH] fix(extension): support iframe content in VOM observations --- .../src/tools/__tests__/observation.test.ts | 284 ++++++++++++++++++ apps/extension/src/tools/observation.ts | 167 ++++++++-- .../src/tools/vom/__tests__/capture.test.ts | 56 ++-- .../tools/vom/__tests__/frame-capture.test.ts | 63 ++++ .../vom/__tests__/frame-document.test.ts | 110 +++++++ apps/extension/src/tools/vom/capture.ts | 187 ++++++++---- apps/extension/src/tools/vom/frame-capture.ts | 75 +++++ .../extension/src/tools/vom/frame-document.ts | 142 +++++++++ packages/vom/src/__tests__/render.test.ts | 197 +++++++++++- packages/vom/src/index.ts | 1 + packages/vom/src/layers.ts | 14 +- packages/vom/src/render.ts | 84 +++++- packages/vom/src/types.ts | 13 +- 13 files changed, 1276 insertions(+), 117 deletions(-) create mode 100644 apps/extension/src/tools/vom/__tests__/frame-capture.test.ts create mode 100644 apps/extension/src/tools/vom/__tests__/frame-document.test.ts create mode 100644 apps/extension/src/tools/vom/frame-capture.ts create mode 100644 apps/extension/src/tools/vom/frame-document.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 603a7c4..1f9b8cb 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -5,6 +5,7 @@ import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridg import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { + buildFrameVomScene, buildVomScene, type CdpAxNode, handleGetHtml, @@ -508,6 +509,284 @@ describe("handleScreenshot overlay suppression", () => { // --------------------------------------------------------------------------- describe("buildVomScene", () => { + it("merges multiple sibling and nested frames without mixing their targets", () => { + const node = ( + backendNodeId: number, + parentBackendNodeId: number | null, + frameId: string, + tag: string, + attrs: Record = {}, + ): CapturedNode => ({ + backendNodeId, + parentBackendNodeId, + frameId, + tag, + attrs, + rect: { x: 0, y: 0, w: 100, h: 30 }, + localRect: { x: 0, y: 0, w: 100, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }); + const mainNodes = [ + node(1, null, "main", "body"), + node(10, 1, "main", "iframe", { title: "First" }), + node(20, 1, "main", "iframe", { title: "Second" }), + ]; + const firstNodes = [ + node(100, null, "first", "body"), + node(101, 100, "first", "button", { "aria-label": "First action" }), + ]; + const secondNodes = [ + node(200, null, "second", "body"), + node(201, 200, "second", "h2"), + node(210, 200, "second", "iframe", { title: "Nested" }), + ]; + const nestedNodes = [ + node(300, null, "nested", "body"), + node(301, 300, "nested", "input", { placeholder: "Nested field" }), + ]; + const captured: CapturedViewModel = { + viewport: { width: 1200, height: 800 }, + nodes: mainNodes, + iframeNodes: new Map([ + [10, firstNodes], + [20, secondNodes], + [210, nestedNodes], + ]), + frameNodes: new Map([ + ["main", mainNodes], + ["first", firstNodes], + ["second", secondNodes], + ["nested", nestedNodes], + ]), + frameOwnerBackendNodeIds: new Map([ + ["first", 10], + ["second", 20], + ["nested", 210], + ]), + rootFrameId: "main", + excludedBackendNodeIds: new Set(), + }; + const axNode = ( + nodeId: string, + backendDOMNodeId: number, + role: string, + parentId?: string, + name?: string, + ): CdpAxNode => ({ + nodeId, + backendDOMNodeId, + role: { type: "role", value: role }, + ...(parentId ? { parentId } : {}), + ...(name ? { name: { type: "computedString", value: name } } : {}), + }); + const result = buildFrameVomScene( + [ + { + frameId: "main", + contextScopeId: "main", + domNodes: mainNodes, + axNodes: [ + axNode("root", 1, "RootWebArea"), + axNode("frame-1", 10, "Iframe", "root", "First"), + axNode("frame-2", 20, "Iframe", "root", "Second"), + ], + }, + { + frameId: "first", + contextScopeId: "first", + parentFrameId: "main", + ownerBackendNodeId: 10, + domNodes: firstNodes, + axNodes: [ + axNode("root", 100, "RootWebArea"), + axNode("button", 101, "button", "root", "First action"), + ], + }, + { + frameId: "second", + contextScopeId: "second", + parentFrameId: "main", + ownerBackendNodeId: 20, + domNodes: secondNodes, + axNodes: [ + axNode("root", 200, "RootWebArea"), + axNode("heading", 201, "heading", "root", "Second heading"), + axNode("nested-frame", 210, "Iframe", "root", "Nested"), + ], + }, + { + frameId: "nested", + contextScopeId: "nested", + parentFrameId: "second", + ownerBackendNodeId: 210, + domNodes: nestedNodes, + axNodes: [ + axNode("root", 300, "RootWebArea"), + axNode("input", 301, "textbox", "root", "Nested field"), + ], + }, + ], + captured, + ); + + const rendered = renderVom(result); + expect(rendered.text).toContain('Iframe "First"'); + expect(rendered.text).toContain('@e1 button "First action"'); + expect(rendered.text).toContain('heading "Second heading"'); + expect(rendered.text).toContain('Iframe "Nested"'); + expect(rendered.text).toContain('textbox "Nested field"'); + expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first"); + expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested"); + + const firstAction = result.nodes.find((item) => item.backendNodeId === 101); + const nestedInput = result.nodes.find((item) => item.backendNodeId === 301); + expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" }); + expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" }); + expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id); + }); + + it("does not place a child document at the page root when its frame boundary is unresolved", () => { + const childNode: CapturedNode = { + backendNodeId: 101, + parentBackendNodeId: null, + frameId: "child", + tag: "button", + attrs: {}, + rect: { x: 0, y: 0, w: 100, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }; + const captured: CapturedViewModel = { + viewport: { width: 800, height: 600 }, + nodes: [], + iframeNodes: new Map(), + frameNodes: new Map([["child", [childNode]]]), + rootFrameId: "main", + excludedBackendNodeIds: new Set(), + }; + + const result = buildFrameVomScene( + [ + { + frameId: "main", + contextScopeId: "main", + domNodes: [], + axNodes: [], + }, + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "main", + domNodes: [childNode], + axNodes: [ + { + nodeId: "button", + frameId: "child", + backendDOMNodeId: 101, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Child action" }, + }, + ], + }, + ], + captured, + ); + + expect(result.nodes).toEqual([]); + }); + + it("keeps AX-only frame content attached to its iframe boundary", () => { + const mainNodes: CapturedNode[] = [ + { + backendNodeId: 1, + parentBackendNodeId: null, + frameId: "main", + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 800, h: 600 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 10, + parentBackendNodeId: 1, + frameId: "main", + tag: "iframe", + attrs: { title: "Remote" }, + rect: { x: 100, y: 100, w: 400, h: 300 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ]; + const captured: CapturedViewModel = { + viewport: { width: 800, height: 600 }, + nodes: mainNodes, + iframeNodes: new Map(), + frameNodes: new Map([ + ["main", mainNodes], + ["child", []], + ]), + rootFrameId: "main", + excludedBackendNodeIds: new Set(), + }; + + const result = buildFrameVomScene( + [ + { + frameId: "main", + contextScopeId: "main", + domNodes: mainNodes, + axNodes: [ + { + nodeId: "main-root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + }, + { + nodeId: "frame-owner", + parentId: "main-root", + backendDOMNodeId: 10, + role: { type: "role", value: "Iframe" }, + name: { type: "computedString", value: "Remote" }, + }, + ], + }, + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + domNodes: [], + axNodes: [ + { + nodeId: "child-root", + backendDOMNodeId: 100, + role: { type: "role", value: "RootWebArea" }, + }, + { + nodeId: "child-button", + parentId: "child-root", + backendDOMNodeId: 101, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "AX fallback action" }, + }, + ], + }, + ], + captured, + ); + + const childButton = result.nodes.find((node) => node.backendNodeId === 101); + expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" }); + expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id); + expect(renderVom(result).text).toContain('@e1 button "AX fallback action"'); + }); + it("joins AX semantics with captured geometry by backendDOMNodeId", () => { const axNodes: CdpAxNode[] = [ { @@ -816,6 +1095,7 @@ describe("buildVomScene", () => { parentId: "2", role: { type: "role", value: "textbox" }, name: { type: "x", value: "输入密码" }, + value: { value: "iframe-secret" }, backendDOMNodeId: 202, properties: [{ name: "inputType", value: { value: "password" } }], }, @@ -870,6 +1150,10 @@ describe("buildVomScene", () => { expect(passwordNodes).toHaveLength(1); expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); + expect(passwordNodes[0].value).toBeUndefined(); + const rendered = renderVom(buildVomScene(axNodes, captured)).text; + expect(rendered).toContain('textbox "输入密码" [filled] ="•••"'); + expect(rendered).not.toContain("iframe-secret"); }); it("keeps unnamed iframe controls but skips unnamed iframe links", () => { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 12c3efc..c7bb973 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -50,6 +50,7 @@ import { captureViewModel, collectOverlayExcludedBackendIds, } from "./vom/capture"; +import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; // --------------------------------------------------------------------------- // Shared helpers (legacy aliases — observation.ts kept exporting these @@ -352,6 +353,7 @@ export type CdpRunner = SharedCdpRunner; /** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */ export interface CdpAxNode { nodeId: string; + frameId?: string; parentId?: string; backendDOMNodeId?: number; ignored?: boolean; @@ -584,7 +586,7 @@ function iframeNameFor(node: CapturedNode): string | undefined { function isRenderableIframeControl(node: CapturedNode): boolean { const tag = normalizeTag(node.tag); if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false; - if (!node.rect) return false; + if (!node.localRect && !node.rect) return false; if (node.pointerEvents === "none") return false; if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false; @@ -756,7 +758,12 @@ function applyCapturedSignals( signals: VomNodeDomSignals, ): VomNode { if (!capturedNode) return node; - const attrs = capturedNode.attrs; + const attrs = + node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value") + ? Object.fromEntries( + Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"), + ) + : capturedNode.attrs; const text = cleanAttr(capturedNode.textContent); const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId); const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder); @@ -823,8 +830,7 @@ function vomNodeFromCaptured( signals: VomNodeDomSignals, ): VomNode { const sensitive = isSensitive(capturedNode); - const value = - capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined); + const value = sensitive ? undefined : capturedNode.formValue; const tag = normalizeTag(capturedNode.tag); const node = applyCapturedSignals( { @@ -860,8 +866,17 @@ function axVomNode( ): VomNode { const role = axString(axNode.role); const capturedValue = capturedNode?.formValue; - const value = capturedValue ?? axValue(axNode.value); + const sourceValue = capturedValue ?? axValue(axNode.value); const signalsForNode = axSignals.get(axNode.nodeId); + const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true; + const value = sensitive ? undefined : sourceValue; + const inputState = + inputStateFor(capturedNode, sourceValue) ?? + (sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "") + ? sourceValue === undefined || sourceValue === "" + ? "empty" + : "filled" + : undefined); let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag)) ? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals) : (axString(axNode.name) ?? signalsForNode?.aggregatedText); @@ -877,8 +892,8 @@ function axVomNode( position: capturedNode?.position || "static", pointerEvents: capturedNode?.pointerEvents || "auto", modal: isModalSignal(axNode, capturedNode), - sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true, - inputState: inputStateFor(capturedNode, value), + sensitive, + inputState, }; if (role) node.role = role; if (name) node.name = name; @@ -1141,6 +1156,8 @@ export interface BuildVomSceneOptions { pageUrl?: string; } +export type VomFrameDocument = CapturedFrameDocument; + function externalHrefHost( href: string | undefined, pageUrl: string | undefined, @@ -1234,6 +1251,119 @@ export function buildVomScene( }; } +export function buildFrameVomScene( + documents: VomFrameDocument[], + captured: CapturedViewModel, + options: BuildVomSceneOptions = {}, +): VomScene { + const documentByFrameId = new Map(documents.map((document) => [document.frameId, document])); + const rootFrameId = + captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId; + const orderedFrameIds = [ + ...(rootFrameId ? [rootFrameId] : []), + ...documents.map((document) => document.frameId), + ].filter((frameId, index, all) => all.indexOf(frameId) === index); + + const localScenes = new Map(); + for (const frameId of orderedFrameIds) { + const document = documentByFrameId.get(frameId); + if (!document) continue; + localScenes.set( + frameId, + buildVomScene( + document.axNodes, + { + viewport: captured.viewport, + nodes: document.domNodes, + iframeNodes: new Map(), + surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [], + excludedBackendNodeIds: + frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(), + }, + { pageUrl: options.pageUrl }, + ), + ); + } + + const sceneIdByFrameAndBackend = new Map(); + let nextSceneId = 1; + for (const [frameId, scene] of localScenes) { + for (const node of scene.nodes) { + sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId); + nextSceneId += 1; + } + } + + const nodes: VomNode[] = []; + const surfaces: CondSurface[] = []; + const activeScopeBlocks: ActiveScopeBlock[] = []; + + for (const [frameId, scene] of localScenes) { + const document = documentByFrameId.get(frameId); + const ownerBackendNodeId = document?.ownerBackendNodeId; + const parentFrameId = document?.parentFrameId; + const ownerSceneId = + ownerBackendNodeId !== undefined && parentFrameId + ? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`) + : undefined; + if (frameId !== rootFrameId && ownerSceneId === undefined) continue; + + const suppressedRoots = new Set( + frameId === rootFrameId + ? [] + : scene.nodes + .filter( + (node) => + node.parentId === null && + ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""), + ) + .map((node) => node.id), + ); + + for (const node of scene.nodes) { + if (suppressedRoots.has(node.id)) continue; + const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number; + let parentId = + node.parentId !== null + ? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`) + : ownerSceneId; + if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId; + const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`); + nodes.push({ + ...node, + id: sceneNodeId, + backendNodeId: node.id, + frameId, + contextScopeId: document?.contextScopeId ?? frameId, + parentId: parentId ?? null, + ...(node.domParentId !== undefined + ? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) } + : {}), + ...(node.domAncestorIds + ? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) } + : {}), + }); + } + + for (const surface of scene.surfaces ?? []) { + const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`); + if (triggerId !== undefined) surfaces.push({ ...surface, triggerId }); + } + for (const block of scene.activeScopeBlocks ?? []) { + const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`); + if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId }); + } + } + + return { + viewport: captured.viewport, + nodes, + ...(rootFrameId ? { rootFrameId } : {}), + ...(surfaces.length > 0 ? { surfaces } : {}), + ...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}), + }; +} + export interface SnapshotDeps { cdp: CdpRunner; tabsApi: { @@ -1447,15 +1577,6 @@ async function handleVomObservation( deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); throwIfAborted(signal, toolName); - await deps.cdp.send(target.tabId, "Accessibility.enable", {}); - throwIfAborted(signal, toolName); - const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>( - target.tabId, - "Accessibility.getFullAXTree", - {}, - ); - throwIfAborted(signal, toolName); - const axNodes = result.nodes ?? []; const effectiveConditionalSurfaceProbe = deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; const captured = await captureForVom( @@ -1466,7 +1587,17 @@ async function handleVomObservation( signal, ); throwIfAborted(signal, toolName); - const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); + const frameDocuments = await captureFrameData( + deps.cdp, + target.tabId, + captured, + signal, + ); + throwIfAborted(signal, toolName); + const scene = + captured.rootFrameId && captured.frameNodes?.size + ? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url }) + : buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens, @@ -1475,7 +1606,7 @@ async function handleVomObservation( throwIfAborted(signal, toolName); ctx.refStore.replace( rendered.refs.map( - (r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const, + (ref) => [ref.ref, { backendNodeId: ref.backendNodeId, tabId: target.tabId }] as const, ), ); return attachDialogs(deps.cdp, target.tabId, dialogCursor, { diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index bc4cd2a..cc0e1f4 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -18,6 +18,8 @@ function fakeSnapshotReply() { "auto", "type", "password", + "value", + "hunter2", ]; const i = (s: string) => S.indexOf(s); return { @@ -29,7 +31,8 @@ function fakeSnapshotReply() { nodeType: [1, 1, 1, 1], nodeName: [i("html"), i("body"), i("div"), i("input")], backendNodeId: [10, 11, 12, 13], - attributes: [[], [], [], [i("type"), i("password")]], + attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]], + inputValue: { index: [3], value: [i("hunter2")] }, }, layout: { nodeIndex: [1, 2, 3], @@ -217,7 +220,8 @@ function makeCdp(snapshot: unknown) { { state: "filled", sensitive: true, - defaultValue: "", + defaultValue: "hunter2", + value: "hunter2", placeholder: "secret", }, ], @@ -262,10 +266,11 @@ describe("captureViewModel", () => { expect(input).toMatchObject({ formState: "filled", - formDefaultValue: "", formPlaceholder: "secret", + attrs: { type: "password" }, }); expect(input?.formValue).toBeUndefined(); + expect(input?.formDefaultValue).toBeUndefined(); expect(cdp.send).toHaveBeenCalledWith( 4, "Runtime.evaluate", @@ -692,10 +697,8 @@ describe("captureViewModel", () => { }; const { nodes } = await captureViewModel(cdp, 4); const div = nodes.find((n) => n.backendNodeId === 12); - // local rect preserves viewport-relative scroll math. expect(div?.localRect?.y).toBe(-200); - // exported rect is clipped to the top-level viewport before VOM consumes it. - expect(div?.rect).toMatchObject({ y: 0, h: 600 }); + expect(div?.rect).toMatchObject({ y: -200, h: 800 }); }); it("normalizes device-pixel bounds by devicePixelRatio", async () => { @@ -839,6 +842,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "main-frame", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -866,6 +870,7 @@ describe("captureViewModel", () => { }, // documents[1]: the iframe's sub-document with a login input { + frameId: "child-frame", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -895,7 +900,14 @@ describe("captureViewModel", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes, iframeNodes } = await captureViewModel(cdp, 4); + const { + nodes, + iframeNodes, + frameNodes, + frameOwnerBackendNodeIds, + frameParentIds, + rootFrameId, + } = await captureViewModel(cdp, 4); // Main frame has 4 nodes; iframe is included expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe"); // iframeNodes keyed by the