import { renderVom } from "@browser-skill/vom"; import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { buildVomScene, type CdpAxNode, handleGetHtml, handleObserve, handleScreenshot, handleSnapshot, parsePngDimensions, type ScreenshotDeps, stripDataUrlPrefix, } from "../observation"; import type { CapturedNode, CapturedViewModel } from "../vom/capture"; function fakeAgentWindow(ids: number[]) { let i = 0; return { create: vi.fn(async () => { const id = ids[i++]; if (id === undefined) throw new Error("ran out of fake ids"); return id; }), remove: vi.fn(async () => {}), ensureActiveTab: vi.fn(async () => {}), }; } // 1x1 transparent PNG, base64-encoded. Width 1, height 1. const TINY_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABh6FO1AAAAABJRU5ErkJggg=="; function makeScreenshotDeps( opts: { cdp?: CdpRunner; get?: ScreenshotDeps["tabsApi"]["get"]; query?: ScreenshotDeps["tabsApi"]["query"]; captureVisibleTab?: ScreenshotDeps["captureApi"]["captureVisibleTab"]; } = {}, ): ScreenshotDeps { const get = opts.get ?? vi.fn(async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab); const query = opts.query ?? vi.fn(async () => [{ id: 7, windowId: 100, active: true } as chrome.tabs.Tab]); const captureVisibleTab = opts.captureVisibleTab ?? vi.fn(async () => `data:image/png;base64,${TINY_PNG}`); const tabsApi = { get, query }; return { cdp: opts.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab }, }; } function makeFakeCdp(handlers: Record unknown>) { const sent: Array<{ method: string; params?: object }> = []; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { sent.push({ method, params }); const handler = handlers[method]; if (!handler) throw new Error(`unexpected CDP call ${method}`); return handler(params); }); return { cdp: { send, trackSessionTab: vi.fn() } as unknown as CdpRunner, sent }; } describe("stripDataUrlPrefix", () => { it("strips well-formed image/* data URLs", () => { expect(stripDataUrlPrefix(`data:image/png;base64,${TINY_PNG}`)).toBe(TINY_PNG); expect(stripDataUrlPrefix(`data:image/jpeg;base64,abc`)).toBe("abc"); }); it("leaves plain base64 untouched", () => { expect(stripDataUrlPrefix(TINY_PNG)).toBe(TINY_PNG); }); }); describe("parsePngDimensions", () => { it("parses width/height from the IHDR chunk", () => { expect(parsePngDimensions(TINY_PNG)).toEqual({ width: 1, height: 1 }); }); it("returns null on non-PNG input", () => { expect(parsePngDimensions("not-a-png-payload-just-random-base64-text-zzzzzzzzz")).toBeNull(); }); }); describe("handleScreenshot", () => { const emptyGet = vi.fn(async () => { throw new Error("tab not found"); }); it("captures the Agent Window's active tab when tab_id is omitted", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`); const get = vi.fn(); const query = vi.fn(async (_q: chrome.tabs.QueryInfo) => [ { id: 7, windowId: 100, active: true } as chrome.tabs.Tab, ]); const res = await handleScreenshot( sm, { session_id: "aa11" }, makeScreenshotDeps({ captureVisibleTab: capture, get, query }), ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.tab_id).toBe(7); expect(res.image_base64).toBe(TINY_PNG); expect(res.format).toBe("png"); expect(res.width).toBe(1); expect(res.height).toBe(1); expect(capture).toHaveBeenCalledWith(100, { format: "png" }); expect(get).not.toHaveBeenCalled(); }); it("returns not_found when Agent Window has no active tab", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const res = await handleScreenshot( sm, { session_id: "aa11" }, makeScreenshotDeps({ captureVisibleTab: vi.fn(), get: emptyGet, query: vi.fn(async () => []), }), ); expect(res).toMatchObject({ code: "not_found" }); }); it("captures an explicit active user tab in its real window", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`); const get = vi.fn(async () => ({ id: 9, windowId: 200, active: true }) as chrome.tabs.Tab); const res = await handleScreenshot( sm, { session_id: "aa11", tab_id: 9 }, makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn() }), ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.tab_id).toBe(9); expect(capture).toHaveBeenCalledWith(200, { format: "png" }); }); it("rejects screenshots for inactive explicit tabs", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`); const get = vi.fn(async () => ({ id: 9, windowId: 100, active: false }) as chrome.tabs.Tab); const res = await handleScreenshot( sm, { session_id: "aa11", tab_id: 9 }, makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn() }), ); expect(res).toMatchObject({ code: "invalid_params", message: /not active/, data: { reason: "tab_not_active" }, }); expect(capture).not.toHaveBeenCalled(); }); it("hides other sessions' Agent Window tabs", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100, 101]) }); await sm.start("aa11"); await sm.start("bb22"); const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`); const get = vi.fn(async () => ({ id: 9, windowId: 101, active: true }) as chrome.tabs.Tab); const res = await handleScreenshot( sm, { session_id: "aa11", tab_id: 9 }, makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn() }), ); expect(res).toMatchObject({ code: "not_found" }); expect(capture).not.toHaveBeenCalled(); }); it("propagates capture errors as cdp_failed", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const capture = vi.fn(async () => { throw new Error("captureVisibleTab refused"); }); const res = await handleScreenshot( sm, { session_id: "aa11", tab_id: 9 }, makeScreenshotDeps({ captureVisibleTab: capture, get: vi.fn(async () => ({ id: 9, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(), }), ); expect(res).toMatchObject({ code: "cdp_failed", message: /captureVisibleTab refused/ }); }); it("captures a clipped PNG when ref is given", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e5", 999, { tabId: 7 }); const { cdp, sent } = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "Page.captureScreenshot": () => ({ data: TINY_PNG }), }); const captureVisibleTab = vi.fn(); const res = await handleScreenshot( sm, { session_id: "aa11", ref: "@e5", tab_id: 7 }, makeScreenshotDeps({ cdp, get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab), query: vi.fn(), captureVisibleTab, }), ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.tab_id).toBe(7); expect(res.image_base64).toBe(TINY_PNG); expect(res.width).toBe(1); expect(res.height).toBe(1); expect(captureVisibleTab).not.toHaveBeenCalled(); const clip = ( sent.find((c) => c.method === "Page.captureScreenshot")?.params as { clip?: { x: number; y: number; width: number; height: number }; } )?.clip; expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); }); it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const { cdp } = makeFakeCdp({}); const res = await handleScreenshot( sm, { session_id: "aa11", ref: "@e99", tab_id: 7 }, makeScreenshotDeps({ cdp, get: vi.fn(async () => ({ id: 7, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(), captureVisibleTab: vi.fn(), }), ); expect(res).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } }); expect(cdp.send).not.toHaveBeenCalled(); }); it("accepts bare eN ref form", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e5", 999, { tabId: 7 }); const { cdp, sent } = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "Page.captureScreenshot": () => ({ data: TINY_PNG }), }); const captureVisibleTab = vi.fn(); const res = await handleScreenshot( sm, { session_id: "aa11", ref: "e5", tab_id: 7 }, makeScreenshotDeps({ cdp, get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab), query: vi.fn(), captureVisibleTab, }), ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.image_base64).toBe(TINY_PNG); expect(captureVisibleTab).not.toHaveBeenCalled(); expect(sent.some((c) => c.method === "Page.captureScreenshot")).toBe(true); }); it("returns not_found when ref belongs to another tab", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e7", 4242, { tabId: 4 }); const { cdp } = makeFakeCdp({}); const res = await handleScreenshot( sm, { session_id: "aa11", tab_id: 5, ref: "@e7" }, makeScreenshotDeps({ cdp, get: vi.fn(async () => ({ id: 5, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(), captureVisibleTab: vi.fn(), }), ); expect(res).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } }); expect(cdp.send).not.toHaveBeenCalled(); }); it("returns permission_denied when element has no visible box", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e1", 555, { tabId: 7 }); const { cdp } = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [] }), "DOM.getBoxModel": () => { throw new Error("no box"); }, }); const res = await handleScreenshot( sm, { session_id: "aa11", ref: "@e1", tab_id: 7 }, makeScreenshotDeps({ cdp, get: vi.fn(async () => ({ id: 7, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(), captureVisibleTab: vi.fn(), }), ); expect(res).toMatchObject({ code: "permission_denied", message: /not visible/i, data: { reason: "element_not_visible" }, }); }); it("propagates Page.captureScreenshot errors as cdp_failed", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e2", 888, { tabId: 7 }); const { cdp } = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }), "Page.captureScreenshot": () => { throw new Error("capture refused"); }, }); const res = await handleScreenshot( sm, { session_id: "aa11", ref: "@e2", tab_id: 7 }, makeScreenshotDeps({ cdp, get: vi.fn(async () => ({ id: 7, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(), captureVisibleTab: vi.fn(), }), ); expect(res).toMatchObject({ code: "cdp_failed", message: /capture refused/ }); }); }); // --------------------------------------------------------------------------- // buildVomScene // --------------------------------------------------------------------------- describe("buildVomScene", () => { it("joins AX semantics with captured geometry by backendDOMNodeId", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "Example" }, backendDOMNodeId: 100, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Submit" }, backendDOMNodeId: 200, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 100, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 200, parentBackendNodeId: 100, tag: "button", attrs: {}, rect: { x: 20, y: 20, w: 120, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.viewport).toEqual({ width: 1000, height: 800 }); expect(scene.nodes[0]).toEqual( expect.objectContaining({ id: 100, parentId: null, role: "RootWebArea", name: "Example", tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 }, domParentId: null, domAncestorIds: [], }), ); expect(scene.nodes[1]).toEqual( expect.objectContaining({ id: 200, parentId: 100, role: "button", name: "Submit", tag: "button", rect: { x: 20, y: 20, w: 120, h: 40 }, domParentId: 100, domAncestorIds: [100], attrs: {}, }), ); }); it("annotates external links from the explicit page URL without reading window.location", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 100, childIds: ["2", "3"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "Link" }, name: { type: "computedString", value: "Docs" }, backendDOMNodeId: 200, }, { nodeId: "3", parentId: "1", role: { type: "role", value: "link" }, name: { type: "computedString", value: "Local" }, backendDOMNodeId: 300, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 100, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 200, parentBackendNodeId: 100, tag: "a", attrs: { href: "https://docs.example.org/start" }, rect: { x: 10, y: 10, w: 60, h: 20 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, { backendNodeId: 300, parentBackendNodeId: 100, tag: "a", attrs: { href: "/local" }, rect: { x: 80, y: 10, w: 60, h: 20 }, paintOrder: 2, position: "static", pointerEvents: "auto", }, ], }; const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org"); expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined(); }); it("uses the nearest backend AX ancestor as parentId", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "generic" }, childIds: ["3"], }, { nodeId: "3", parentId: "2", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Go" }, backendDOMNodeId: 30, }, ]; const captured: CapturedViewModel = { viewport: { width: 400, height: 300 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 400, h: 300 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 30, parentBackendNodeId: null, tag: "button", attrs: {}, rect: { x: 10, y: 10, w: 80, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]); expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10); }); it("maps iframe sub-document controls to VomNodes", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "Iframe" }, backendDOMNodeId: 20, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "iframe", attrs: {}, rect: { x: 100, y: 100, w: 400, h: 300 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], iframeNodes: new Map([ [ 20, [ { backendNodeId: 101, parentBackendNodeId: null, tag: "input", attrs: { type: "text", placeholder: "请输入手机号" }, rect: { x: 0, y: 0, w: 300, h: 40 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 102, parentBackendNodeId: null, tag: "input", attrs: { type: "password", placeholder: "密码" }, rect: { x: 0, y: 50, w: 300, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], ], ]), excludedBackendNodeIds: new Set(), }; const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) => [101, 102].includes(node.id), ); expect(iframeControls).toEqual([ expect.objectContaining({ id: 101, parentId: 20, role: "textbox", name: "请输入手机号", sensitive: false, }), expect.objectContaining({ id: 102, parentId: 20, role: "textbox", name: "密码", sensitive: true, }), ]); }); it("dedupes iframe fallback controls already exposed through AX", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "Iframe" }, backendDOMNodeId: 20, childIds: ["3"], }, { nodeId: "3", parentId: "2", role: { type: "role", value: "textbox" }, name: { type: "x", value: "输入密码" }, backendDOMNodeId: 202, properties: [{ name: "inputType", value: { value: "password" } }], }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "iframe", attrs: {}, rect: { x: 100, y: 100, w: 400, h: 300 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], iframeNodes: new Map([ [ 20, [ { backendNodeId: 102, parentBackendNodeId: null, tag: "input", attrs: { type: "password", placeholder: "输入密码" }, rect: { x: 0, y: 50, w: 300, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], ], ]), excludedBackendNodeIds: new Set(), }; const passwordNodes = buildVomScene(axNodes, captured).nodes.filter( (node) => node.role === "textbox" && node.name === "输入密码", ); expect(passwordNodes).toHaveLength(1); expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); }); it("keeps unnamed iframe controls but skips unnamed iframe links", () => { const scene = buildVomScene( [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "Iframe" }, backendDOMNodeId: 20, }, ], { viewport: { width: 1000, height: 800 }, nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "iframe", attrs: {}, rect: { x: 100, y: 100, w: 400, h: 300 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], iframeNodes: new Map([ [ 20, [ { backendNodeId: 201, parentBackendNodeId: null, tag: "button", attrs: {}, rect: { x: 0, y: 0, w: 80, h: 30 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 202, parentBackendNodeId: null, tag: "a", attrs: {}, rect: { x: 0, y: 40, w: 80, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], ], ]), excludedBackendNodeIds: new Set(), }, ); expect(scene.nodes.find((node) => node.id === 201)).toEqual( expect.objectContaining({ id: 201, parentId: 20, role: "button", }), ); expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name"); expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined(); }); it("preserves captured-only iframe anchors for iframe sub-document controls", () => { const scene = buildVomScene( [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "Checkout" }, backendDOMNodeId: 10, }, ], { viewport: { width: 1000, height: 800 }, nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "iframe", attrs: { title: "支付验证" }, rect: { x: 100, y: 100, w: 400, h: 300 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], iframeNodes: new Map([ [ 20, [ { backendNodeId: 201, parentBackendNodeId: null, tag: "button", attrs: { "aria-label": "继续" }, rect: { x: 0, y: 0, w: 80, h: 30 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 202, parentBackendNodeId: null, tag: "input", attrs: { type: "text", placeholder: "验证码" }, rect: { x: 0, y: 40, w: 160, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], ], ]), excludedBackendNodeIds: new Set(), }, ); expect(scene.nodes.find((node) => node.id === 20)).toEqual( expect.objectContaining({ id: 20, parentId: 10, role: "Iframe", name: "支付验证", }), ); const rendered = renderVom(scene).text; expect(rendered).toContain(' Iframe "支付验证"'); expect(rendered).toContain(' @e1 button "继续"'); expect(rendered).toContain(' @e2 textbox "验证码"'); }); it("does not mark non-password inputs sensitive from password-like labels", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "textbox" }, name: { type: "computedString", value: "Password recovery email" }, backendDOMNodeId: 10, }, ]; const captured: CapturedViewModel = { viewport: { width: 320, height: 240 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "input", attrs: { type: "text", name: "password_hint", placeholder: "Enter password recovery email", autocomplete: "current-password", }, rect: { x: 10, y: 10, w: 200, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }; expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ id: 10, sensitive: false, }), ); }); it("keeps placeholder separate from the accessible name", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "textbox" }, name: { type: "computedString", value: "+86" }, backendDOMNodeId: 10, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "input", attrs: { type: "text", placeholder: "输入手机号" }, rect: { x: 0, y: 0, w: 240, h: 48 }, paintOrder: 1, position: "static", pointerEvents: "auto", cursor: "text", }, ], }; expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ id: 10, role: "textbox", name: "+86", placeholder: "输入手机号", }), ); }); it("keeps the accessible name for a filled field and lets the value carry the input", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "textbox" }, name: { type: "computedString", value: "Email" }, value: { value: "a@b.com" }, backendDOMNodeId: 10, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "input", attrs: { type: "text", placeholder: "you@example.com" }, rect: { x: 0, y: 0, w: 240, h: 48 }, paintOrder: 1, position: "static", pointerEvents: "auto", cursor: "text", }, ], }; expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }), ); }); it("uses nearby preceding text as a form control label", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 1, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "textbox" }, backendDOMNodeId: 20, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 1, parentBackendNodeId: null, tag: "div", attrs: {}, rect: { x: 0, y: 0, w: 500, h: 80 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 10, parentBackendNodeId: 1, tag: "span", attrs: {}, textContent: "验证码:", rect: { x: 0, y: 0, w: 80, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 1, tag: "input", attrs: { type: "text" }, rect: { x: 90, y: 0, w: 120, h: 30 }, paintOrder: 2, position: "static", pointerEvents: "auto", cursor: "text", }, ], }; expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( expect.objectContaining({ role: "textbox", name: "验证码" }), ); }); it("uses preceding AX static text as a form control label", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 1, childIds: ["2", "3"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "StaticText" }, name: { type: "computedString", value: "验证码:" }, }, { nodeId: "3", parentId: "1", role: { type: "role", value: "textbox" }, backendDOMNodeId: 20, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 20, parentBackendNodeId: null, tag: "input", attrs: { type: "text" }, rect: { x: 90, y: 0, w: 120, h: 30 }, paintOrder: 2, position: "static", pointerEvents: "auto", cursor: "text", }, ], }; expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }), ); }); it("marks runtime default input values without treating them as ordinary filled input", () => { const scene = buildVomScene( [ { nodeId: "1", role: { type: "role", value: "textbox" }, backendDOMNodeId: 10, }, ], { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "input", attrs: { type: "text" }, formValue: "没有手机号的用编码代替", formDefaultValue: "没有手机号的用编码代替", rect: { x: 0, y: 0, w: 240, h: 48 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }, ); expect(scene.nodes[0]).toEqual( expect.objectContaining({ inputState: "default", value: "没有手机号的用编码代替", }), ); }); it("promotes a cursor:pointer icon control to a button named from its sprite", () => { // A close button built as
// — the AX tree exposes it as `generic` with no name, so it would be dropped. const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "generic" }, backendDOMNodeId: 20, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", cursor: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "div", attrs: { class: "close-button" }, rect: { x: 940, y: 20, w: 40, h: 40 }, paintOrder: 5, position: "absolute", pointerEvents: "auto", cursor: "pointer", }, { backendNodeId: 21, parentBackendNodeId: 20, tag: "svg", attrs: {}, rect: { x: 945, y: 25, w: 20, h: 20 }, paintOrder: 6, position: "static", pointerEvents: "auto", cursor: "pointer", }, { backendNodeId: 22, parentBackendNodeId: 21, tag: "use", attrs: { "xlink:href": "#close" }, rect: null, paintOrder: 0, position: "static", pointerEvents: "auto", cursor: "auto", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.nodes.find((n) => n.id === 20)).toEqual( expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }), ); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "close"'); expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); }); it("does not promote a clickable container that wraps a real interactive control", () => { // A clickable card wrapping a real link — the card must stay a container so // the inner link keeps its own ref instead of being collapsed. const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "generic" }, backendDOMNodeId: 20, childIds: ["3"], }, { nodeId: "3", parentId: "2", role: { type: "role", value: "link" }, name: { type: "computedString", value: "Open note" }, backendDOMNodeId: 30, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", cursor: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "div", attrs: { class: "card" }, rect: { x: 0, y: 0, w: 300, h: 200 }, paintOrder: 1, position: "static", pointerEvents: "auto", cursor: "pointer", }, { backendNodeId: 30, parentBackendNodeId: 20, tag: "a", attrs: {}, rect: { x: 0, y: 0, w: 300, h: 200 }, paintOrder: 2, position: "static", pointerEvents: "auto", cursor: "pointer", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic"); expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link"); }); it("promotes only the outermost clickable in a nested pointer chain", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "generic" }, backendDOMNodeId: 20, childIds: ["3"], }, { nodeId: "3", parentId: "2", role: { type: "role", value: "generic" }, backendDOMNodeId: 30, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", cursor: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "span", attrs: { "aria-label": "收藏" }, rect: { x: 10, y: 10, w: 60, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", cursor: "pointer", }, { backendNodeId: 30, parentBackendNodeId: 20, tag: "span", attrs: {}, rect: { x: 12, y: 12, w: 24, h: 24 }, paintOrder: 2, position: "static", pointerEvents: "auto", cursor: "pointer", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.nodes.find((n) => n.id === 20)).toEqual( expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }), ); expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic"); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); }); it("builds active scope blocks from active aria-controls relationships", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "tab" }, name: { type: "computedString", value: "Reviews (12)" }, backendDOMNodeId: 20, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "button", attrs: { role: "tab", "aria-selected": "true", "aria-controls": "reviews-panel" }, rect: { x: 20, y: 20, w: 120, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, { backendNodeId: 30, parentBackendNodeId: 10, tag: "div", attrs: { id: "reviews-panel" }, rect: null, paintOrder: 0, position: "static", pointerEvents: "auto", textContent: "Jane - ear cups are small", }, { backendNodeId: 31, parentBackendNodeId: 30, tag: "p", attrs: {}, rect: null, paintOrder: 0, position: "static", pointerEvents: "auto", textContent: "Bob - great sound", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.activeScopeBlocks).toEqual([ { triggerId: 20, label: "Reviews (12)", lines: ["Jane - ear cups are small", "Bob - great sound"], }, ]); expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]"); }); it("maps hover surface probes to matching backend node ids", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Products" }, backendDOMNodeId: 20, }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ], nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "button", attrs: {}, rect: { x: 20, y: 20, w: 120, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }); expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ]); expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); it("maps hover surface probes to rendered descendants in the same trigger subtree", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "image" }, backendDOMNodeId: 21, }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, ], nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "div", attrs: { class: "tg-avatar" }, rect: { x: 900, y: 10, w: 30, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, { backendNodeId: 21, parentBackendNodeId: 20, tag: "div", attrs: { class: "tg-avatar__inner" }, rect: { x: 902, y: 12, w: 26, h: 26 }, paintOrder: 2, position: "static", pointerEvents: "auto", }, ], }); expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]'); }); it("deduplicates hover surface probes by original trigger backend id", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "image" }, backendDOMNodeId: 21, }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] }, ], nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "div", attrs: { class: "tg-avatar" }, rect: { x: 900, y: 10, w: 30, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, { backendNodeId: 21, parentBackendNodeId: 20, tag: "div", attrs: { class: "tg-avatar__inner" }, rect: { x: 902, y: 12, w: 26, h: 26 }, paintOrder: 2, position: "static", pointerEvents: "auto", }, ], }); expect(scene.surfaces).toEqual([ { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, ]); }); it("maps hover surface probes to DOM-recovered custom controls", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "img" }, name: { type: "computedString", value: "image" }, backendDOMNodeId: 21, }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ { triggerBackendNodeId: 20, triggerPoint: { x: 915, y: 25 }, triggerAction: "hover", subItems: ["My profile", "Sign out"], }, ], nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", cursor: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "div", attrs: { class: "tg-avatar" }, rect: { x: 900, y: 10, w: 30, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", cursor: "pointer", }, { backendNodeId: 21, parentBackendNodeId: 20, tag: "div", attrs: { class: "tg-avatar__inner" }, rect: { x: 902, y: 12, w: 26, h: 26 }, paintOrder: 2, position: "static", pointerEvents: "auto", cursor: "pointer", }, ], }); expect(scene.surfaces).toEqual([ { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, ]); expect(renderVom(scene).text).toContain( '@e1 button "image" [hover first: My profile | Sign out]', ); }); it("does not attach hover probes to distant geometry matches", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "link" }, name: { type: "computedString", value: "0" }, backendDOMNodeId: 42, }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ { triggerBackendNodeId: 999, triggerPoint: { x: 900, y: 20 }, triggerAction: "hover", subItems: ["My profile"], }, ], nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 42, parentBackendNodeId: 10, tag: "a", attrs: {}, rect: { x: 860, y: 120, w: 40, h: 30 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }); expect(scene.surfaces).toBeUndefined(); expect(renderVom(scene).text).not.toContain("[hover first:"); }); it("enriches names and active scope signals from AX properties", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2", "3"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Products" }, backendDOMNodeId: 20, properties: [ { name: "hasPopup", value: { value: "menu" } }, { name: "expanded", value: { value: "true" } }, { name: "controls", value: { value: "products-panel" } }, ], }, { nodeId: "3", parentId: "1", role: { type: "role", value: "textbox" }, name: { type: "computedString", value: "Card number" }, backendDOMNodeId: 30, properties: [{ name: "inputType", value: { value: "credit-card" } }], }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "button", attrs: {}, rect: { x: 20, y: 20, w: 120, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, { backendNodeId: 25, parentBackendNodeId: 10, tag: "div", attrs: { id: "products-panel" }, rect: null, paintOrder: 0, position: "static", pointerEvents: "auto", textContent: "Shoes Bags", }, { backendNodeId: 30, parentBackendNodeId: 10, tag: "input", attrs: {}, rect: { x: 20, y: 80, w: 200, h: 40 }, paintOrder: 2, position: "static", pointerEvents: "auto", }, ], }); expect(scene.nodes.find((node) => node.id === 20)).toEqual( expect.objectContaining({ name: "Products [expanded]", attrs: expect.objectContaining({ "aria-expanded": "true", "aria-controls": "products-panel", }), }), ); expect(scene.activeScopeBlocks).toEqual([ { triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] }, ]); expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true); }); it("aggregates AX virtual text into unnamed structural nodes", () => { const scene = buildVomScene( [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 10, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "paragraph" }, backendDOMNodeId: 20, childIds: ["3"], }, { nodeId: "3", parentId: "2", role: { type: "role", value: "InlineTextBox" }, name: { type: "computedString", value: "Inline only text" }, }, ], { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 10, parentBackendNodeId: null, tag: "body", attrs: {}, rect: { x: 0, y: 0, w: 1000, h: 800 }, paintOrder: 0, position: "static", pointerEvents: "auto", }, { backendNodeId: 20, parentBackendNodeId: 10, tag: "p", attrs: {}, rect: { x: 20, y: 20, w: 400, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }, ); expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text"); }); it("marks captured dialog elements as modal without AX role or aria-modal", () => { const scene = buildVomScene([], { viewport: { width: 640, height: 480 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { backendNodeId: 50, parentBackendNodeId: null, tag: "dialog", attrs: {}, rect: { x: 100, y: 100, w: 300, h: 200 }, paintOrder: 10, position: "fixed", pointerEvents: "auto", }, ], }); expect(scene.nodes[0]).toEqual( expect.objectContaining({ id: 50, modal: true, }), ); }); it("filters AX overlay nodes listed in excludedBackendNodeIds", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 1, childIds: ["2", "3"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "x", value: "Real Button" }, backendDOMNodeId: 2, }, { nodeId: "3", parentId: "1", role: { type: "role", value: "button" }, name: { type: "x", value: "中断" }, backendDOMNodeId: 202, }, ]; const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set([200, 201, 202]), nodes: [ { backendNodeId: 2, parentBackendNodeId: 1, tag: "button", attrs: {}, rect: { x: 10, y: 10, w: 100, h: 40 }, paintOrder: 1, position: "static", pointerEvents: "auto", }, ], }; const scene = buildVomScene(axNodes, captured); expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined(); expect(scene.nodes.find((n) => n.id === 2)).toEqual( expect.objectContaining({ role: "button", name: "Real Button" }), ); }); it("filters AX overlay descendants when an ancestor backend id is excluded", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, backendDOMNodeId: 1, childIds: ["2"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "generic" }, backendDOMNodeId: 200, childIds: ["3"], }, { nodeId: "3", parentId: "2", role: { type: "role", value: "button" }, name: { type: "x", value: "中断" }, backendDOMNodeId: 203, }, ]; const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, iframeNodes: new Map(), excludedBackendNodeIds: new Set([200]), nodes: [], }); expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined(); }); }); describe("handleSnapshot", () => { function makeDeps(nodes: CdpAxNode[]) { const sendImpl = async (_tabId: number, method: string, _params?: object) => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes }; if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") throw new Error("snapshot unsupported"); throw new Error(`unexpected CDP method ${method}`); }; const send = vi.fn(sendImpl); const trackSessionTab = vi.fn(); const cdp = { send: send as unknown as ( tabId: number, method: string, params?: object, ) => Promise, trackSessionTab, }; return { cdp, tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }, send, trackSessionTab, }; } it("populates the session's RefStore with backendNodeIds", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); const root: CdpAxNode = { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "Example" }, backendDOMNodeId: 100, childIds: ["2"], }; const button: CdpAxNode = { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Click" }, backendDOMNodeId: 200, }; const deps = makeDeps([root, button]); const res = await handleSnapshot(sm, { session_id: "aa11" }, deps); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.ref_count).toBe(1); expect(res.tab_id).toBe(4); expect(res.text).toContain("@vom 1"); expect(res.text).toContain("@layers 1 focus=L1"); expect(res.text).toContain("L1 page"); expect(res.text).toContain('RootWebArea "Example"'); expect(res.text).toContain('@e1 button "Click"'); expect(deps.trackSessionTab).toHaveBeenCalledWith("aa11", 4); expect(ctx.refStore.resolve("e1")).toBe(200); expect(ctx.refStore.resolve("e1", { tabId: 4 })).toBe(200); expect(ctx.refStore.resolve("e1", { tabId: 5 })).toBeNull(); expect(ctx.refStore.resolveEntry("e1")).toMatchObject({ backendNodeId: 200, tabId: 4 }); expect(ctx.refStore.resolve("e2")).toBeNull(); }); it("keeps snapshots static without conditional surface probing", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const root: CdpAxNode = { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "Example" }, backendDOMNodeId: 100, childIds: ["2"], }; const button: CdpAxNode = { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Products" }, backendDOMNodeId: 200, }; const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"]; const i = (s: string) => strings.indexOf(s); const send = vi.fn(async (_tabId: number, method: string) => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { return { strings, documents: [ { nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], backendNodeId: [100, 200], attributes: [[], []], }, layout: { nodeIndex: [0, 1], styles: [ [i("static"), i("auto"), i("auto")], [i("static"), i("auto"), i("auto")], ], bounds: [ [0, 0, 1000, 800], [20, 20, 120, 40], ], paintOrders: [0, 1], }, }, ], }; } if (method === "Runtime.evaluate") return { result: { value: [] } }; throw new Error(`unexpected CDP method ${method}`); }); const res = await handleSnapshot( sm, { session_id: "aa11" }, { cdp: { send: send as unknown as ( tabId: number, method: string, params?: object, ) => Promise, trackSessionTab: vi.fn(), }, tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }, }, ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(send).not.toHaveBeenCalledWith( 4, "Runtime.evaluate", expect.objectContaining({ returnByValue: true }), ); }); it("enables conditional surface probing for semantic observe", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const root: CdpAxNode = { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "Example" }, backendDOMNodeId: 100, childIds: ["2"], }; const button: CdpAxNode = { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Products" }, backendDOMNodeId: 200, }; const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"]; const i = (s: string) => strings.indexOf(s); const send = vi.fn(async (_tabId: number, method: string) => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; } if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { return { strings, documents: [ { nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], backendNodeId: [100, 200], attributes: [[], []], }, layout: { nodeIndex: [0, 1], styles: [ [i("static"), i("auto"), i("auto")], [i("static"), i("auto"), i("auto")], ], bounds: [ [0, 0, 1000, 800], [20, 20, 120, 40], ], paintOrders: [0, 1], }, }, ], }; } if (method === "Runtime.evaluate") return { result: { value: [] } }; throw new Error(`unexpected CDP method ${method}`); }); const res = await handleObserve( sm, { session_id: "aa11", debug_surfaces: true }, { cdp: { send: send as unknown as ( tabId: number, method: string, params?: object, ) => Promise, trackSessionTab: vi.fn(), }, tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }, }, ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.debug).toEqual({ surface_probes: [] }); expect(send).toHaveBeenCalledWith( 4, "Runtime.evaluate", expect.objectContaining({ returnByValue: true }), ); }); it("allows passive snapshots of explicit user-window tabs", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const root: CdpAxNode = { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "User tab" }, backendDOMNodeId: 100, childIds: ["2"], }; const button: CdpAxNode = { nodeId: "2", parentId: "1", role: { type: "role", value: "button" }, name: { type: "computedString", value: "Read" }, backendDOMNodeId: 200, }; const deps = makeDeps([root, button]); deps.tabsApi.get = vi.fn( async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab, ); const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.tab_id).toBe(9); expect(res.text).toContain('@e1 button "Read"'); expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {}); }); it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const deps = makeDeps([]); deps.tabsApi.get = vi.fn( async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab, ); const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps); expect(res).toMatchObject({ code: "permission_denied", data: { reason: "agent_window_scope" }, }); expect(deps.send).not.toHaveBeenCalled(); }); it("resets the RefStore on every fresh snapshot", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e9", 9999); // stale entry from a previous snapshot const deps = makeDeps([ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "computedString", value: "Doc" }, backendDOMNodeId: 1, }, ]); await handleSnapshot(sm, { session_id: "aa11" }, deps); expect(ctx.refStore.resolve("e9")).toBeNull(); expect(ctx.refStore.resolve("e1")).toBeNull(); }); it("surfaces CDP failures as cdp_failed", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const sendImpl = async () => { throw new Error("debugger detached"); }; const send = vi.fn(sendImpl); const deps = { cdp: { send: send as unknown as ( tabId: number, method: string, params?: object, ) => Promise, }, tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }, }; const res = await handleSnapshot(sm, { session_id: "aa11" }, deps); expect(res).toMatchObject({ code: "cdp_failed", message: /debugger detached/ }); }); function makeOverlayDeps(axNodes: CdpAxNode[], snapshotReply: unknown, metrics: unknown) { const send = vi.fn(async (_tabId: number, method: string) => { if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: axNodes }; if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshotReply; if (method === "Page.getLayoutMetrics") return metrics; throw new Error(`unexpected CDP method ${method}`); }); return { cdp: { send: send as unknown as (t: number, m: string, p?: object) => Promise, trackSessionTab: vi.fn(), }, tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }, send, }; } // Reusable: html>body>div(fixed full-screen)>input[type=password] function loginSnapshotReply() { const S = [ "html", "body", "div", "input", "position", "fixed", "static", "pointer-events", "auto", "type", "password", ]; const i = (s: string) => S.indexOf(s); return { strings: S, documents: [ { nodes: { parentIndex: [-1, 0, 1, 2], nodeName: [i("html"), i("body"), i("div"), i("input")], backendNodeId: [10, 11, 12, 13], attributes: [[], [], [], [i("type"), i("password")]], }, layout: { nodeIndex: [1, 2, 3], styles: [ [i("static"), i("auto")], [i("fixed"), i("auto")], [i("static"), i("auto")], ], bounds: [ [0, 0, 1000, 4000], [0, 0, 1000, 800], [400, 300, 200, 40], ], paintOrders: [0, 50, 51], }, }, ], }; } const VP_METRICS = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; it("renders a blocking login overlay as the focused top layer", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const ax: CdpAxNode[] = [ { nodeId: "1", role: { type: "role", value: "RootWebArea" }, name: { type: "x", value: "JD" }, backendDOMNodeId: 11, childIds: ["2", "3"], }, { nodeId: "2", parentId: "1", role: { type: "role", value: "dialog" }, name: { type: "x", value: "登录" }, backendDOMNodeId: 12, childIds: ["4"], }, { nodeId: "4", parentId: "2", role: { type: "role", value: "textbox" }, name: { type: "x", value: "密码" }, backendDOMNodeId: 13, }, { nodeId: "3", parentId: "1", role: { type: "role", value: "button" }, name: { type: "x", value: "底层按钮" }, backendDOMNodeId: 999, }, ]; const res = await handleSnapshot( sm, { session_id: "aa11" }, makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS), ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.text).toContain("@vom 1"); expect(res.text).toContain("L1 modal cover=100%"); expect(res.text).toContain("occluded by L1"); // The underlying page button (backendNodeId 999) must NOT be rendered. expect(res.text).not.toContain("底层按钮"); expect(res.text).toContain('dialog "登录"'); expect(res.text).toContain('@e1 textbox "密码"'); // AX subtree is indented under the L1 layer line. expect(res.text).toMatch(/\n {2}dialog "登录"/); // ref-store only carries interactive overlay refs, not structural nodes or the occluded page. expect(res.ref_count).toBe(1); }); it("renders cross-origin iframe login form in L1 when AX tree has no iframe children (JD-style)", async () => { // Mirrors JD.com: the main frame has a fixed wrapper that contains a // cross-origin