From c002c394e691e89081c5db034e00dec939dc4a47 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Mon, 27 Jul 2026 14:53:10 +0800 Subject: [PATCH] feat(vom): vom fix --- .../src/tools/__tests__/observation.test.ts | 67 +++++++++++++++++++ apps/extension/src/tools/observation.ts | 38 ++++++++--- packages/vom/src/__tests__/layers.test.ts | 21 ++++++ packages/vom/src/__tests__/render.test.ts | 18 +++++ packages/vom/src/layers.ts | 21 +++--- packages/vom/src/render.ts | 13 ++-- 6 files changed, 154 insertions(+), 24 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index bbb5900..d6cd621 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -424,6 +424,73 @@ describe("buildVomScene", () => { ); }); + it("annotates external links from the explicit page URL without reading window.location", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 100, + childIds: ["2", "3"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "Link" }, + name: { type: "computedString", value: "Docs" }, + backendDOMNodeId: 200, + }, + { + nodeId: "3", + parentId: "1", + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "Local" }, + backendDOMNodeId: 300, + }, + ]; + const captured: CapturedViewModel = { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 100, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 200, + parentBackendNodeId: 100, + tag: "a", + attrs: { href: "https://docs.example.org/start" }, + rect: { x: 10, y: 10, w: 60, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 300, + parentBackendNodeId: 100, + tag: "a", + attrs: { href: "/local" }, + rect: { x: 80, y: 10, w: 60, h: 20 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }; + + const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); + + expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org"); + expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined(); + }); + it("uses the nearest backend AX ancestor as parentId", () => { const axNodes: CdpAxNode[] = [ { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 3c39e33..48fe799 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -969,7 +969,29 @@ function axNodeInOverlaySubtree( ); } -export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene { +export interface BuildVomSceneOptions { + pageUrl?: string; +} + +function externalHrefHost( + href: string | undefined, + pageUrl: string | undefined, +): string | undefined { + if (!href || !pageUrl) return undefined; + try { + const page = new URL(pageUrl); + const target = new URL(href, page); + return target.origin !== page.origin ? target.hostname : undefined; + } catch { + return undefined; + } +} + +export function buildVomScene( + axNodes: CdpAxNode[], + captured: CapturedViewModel, + options: BuildVomSceneOptions = {}, +): VomScene { const signals = capturedOnlySignals(captured.nodes); const { capturedByBackendId } = signals; const axSignals = buildAxSignals(axNodes); @@ -993,15 +1015,9 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) // For link nodes, attach external hostname so the renderer can annotate it. // Relative hrefs and same-origin hrefs are omitted to avoid noise. - if (role === "link" && capturedNode?.attrs.href) { - try { - const target = new URL(capturedNode.attrs.href, window.location.href); - if (target.origin !== window.location.origin) { - vomNode.href = target.hostname; - } - } catch { - // Relative or non-URL href (e.g. javascript:) — skip - } + if (role?.toLowerCase() === "link") { + const hrefHost = externalHrefHost(capturedNode?.attrs.href, options.pageUrl); + if (hrefHost) vomNode.href = hrefHost; } seenBackendIds.add(axNode.backendDOMNodeId); @@ -1243,7 +1259,7 @@ async function handleVomObservation( ); const axNodes = result.nodes ?? []; const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe); - const scene = buildVomScene(axNodes, captured); + const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens, diff --git a/packages/vom/src/__tests__/layers.test.ts b/packages/vom/src/__tests__/layers.test.ts index 16d280f..acc7dc8 100644 --- a/packages/vom/src/__tests__/layers.test.ts +++ b/packages/vom/src/__tests__/layers.test.ts @@ -115,6 +115,27 @@ describe("detectBlockingLayer", () => { expect(layer?.kind).toBe("modal"); }); + it("detects modal features regardless of role or tag casing", () => { + const layer = detectBlockingLayer( + [ + node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }), + node({ + id: 2, + parentId: 1, + tag: "DIV", + role: "Dialog", + rect: { x: 300, y: 200, w: 400, h: 300 }, + paintOrder: 90, + position: "fixed", + }), + ], + VP, + ); + + expect(layer?.rootId).toBe(2); + expect(layer?.kind).toBe("modal"); + }); + it("ignores small toasts and pointer-events:none covers", () => { expect( detectBlockingLayer( diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 4eeccc8..7d964d3 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -87,6 +87,24 @@ describe("renderVom single-layer page", () => { expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); + it("annotates external links regardless of role casing", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "Link", + name: "Docs", + href: "docs.example.org", + tag: "a", + }), + ]), + ); + + expect(out.text).toContain('@e1 Link "Docs" [→ docs.example.org]'); + }); + it("renders additional structural roles without assigning refs", () => { const out = renderVom( scene([ diff --git a/packages/vom/src/layers.ts b/packages/vom/src/layers.ts index c5a6774..676d23e 100644 --- a/packages/vom/src/layers.ts +++ b/packages/vom/src/layers.ts @@ -8,6 +8,14 @@ export const MASK_COVERAGE_THRESHOLD = 0.9; const POSITIONED = new Set(["fixed", "absolute", "sticky"]); const FORM_TAGS = new Set(["input", "textarea", "select"]); +function normalizedRole(node: VomNode): string { + return node.role?.toLowerCase() ?? ""; +} + +function normalizedTag(node: VomNode): string { + return node.tag.toLowerCase(); +} + export function coverage(rect: Rect | null, vp: Viewport): number { if (!rect || vp.width <= 0 || vp.height <= 0) return 0; const ix = Math.max(0, Math.min(rect.x + rect.w, vp.width) - Math.max(rect.x, 0)); @@ -23,12 +31,9 @@ function isBlockingCandidate(node: VomNode): boolean { } function hasModalFeature(node: VomNode): boolean { - return ( - node.modal === true || - node.tag === "dialog" || - node.role === "dialog" || - node.role === "alertdialog" - ); + const role = normalizedRole(node); + const tag = normalizedTag(node); + return node.modal === true || tag === "dialog" || role === "dialog" || role === "alertdialog"; } function classifyLayer( @@ -39,8 +44,8 @@ function classifyLayer( for (const node of nodes) { if (!members.has(node.id)) continue; if (hasModalFeature(node)) return "modal"; - if (FORM_TAGS.has(node.tag)) return "modal"; - if (node.tag === "iframe") return "modal"; + if (FORM_TAGS.has(normalizedTag(node))) return "modal"; + if (normalizedTag(node) === "iframe") return "modal"; } return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal"; } diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 5f72cb0..8af814e 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -166,6 +166,10 @@ function normalizedRole(node: VomNode): string { return node.role?.toLowerCase() ?? ""; } +function normalizedTag(node: VomNode): string { + return node.tag.toLowerCase(); +} + function normalizeContextKey(s: string): string { return s.split(/\s+/).join(" ").toLowerCase(); } @@ -296,7 +300,7 @@ function rectArea(node: VomNode): number { function passesRecoveryGuards(node: VomNode, children: Map): boolean { const area = rectArea(node); - const tag = node.tag.toLowerCase(); + const tag = normalizedTag(node); if (node.pointerEvents === "none") return false; if (node.disabled || node.inert || node.insideNative) return false; @@ -611,7 +615,7 @@ function renderNodeLine( // For link nodes with an external href, annotate so the agent can // distinguish external navigation from same-origin links. - if (node.role === "link" && node.href) { + if (normalizedRole(node) === "link" && node.href) { line += ` [→ ${node.href}]`; } @@ -838,9 +842,8 @@ function interactionPoints(rect: Rect): Array<[number, number]> { function isModalLike(node: VomNode): boolean { const role = normalizedRole(node); - return ( - node.modal === true || node.tag === "dialog" || role === "dialog" || role === "alertdialog" - ); + const tag = normalizedTag(node); + return node.modal === true || tag === "dialog" || role === "dialog" || role === "alertdialog"; } function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number): number {