figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(extension): support iframe content in VOM observations

Browse Source
This commit is contained in:
Ljy-0827
2026-08-18 19:18:39 +08:00
committed by GitHub
parent a9489282fa
commit 8d7c0aa66c
13 changed files with 1276 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/observation.test.ts
+284
View File
@@ -5,6 +5,7 @@ import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridg
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import {
buildFrameVomScene,
buildVomScene,
type CdpAxNode,
handleGetHtml,
@@ -508,6 +509,284 @@ describe("handleScreenshot overlay suppression", () => {
// ---------------------------------------------------------------------------
describe("buildVomScene", () => {
it("merges multiple sibling and nested frames without mixing their targets", () => {
const node = (
backendNodeId: number,
parentBackendNodeId: number | null,
frameId: string,
tag: string,
attrs: Record<string, string> = {},
): CapturedNode => ({
backendNodeId,
parentBackendNodeId,
frameId,
tag,
attrs,
rect: { x: 0, y: 0, w: 100, h: 30 },
localRect: { x: 0, y: 0, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
});
const mainNodes = [
node(1, null, "main", "body"),
node(10, 1, "main", "iframe", { title: "First" }),
node(20, 1, "main", "iframe", { title: "Second" }),
];
const firstNodes = [
node(100, null, "first", "body"),
node(101, 100, "first", "button", { "aria-label": "First action" }),
];
const secondNodes = [
node(200, null, "second", "body"),
node(201, 200, "second", "h2"),
node(210, 200, "second", "iframe", { title: "Nested" }),
];
const nestedNodes = [
node(300, null, "nested", "body"),
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
];
const captured: CapturedViewModel = {
viewport: { width: 1200, height: 800 },
nodes: mainNodes,
iframeNodes: new Map([
[10, firstNodes],
[20, secondNodes],
[210, nestedNodes],
]),
frameNodes: new Map([
["main", mainNodes],
["first", firstNodes],
["second", secondNodes],
["nested", nestedNodes],
]),
frameOwnerBackendNodeIds: new Map([
["first", 10],
["second", 20],
["nested", 210],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const axNode = (
nodeId: string,
backendDOMNodeId: number,
role: string,
parentId?: string,
name?: string,
): CdpAxNode => ({
nodeId,
backendDOMNodeId,
role: { type: "role", value: role },
...(parentId ? { parentId } : {}),
...(name ? { name: { type: "computedString", value: name } } : {}),
});
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
domNodes: mainNodes,
axNodes: [
axNode("root", 1, "RootWebArea"),
axNode("frame-1", 10, "Iframe", "root", "First"),
axNode("frame-2", 20, "Iframe", "root", "Second"),
],
},
{
frameId: "first",
contextScopeId: "first",
parentFrameId: "main",
ownerBackendNodeId: 10,
domNodes: firstNodes,
axNodes: [
axNode("root", 100, "RootWebArea"),
axNode("button", 101, "button", "root", "First action"),
],
},
{
frameId: "second",
contextScopeId: "second",
parentFrameId: "main",
ownerBackendNodeId: 20,
domNodes: secondNodes,
axNodes: [
axNode("root", 200, "RootWebArea"),
axNode("heading", 201, "heading", "root", "Second heading"),
axNode("nested-frame", 210, "Iframe", "root", "Nested"),
],
},
{
frameId: "nested",
contextScopeId: "nested",
parentFrameId: "second",
ownerBackendNodeId: 210,
domNodes: nestedNodes,
axNodes: [
axNode("root", 300, "RootWebArea"),
axNode("input", 301, "textbox", "root", "Nested field"),
],
},
],
captured,
);
const rendered = renderVom(result);
expect(rendered.text).toContain('Iframe "First"');
expect(rendered.text).toContain('@e1 button "First action"');
expect(rendered.text).toContain('heading "Second heading"');
expect(rendered.text).toContain('Iframe "Nested"');
expect(rendered.text).toContain('textbox "Nested field"');
expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first");
expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested");
const firstAction = result.nodes.find((item) => item.backendNodeId === 101);
const nestedInput = result.nodes.find((item) => item.backendNodeId === 301);
expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" });
expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" });
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
});
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
const childNode: CapturedNode = {
backendNodeId: 101,
parentBackendNodeId: null,
frameId: "child",
tag: "button",
attrs: {},
rect: { x: 0, y: 0, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
const captured: CapturedViewModel = {
viewport: { width: 800, height: 600 },
nodes: [],
iframeNodes: new Map(),
frameNodes: new Map([["child", [childNode]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
domNodes: [],
axNodes: [],
},
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "main",
domNodes: [childNode],
axNodes: [
{
nodeId: "button",
frameId: "child",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Child action" },
},
],
},
],
captured,
);
expect(result.nodes).toEqual([]);
});
it("keeps AX-only frame content attached to its iframe boundary", () => {
const mainNodes: CapturedNode[] = [
{
backendNodeId: 1,
parentBackendNodeId: null,
frameId: "main",
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 800, h: 600 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 10,
parentBackendNodeId: 1,
frameId: "main",
tag: "iframe",
attrs: { title: "Remote" },
rect: { x: 100, y: 100, w: 400, h: 300 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
];
const captured: CapturedViewModel = {
viewport: { width: 800, height: 600 },
nodes: mainNodes,
iframeNodes: new Map(),
frameNodes: new Map([
["main", mainNodes],
["child", []],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
domNodes: mainNodes,
axNodes: [
{
nodeId: "main-root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
},
{
nodeId: "frame-owner",
parentId: "main-root",
backendDOMNodeId: 10,
role: { type: "role", value: "Iframe" },
name: { type: "computedString", value: "Remote" },
},
],
},
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
domNodes: [],
axNodes: [
{
nodeId: "child-root",
backendDOMNodeId: 100,
role: { type: "role", value: "RootWebArea" },
},
{
nodeId: "child-button",
parentId: "child-root",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "AX fallback action" },
},
],
},
],
captured,
);
const childButton = result.nodes.find((node) => node.backendNodeId === 101);
expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" });
expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id);
expect(renderVom(result).text).toContain('@e1 button "AX fallback action"');
});
it("joins AX semantics with captured geometry by backendDOMNodeId", () => {
const axNodes: CdpAxNode[] = [
{
@@ -816,6 +1095,7 @@ describe("buildVomScene", () => {
parentId: "2",
role: { type: "role", value: "textbox" },
name: { type: "x", value: "输入密码" },
value: { value: "iframe-secret" },
backendDOMNodeId: 202,
properties: [{ name: "inputType", value: { value: "password" } }],
},
@@ -870,6 +1150,10 @@ describe("buildVomScene", () => {
expect(passwordNodes).toHaveLength(1);
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
expect(passwordNodes[0].value).toBeUndefined();
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
expect(rendered).not.toContain("iframe-secret");
});
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
apps/extension/src/tools/observation.ts
+149 -18
View File
@@ -50,6 +50,7 @@ import {
captureViewModel,
collectOverlayExcludedBackendIds,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
// ---------------------------------------------------------------------------
// Shared helpers (legacy aliases — observation.ts kept exporting these
@@ -352,6 +353,7 @@ export type CdpRunner = SharedCdpRunner;
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
export interface CdpAxNode {
nodeId: string;
frameId?: string;
parentId?: string;
backendDOMNodeId?: number;
ignored?: boolean;
@@ -584,7 +586,7 @@ function iframeNameFor(node: CapturedNode): string | undefined {
function isRenderableIframeControl(node: CapturedNode): boolean {
const tag = normalizeTag(node.tag);
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
if (!node.rect) return false;
if (!node.localRect && !node.rect) return false;
if (node.pointerEvents === "none") return false;
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
@@ -756,7 +758,12 @@ function applyCapturedSignals(
signals: VomNodeDomSignals,
): VomNode {
if (!capturedNode) return node;
const attrs = capturedNode.attrs;
const attrs =
node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value")
? Object.fromEntries(
Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"),
)
: capturedNode.attrs;
const text = cleanAttr(capturedNode.textContent);
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
@@ -823,8 +830,7 @@ function vomNodeFromCaptured(
signals: VomNodeDomSignals,
): VomNode {
const sensitive = isSensitive(capturedNode);
const value =
capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
const value = sensitive ? undefined : capturedNode.formValue;
const tag = normalizeTag(capturedNode.tag);
const node = applyCapturedSignals(
{
@@ -860,8 +866,17 @@ function axVomNode(
): VomNode {
const role = axString(axNode.role);
const capturedValue = capturedNode?.formValue;
const value = capturedValue ?? axValue(axNode.value);
const sourceValue = capturedValue ?? axValue(axNode.value);
const signalsForNode = axSignals.get(axNode.nodeId);
const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true;
const value = sensitive ? undefined : sourceValue;
const inputState =
inputStateFor(capturedNode, sourceValue) ??
(sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "")
? sourceValue === undefined || sourceValue === ""
? "empty"
: "filled"
: undefined);
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
@@ -877,8 +892,8 @@ function axVomNode(
position: capturedNode?.position || "static",
pointerEvents: capturedNode?.pointerEvents || "auto",
modal: isModalSignal(axNode, capturedNode),
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
inputState: inputStateFor(capturedNode, value),
sensitive,
inputState,
};
if (role) node.role = role;
if (name) node.name = name;
@@ -1141,6 +1156,8 @@ export interface BuildVomSceneOptions {
pageUrl?: string;
}
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function externalHrefHost(
href: string | undefined,
pageUrl: string | undefined,
@@ -1234,6 +1251,119 @@ export function buildVomScene(
};
}
export function buildFrameVomScene(
documents: VomFrameDocument[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const rootFrameId =
captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId;
const orderedFrameIds = [
...(rootFrameId ? [rootFrameId] : []),
...documents.map((document) => document.frameId),
].filter((frameId, index, all) => all.indexOf(frameId) === index);
const localScenes = new Map<string, VomScene>();
for (const frameId of orderedFrameIds) {
const document = documentByFrameId.get(frameId);
if (!document) continue;
localScenes.set(
frameId,
buildVomScene(
document.axNodes,
{
viewport: captured.viewport,
nodes: document.domNodes,
iframeNodes: new Map(),
surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [],
excludedBackendNodeIds:
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
},
{ pageUrl: options.pageUrl },
),
);
}
const sceneIdByFrameAndBackend = new Map<string, number>();
let nextSceneId = 1;
for (const [frameId, scene] of localScenes) {
for (const node of scene.nodes) {
sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId);
nextSceneId += 1;
}
}
const nodes: VomNode[] = [];
const surfaces: CondSurface[] = [];
const activeScopeBlocks: ActiveScopeBlock[] = [];
for (const [frameId, scene] of localScenes) {
const document = documentByFrameId.get(frameId);
const ownerBackendNodeId = document?.ownerBackendNodeId;
const parentFrameId = document?.parentFrameId;
const ownerSceneId =
ownerBackendNodeId !== undefined && parentFrameId
? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`)
: undefined;
if (frameId !== rootFrameId && ownerSceneId === undefined) continue;
const suppressedRoots = new Set(
frameId === rootFrameId
? []
: scene.nodes
.filter(
(node) =>
node.parentId === null &&
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
)
.map((node) => node.id),
);
for (const node of scene.nodes) {
if (suppressedRoots.has(node.id)) continue;
const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number;
let parentId =
node.parentId !== null
? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`)
: ownerSceneId;
if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId;
const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`);
nodes.push({
...node,
id: sceneNodeId,
backendNodeId: node.id,
frameId,
contextScopeId: document?.contextScopeId ?? frameId,
parentId: parentId ?? null,
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) }
: {}),
});
}
for (const surface of scene.surfaces ?? []) {
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`);
if (triggerId !== undefined) surfaces.push({ ...surface, triggerId });
}
for (const block of scene.activeScopeBlocks ?? []) {
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`);
if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId });
}
}
return {
viewport: captured.viewport,
nodes,
...(rootFrameId ? { rootFrameId } : {}),
...(surfaces.length > 0 ? { surfaces } : {}),
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
};
}
export interface SnapshotDeps {
cdp: CdpRunner;
tabsApi: {
@@ -1447,15 +1577,6 @@ async function handleVomObservation(
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
throwIfAborted(signal, toolName);
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
throwIfAborted(signal, toolName);
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
target.tabId,
"Accessibility.getFullAXTree",
{},
);
throwIfAborted(signal, toolName);
const axNodes = result.nodes ?? [];
const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom(
@@ -1466,7 +1587,17 @@ async function handleVomObservation(
signal,
);
throwIfAborted(signal, toolName);
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
const frameDocuments = await captureFrameData<CdpAxNode>(
deps.cdp,
target.tabId,
captured,
signal,
);
throwIfAborted(signal, toolName);
const scene =
captured.rootFrameId && captured.frameNodes?.size
? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
: buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
const rendered = renderVom(scene, {
maxDepth: params.max_depth,
maxTokens: params.max_tokens,
@@ -1475,7 +1606,7 @@ async function handleVomObservation(
throwIfAborted(signal, toolName);
ctx.refStore.replace(
rendered.refs.map(
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
(ref) => [ref.ref, { backendNodeId: ref.backendNodeId, tabId: target.tabId }] as const,
),
);
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
apps/extension/src/tools/vom/__tests__/capture.test.ts
+34 -22
View File
@@ -18,6 +18,8 @@ function fakeSnapshotReply() {
"auto",
"type",
"password",
"value",
"hunter2",
];
const i = (s: string) => S.indexOf(s);
return {
@@ -29,7 +31,8 @@ function fakeSnapshotReply() {
nodeType: [1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("div"), i("input")],
backendNodeId: [10, 11, 12, 13],
attributes: [[], [], [], [i("type"), i("password")]],
attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
inputValue: { index: [3], value: [i("hunter2")] },
},
layout: {
nodeIndex: [1, 2, 3],
@@ -217,7 +220,8 @@ function makeCdp(snapshot: unknown) {
{
state: "filled",
sensitive: true,
defaultValue: "",
defaultValue: "hunter2",
value: "hunter2",
placeholder: "secret",
},
],
@@ -262,10 +266,11 @@ describe("captureViewModel", () => {
expect(input).toMatchObject({
formState: "filled",
formDefaultValue: "",
formPlaceholder: "secret",
attrs: { type: "password" },
});
expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
@@ -692,10 +697,8 @@ describe("captureViewModel", () => {
};
const { nodes } = await captureViewModel(cdp, 4);
const div = nodes.find((n) => n.backendNodeId === 12);
// local rect preserves viewport-relative scroll math.
expect(div?.localRect?.y).toBe(-200);
// exported rect is clipped to the top-level viewport before VOM consumes it.
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
expect(div?.rect).toMatchObject({ y: -200, h: 800 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
@@ -839,6 +842,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "main-frame",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -866,6 +870,7 @@ describe("captureViewModel", () => {
},
// documents[1]: the iframe's sub-document with a login input
{
frameId: "child-frame",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -895,7 +900,14 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
const {
nodes,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds,
frameParentIds,
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13
@@ -907,10 +919,14 @@ describe("captureViewModel", () => {
expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(input?.rect).toBeNull();
expect(rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
});
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
it("captures iframe documents without requiring owner-frame geometry", async () => {
const S = [
"html",
"body",
@@ -938,16 +954,10 @@ describe("captureViewModel", () => {
contentDocumentIndex: { index: [2], value: [1] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[950, 760, 200, 100],
],
paintOrders: [0, 1],
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
bounds: [[0, 0, 1000, 800]],
paintOrders: [0],
},
},
{
@@ -985,7 +995,7 @@ describe("captureViewModel", () => {
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
expect(input?.rect).toBeNull();
});
it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1080,9 +1090,11 @@ describe("captureViewModel", () => {
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toBeNull();
expect(input?.ownerFrameBackendNodeId).toBe(23);
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
expect(input?.rect).toBeNull();
});
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
apps/extension/src/tools/vom/__tests__/frame-capture.test.ts
+63
View File
@@ -0,0 +1,63 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function node(frameId: string, backendNodeId: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "div",
attrs: {},
rect: null,
localRect: { x: 0, y: 0, w: 20, h: 20 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("captureFrameData", () => {
it("captures AX trees for every same-target frame found in the DOM snapshot", async () => {
const captured: CapturedViewModel = {
nodes: [node("main", 1)],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map([[10, [node("child", 101)]]]),
frameNodes: new Map([
["main", [node("main", 1)]],
["child", [node("child", 101)]],
]),
frameOwnerBackendNodeIds: new Map([["child", 10]]),
frameParentIds: new Map([["child", "main"]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") {
const frameId = (params as { frameId?: string })?.frameId ?? "main";
return {
nodes: [
{
nodeId: `${frameId}-root`,
frameId,
backendDOMNodeId: frameId === "main" ? 1 : 101,
},
],
};
}
throw new Error(`unexpected ${method}`);
});
const documents = await captureFrameData({ send } as CdpRunner, 4, captured);
expect(send).toHaveBeenCalledWith(4, "Accessibility.enable", {});
expect(documents.map((document) => document.frameId)).toEqual(["main", "child"]);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "main",
ownerBackendNodeId: 10,
axNodes: [{ nodeId: "child-root", frameId: "child", backendDOMNodeId: 101 }],
});
});
});
apps/extension/src/tools/vom/__tests__/frame-document.test.ts
+110
View File
@@ -0,0 +1,110 @@
import { describe, expect, it } from "vitest";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "div",
attrs: {},
rect: null,
localRect: { x: 0, y: 0, w: 20, h: 20 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
return {
nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes,
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
}
describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
{
frameId: "main",
nodes: [
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
{
nodeId: "h",
frameId: "child",
parentId: "root",
backendDOMNodeId: 100,
childIds: ["virtual"],
},
{ nodeId: "virtual", parentId: "h" },
],
},
];
const documents = buildFrameDocuments(
batches,
captured(
new Map([
["main", [domNode("main", 1)]],
["child", [domNode("child", 100)]],
]),
),
);
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
});
it("deduplicates overlapping AX results using the strongest ownership", () => {
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
{ frameId: "main", nodes: [{ ...childNode, frameId: "child" }] },
{ frameId: "child", nodes: [childNode] },
];
const documents = buildFrameDocuments(
batches,
captured(
new Map([
["main", [domNode("main", 1)]],
["child", [domNode("child", 100)]],
]),
),
);
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
});
it("uses captured frame parent identity instead of guessing from backend ids", () => {
const frameNodes = new Map([
["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]],
["right", [domNode("right", 8)]],
["child", [domNode("child", 9)]],
]);
const capture = captured(frameNodes);
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
capture.frameParentIds = new Map([["child", "left"]]);
const batches = ["main", "left", "right", "child"].map((frameId) => ({
frameId,
nodes: [],
}));
const documents = buildFrameDocuments(batches, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left",
ownerBackendNodeId: 7,
});
});
});
apps/extension/src/tools/vom/capture.ts
+125 -62
View File
@@ -17,13 +17,14 @@ const STYLE_COL = Object.fromEntries(
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
/** Top-level viewport geometry when this node belongs to the root document. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
/** Owning document viewport-relative CSS px. */
localRect?: Rect | null;
paintOrder: number;
position: string;
@@ -57,6 +58,11 @@ export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
surfaceProbes?: CapturedSurfaceProbe[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
@@ -92,7 +98,12 @@ interface SparseArray {
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
@@ -107,6 +118,10 @@ interface SnapshotDocument {
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
@@ -116,6 +131,12 @@ interface SnapshotDocument {
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
@@ -135,6 +156,25 @@ function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
@@ -166,9 +206,8 @@ function formStateBatchExpression(maxControls: number): string {
return {
state,
sensitive,
defaultValue,
placeholder,
...(sensitive ? {} : { value: rawValue }),
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const collect = (doc) => {
@@ -210,10 +249,14 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
index += 1;
if (!state) continue;
node.formState = state.state;
node.formDefaultValue = state.defaultValue ?? "";
node.formPlaceholder = state.placeholder ?? "";
if (!state.sensitive && state.value !== undefined) {
node.formValue = state.value;
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
@@ -272,9 +315,8 @@ interface ParseDocumentResult {
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
originInTop: { x: number; y: number };
clipRectInTop: Rect;
scrollX: number;
scrollY: number;
}
@@ -293,38 +335,6 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
return dpr >= 1 ? dpr : 1;
}
function viewportRect(viewport: Viewport): Rect {
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
}
function intersectRects(a: Rect, b: Rect): Rect | null {
const x1 = Math.max(a.x, b.x);
const y1 = Math.max(a.y, b.y);
const x2 = Math.min(a.x + a.w, b.x + b.w);
const y2 = Math.min(a.y + a.h, b.y + b.h);
if (x2 <= x1 || y2 <= y1) return null;
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
const translated = {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
return intersectRects(translated, context.clipRectInTop);
}
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
return {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
@@ -805,6 +815,10 @@ function parseDocumentNodes(
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
@@ -855,7 +869,7 @@ function parseDocumentNodes(
w: b[2] / dpr,
h: b[3] / dpr,
};
rect = rectInTop(localRect, context);
rect = context.ownerFrameBackendNodeId === null ? localRect : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
@@ -870,9 +884,24 @@ function parseDocumentNodes(
const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
@@ -883,6 +912,16 @@ function parseDocumentNodes(
pointerEvents,
cursor,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes, excludedBackendNodeIds };
@@ -890,6 +929,8 @@ function parseDocumentNodes(
interface ParseFrameDocumentsResult {
iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
excludedBackendNodeIds: Set<number>;
}
@@ -898,18 +939,20 @@ function parseChildFrameDocuments(
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
if (cdi.size === 0) {
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
@@ -917,20 +960,10 @@ function parseChildFrameDocuments(
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
if (!iframeNode?.localRect) continue;
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
if (!childClip) {
iframeNodes.set(iframeBackendId, []);
continue;
}
const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings),
ownerFrameBackendNodeId: iframeBackendId,
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
clipRectInTop: childClip,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
@@ -938,6 +971,10 @@ function parseChildFrameDocuments(
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments(
@@ -945,17 +982,22 @@ function parseChildFrameDocuments(
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
nextVisited,
);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
}
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return { iframeNodes, excludedBackendNodeIds };
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
export async function captureViewModel(
@@ -998,9 +1040,8 @@ export async function captureViewModel(
}
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
originInTop: { x: 0, y: 0 },
clipRectInTop: viewportRect(viewport),
scrollX,
scrollY,
};
@@ -1008,7 +1049,7 @@ export async function captureViewModel(
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
@@ -1022,5 +1063,27 @@ export async function captureViewModel(
: [];
throwIfAborted(options.signal);
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
}
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
if (!frameNodes.has(frameId)) {
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
}
}
return {
nodes,
viewport,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
surfaceProbes,
excludedBackendNodeIds,
};
}
apps/extension/src/tools/vom/frame-capture.ts
+75
View File
@@ -0,0 +1,75 @@
import type { CdpRunner } from "../shared";
import type { CapturedViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
function capturedFrameIds(captured: CapturedViewModel): string[] {
const frameIds = new Set<string>();
if (captured.rootFrameId) frameIds.add(captured.rootFrameId);
for (const frameId of captured.frameNodes?.keys() ?? []) frameIds.add(frameId);
if (frameIds.size === 0) frameIds.add("root");
return [...frameIds];
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
const frameIds = capturedFrameIds(captured);
let firstFailure: unknown;
await cdp.send(tabId, "Accessibility.enable", {});
const trees = await Promise.all(
frameIds.map(async (frameId): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
const result = await cdp.send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frameId === "root" ? {} : { frameId },
);
throwIfAborted(signal);
return { frameId, nodes: result.nodes ?? [] };
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", { frameId, err });
return { frameId, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<CapturedFrameDocument<T>[]> {
const batches = await captureAxTrees<T>(cdp, tabId, captured, signal);
return buildFrameDocuments(batches, captured);
}
apps/extension/src/tools/vom/frame-document.ts
+142
View File
@@ -0,0 +1,142 @@
import type { CapturedNode, CapturedViewModel } from "./capture";
export interface FrameOwnedAxNode {
nodeId: string;
frameId?: string;
parentId?: string;
childIds?: string[];
backendDOMNodeId?: number;
}
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
frameId: string;
nodes: T[];
}
export interface FrameDocument<T extends FrameOwnedAxNode> {
frameId: string;
parentFrameId?: string;
ownerBackendNodeId?: number;
contextScopeId: string;
axNodes: T[];
domNodes: CapturedNode[];
}
interface Ownership {
frameId: string;
strength: number;
}
interface OwnedCandidate<T> {
node: T;
ownership: Ownership;
}
function frameIds<T extends FrameOwnedAxNode>(
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): string[] {
const ids = new Set<string>();
if (captured.rootFrameId) ids.add(captured.rootFrameId);
for (const frameId of captured.frameNodes?.keys() ?? []) ids.add(frameId);
for (const batch of batches) ids.add(batch.frameId);
return [...ids];
}
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): FrameDocument<T>[] {
const frames = frameIds(batches, captured);
const knownFrames = new Set(frames);
const backendOwner = new Map<number, string>();
for (const frameId of frames) {
for (const node of captured.frameNodes?.get(frameId) ?? []) {
backendOwner.set(node.backendNodeId, frameId);
}
}
const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>();
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && knownFrames.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const owner = backendOwner.get(node.backendDOMNodeId);
if (owner) {
const ownership = { frameId: owner, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
if (node.parentId && !resolving.has(node.nodeId)) {
const parent = nodeById.get(node.parentId);
if (parent) {
resolving.add(node.nodeId);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
const ownership = { frameId: batch.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
for (const node of batch.nodes) {
const ownership = resolveOwnership(node);
const existing = candidates.get(node.nodeId);
if (!existing || ownership.strength > existing.ownership.strength) {
candidates.set(node.nodeId, { node, ownership });
}
}
}
const ownershipByNodeId = new Map(
[...candidates].map(([nodeId, candidate]) => [nodeId, candidate.ownership.frameId]),
);
const axNodesByFrame = new Map<string, T[]>();
for (const { node, ownership } of candidates.values()) {
const parentFrameId = node.parentId ? ownershipByNodeId.get(node.parentId) : undefined;
const childIds = node.childIds?.filter(
(childId) => ownershipByNodeId.get(childId) === ownership.frameId,
);
const ownedNode = {
...node,
frameId: ownership.frameId,
...(parentFrameId === ownership.frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(childIds ? { childIds } : {}),
};
const nodes = axNodesByFrame.get(ownership.frameId) ?? [];
nodes.push(ownedNode);
axNodesByFrame.set(ownership.frameId, nodes);
}
return frames.map((frameId) => ({
frameId,
...(captured.frameParentIds?.get(frameId)
? { parentFrameId: captured.frameParentIds.get(frameId) }
: {}),
...(captured.frameOwnerBackendNodeIds?.get(frameId) !== undefined
? { ownerBackendNodeId: captured.frameOwnerBackendNodeIds.get(frameId) }
: {}),
contextScopeId: frameId,
axNodes: axNodesByFrame.get(frameId) ?? [],
domNodes: captured.frameNodes?.get(frameId) ?? [],
}));
}
packages/vom/src/__tests__/render.test.ts
+196 -1
View File
@@ -25,6 +25,85 @@ function scene(nodes: VomNode[]): VomScene {
}
describe("renderVom single-layer page", () => {
it("does not derive handle context across frame scopes", () => {
const out = renderVom(
scene([
node({
id: 1,
role: "RootWebArea",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 2,
parentId: 1,
role: "StaticText",
name: "创建于",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 3,
parentId: 1,
role: "Iframe",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 4,
parentId: 3,
role: "button",
name: "表格视图",
frameId: "child",
contextScopeId: "child",
}),
node({
id: 5,
parentId: 1,
role: "button",
name: "表格视图",
frameId: "main",
contextScopeId: "main",
}),
]),
);
expect(out.text).toContain('@e1 button "表格视图"');
expect(out.text).not.toContain('@e1 button "表格视图" [ctx: 创建于]');
});
it("preserves handle context within a frame scope", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", contextScopeId: "main" }),
node({ id: 2, parentId: 1, role: "Iframe", contextScopeId: "main" }),
node({
id: 3,
parentId: 2,
role: "StaticText",
name: "工具栏",
contextScopeId: "child",
}),
node({
id: 4,
parentId: 2,
role: "button",
name: "更多",
contextScopeId: "child",
}),
node({
id: 5,
parentId: 1,
role: "button",
name: "更多",
contextScopeId: "main",
}),
]),
);
expect(out.text).toContain('@e1 button "更多" [ctx: 工具栏]');
});
it("always emits a complete @vom single-layer document when there is no blocker", () => {
const out = renderVom(
scene([
@@ -157,7 +236,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("hunter2");
});
it("does not imply a sensitive textbox has a value when the value is missing", () => {
it("renders a sensitive filled state without retaining its clear text value", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Login" }),
@@ -168,6 +247,26 @@ describe("renderVom single-layer page", () => {
name: "密码",
tag: "input",
sensitive: true,
inputState: "filled",
}),
]),
);
expect(out.text).toContain('@e1 textbox "密码" [filled] ="•••"');
});
it("does not imply an empty sensitive textbox has a value", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Login" }),
node({
id: 2,
parentId: 1,
role: "textbox",
name: "密码",
tag: "input",
sensitive: true,
inputState: "empty",
}),
]),
);
@@ -389,6 +488,102 @@ describe("renderVom single-layer page", () => {
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("keeps paint-order comparisons inside their frame document", () => {
const out = renderVom(
{
...scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({
id: 2,
parentId: 1,
role: "button",
name: "Top-level action",
tag: "button",
rect: { x: 120, y: 120, w: 120, h: 40 },
paintOrder: 2,
frameId: "main",
}),
node({
id: 3,
parentId: 1,
role: "Iframe",
tag: "iframe",
rect: { x: 0, y: 0, w: 1000, h: 700 },
paintOrder: 1,
frameId: "main",
}),
node({
id: 4,
parentId: 3,
role: "generic",
rect: { x: 0, y: 0, w: 1000, h: 700 },
paintOrder: 100,
position: "fixed",
frameId: "child",
}),
node({
id: 5,
parentId: 4,
role: "button",
name: "Child action",
tag: "button",
rect: { x: 120, y: 120, w: 120, h: 40 },
paintOrder: 101,
frameId: "child",
}),
]),
rootFrameId: "main",
},
{ activeRegionPolicy: true },
);
expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('button "Top-level action"');
expect(out.text).toContain('button "Child action"');
});
it("uses the iframe owner when a parent-frame region covers child content", () => {
const out = renderVom(
{
...scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({
id: 2,
parentId: 1,
role: "Iframe",
tag: "iframe",
rect: { x: 100, y: 100, w: 400, h: 300 },
paintOrder: 1,
frameId: "main",
}),
node({
id: 3,
parentId: 2,
role: "button",
name: "Covered child action",
tag: "button",
rect: { x: 150, y: 150, w: 120, h: 40 },
paintOrder: 100,
frameId: "child",
}),
node({
id: 4,
parentId: 1,
role: "generic",
rect: { x: 120, y: 120, w: 200, h: 120 },
paintOrder: 2,
position: "fixed",
frameId: "main",
}),
]),
rootFrameId: "main",
},
{ activeRegionPolicy: true },
);
expect(out.text).not.toContain("Covered child action");
});
it("renders conditional surface items inline on the trigger", () => {
const out = renderVom({
...scene([
packages/vom/src/index.ts
+1
View File
@@ -8,6 +8,7 @@ export type {
Viewport,
VomNode,
VomOptions,
VomRef,
VomResult,
VomScene,
} from "./types";
packages/vom/src/layers.ts
+12 -2
View File
@@ -50,10 +50,18 @@ function classifyLayer(
return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal";
}
export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLayer | null {
export function detectBlockingLayer(
nodes: VomNode[],
vp: Viewport,
rootFrameId?: string,
): BlockingLayer | null {
let blocker: { node: VomNode; coverage: number } | null = null;
for (const node of nodes) {
// A document's paint order is local to its iframe stacking context. Only
// the root document can establish a page-level blocking layer; child
// documents are constrained by their iframe owner in the parent document.
if (rootFrameId !== undefined && node.frameId !== rootFrameId) continue;
if (!isBlockingCandidate(node)) continue;
const cov = coverage(node.rect, vp);
const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15);
@@ -72,7 +80,9 @@ export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLay
const threshold = blocker.node.paintOrder;
const members = new Set<number>();
for (const node of nodes) {
if (node.paintOrder >= threshold) members.add(node.id);
if (node.frameId === blocker.node.frameId && node.paintOrder >= threshold) {
members.add(node.id);
}
}
return {
packages/vom/src/render.ts
+73 -11
View File
@@ -6,6 +6,7 @@ import type {
Rect,
VomNode,
VomOptions,
VomRef,
VomResult,
VomScene,
} from "./types";
@@ -503,12 +504,18 @@ function isContextBoundary(node: VomNode): boolean {
);
}
function sharesContextScope(node: VomNode, candidate: VomNode): boolean {
return node.contextScopeId === candidate.contextScopeId;
}
function collectWeakLabelsFromSubtree(
node: VomNode,
target: VomNode,
nodeName: string,
state: RenderState,
labels: string[],
): void {
if (!sharesContextScope(target, node)) return;
if (isVomReferenceNode(node)) return;
const label = weakLabelContext(node, nodeName);
if (label) {
@@ -516,7 +523,7 @@ function collectWeakLabelsFromSubtree(
return;
}
for (const child of state.children.get(node.id) ?? []) {
collectWeakLabelsFromSubtree(child, nodeName, state, labels);
collectWeakLabelsFromSubtree(child, target, nodeName, state, labels);
if (labels.length >= MAX_HANDLE_CONTEXT_ITEMS) return;
}
}
@@ -532,6 +539,7 @@ function collectOwnedContext(
while (current !== null && guard <= state.parentMap.size) {
const parent = state.nodesById.get(current);
if (!parent) break;
if (!sharesContextScope(node, parent)) break;
const label = contextLabel(parent, nodeName);
if (label) contexts.push({ text: label, source: "owned", order: contexts.length });
current = state.parentMap.get(current) ?? null;
@@ -553,12 +561,12 @@ function collectSameContainerContext(
const siblings = state.children.get(parentId) ?? [];
for (const sibling of siblings) {
if (sibling.id === childId) break;
collectWeakLabelsFromSubtree(sibling, nodeName, state, labels);
collectWeakLabelsFromSubtree(sibling, node, nodeName, state, labels);
while (labels.length > MAX_HANDLE_CONTEXT_ITEMS) labels.shift();
}
const parent = state.nodesById.get(parentId);
if (!parent || isContextBoundary(parent)) break;
if (!parent || !sharesContextScope(node, parent) || isContextBoundary(parent)) break;
childId = parentId;
parentId = state.parentMap.get(parentId) ?? null;
guard += 1;
@@ -578,6 +586,7 @@ function collectDomContext(
for (const candidate of state.nodesById.values()) {
if (candidate.id === node.id) continue;
if (!sharesContextScope(node, candidate)) continue;
if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue;
const parentId = candidate.domParentId;
if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue;
@@ -627,8 +636,10 @@ function renderNodeLine(
}
const placeholder = cleaned(node.placeholder);
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
const sensitiveHasValue =
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
const value = node.sensitive
? rawValue !== undefined
? sensitiveHasValue
? SENSITIVE_MASK
: undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
@@ -684,7 +695,7 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
interface RenderState {
lines: string[];
refs: Array<{ ref: string; backendNodeId: number }>;
refs: VomRef[];
nextRef: number;
tokens: number;
maxDepth: number;
@@ -764,7 +775,11 @@ function renderTree(
state.lines.push(line);
state.tokens = nextTokens;
if (ref) {
state.refs.push({ ref, backendNodeId: node.id });
state.refs.push({
ref,
backendNodeId: node.backendNodeId ?? node.id,
...(node.frameId ? { frameId: node.frameId } : {}),
});
state.nextRef += 1;
}
@@ -855,29 +870,75 @@ function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number):
return 1;
}
function paintLineageByFrame(
node: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
): Map<string | undefined, VomNode> {
const lineage = new Map<string | undefined, VomNode>();
let current: VomNode | undefined = node;
let guard = 0;
while (current && guard <= parentMap.size) {
if (!lineage.has(current.frameId)) lineage.set(current.frameId, current);
const parentId: number | null = parentMap.get(current.id) ?? null;
current = parentId === null ? undefined : nodesById.get(parentId);
guard += 1;
}
return lineage;
}
/** Resolve both nodes to the nearest document where their paint order is comparable. */
function comparablePaintNodes(
target: VomNode,
blocker: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
): { target: VomNode; blocker: VomNode } | null {
if (target.frameId === blocker.frameId) return { target, blocker };
const targetLineage = paintLineageByFrame(target, parentMap, nodesById);
let blockerInFrame: VomNode | undefined = blocker;
let guard = 0;
while (blockerInFrame && guard <= parentMap.size) {
const targetInFrame = targetLineage.get(blockerInFrame.frameId);
if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame };
const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null;
blockerInFrame = parentId === null ? undefined : nodesById.get(parentId);
guard += 1;
}
return null;
}
function isBlockedByRegion(
target: VomNode,
blocker: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
viewportCoverage: number,
): boolean {
if (target.id === blocker.id) return false;
if (isAncestorOf(parentMap, blocker.id, target.id)) return false;
if (isAncestorOf(parentMap, target.id, blocker.id)) return false;
if (!target.rect || !blocker.rect) return false;
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById);
if (!paintNodes) return false;
const points = interactionPoints(target.rect);
if (
viewportCoverage < 0.9 &&
target.paintOrder >= blocker.paintOrder &&
paintNodes.target.paintOrder >= paintNodes.blocker.paintOrder &&
points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y))
) {
return false;
}
if (blocker.paintOrder < target.paintOrder) return false;
if (blocker.paintOrder === target.paintOrder && blocker.id <= target.id) return false;
if (paintNodes.blocker.paintOrder < paintNodes.target.paintOrder) return false;
if (
paintNodes.blocker.paintOrder === paintNodes.target.paintOrder &&
paintNodes.blocker.id <= paintNodes.target.id
) {
return false;
}
if (viewportCoverage >= 0.9) return true;
return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y));
@@ -901,6 +962,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<numb
function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
const parentMap = buildParentMap(nodes);
const nodesById = new Map(nodes.map((node) => [node.id, node]));
const candidates = nodes
.filter(isPositionedRegionCandidate)
.map((node) => {
@@ -917,7 +979,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
for (const target of nodes) {
if (!isVomReferenceNode(target)) continue;
const blocked = candidates.some((candidate) =>
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage),
);
if (blocked) blockedRoots.add(target.id);
}
@@ -972,7 +1034,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
const nodes = applyVomInteractionRecovery(scene.nodes);
const renderScene = { ...scene, nodes };
const layer = detectBlockingLayer(nodes, scene.viewport);
const layer = detectBlockingLayer(nodes, scene.viewport, scene.rootFrameId);
if (layer) return renderDoubleLayer(renderScene, layer, options);
const visibleNodes = options.activeRegionPolicy ? applyActiveRegionPolicy(nodes, scene) : nodes;
packages/vom/src/types.ts
+12 -1
View File
@@ -21,6 +21,9 @@ export type LayerKind = "page" | "modal" | "mask";
export interface VomNode {
id: number;
parentId: number | null;
backendNodeId?: number;
frameId?: string;
contextScopeId?: string;
role?: string;
name?: string;
@@ -52,6 +55,8 @@ export interface VomNode {
export interface VomScene {
viewport: Viewport;
nodes: VomNode[];
/** Root document whose paint order defines page-level blocking layers. */
rootFrameId?: string;
surfaces?: CondSurface[];
activeScopeBlocks?: ActiveScopeBlock[];
}
@@ -79,9 +84,15 @@ export interface ActiveScopeBlock {
lines: string[];
}
export interface VomRef {
ref: string;
backendNodeId: number;
frameId?: string;
}
export interface VomResult {
text: string;
refs: Array<{ ref: string; backendNodeId: number }>;
refs: VomRef[];
truncated: boolean;
}