figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(extension): support iframe content in VOM observations

Browse Source
This commit is contained in:
Ljy-0827
2026-08-18 19:18:39 +08:00
committed by GitHub
parent a9489282fa
commit 8d7c0aa66c
13 changed files with 1276 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/vom/src/__tests__/render.test.ts
+196 -1
View File
@@ -25,6 +25,85 @@ function scene(nodes: VomNode[]): VomScene {
}
describe("renderVom single-layer page", () => {
it("does not derive handle context across frame scopes", () => {
const out = renderVom(
scene([
node({
id: 1,
role: "RootWebArea",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 2,
parentId: 1,
role: "StaticText",
name: "创建于",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 3,
parentId: 1,
role: "Iframe",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 4,
parentId: 3,
role: "button",
name: "表格视图",
frameId: "child",
contextScopeId: "child",
}),
node({
id: 5,
parentId: 1,
role: "button",
name: "表格视图",
frameId: "main",
contextScopeId: "main",
}),
]),
);
expect(out.text).toContain('@e1 button "表格视图"');
expect(out.text).not.toContain('@e1 button "表格视图" [ctx: 创建于]');
});
it("preserves handle context within a frame scope", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", contextScopeId: "main" }),
node({ id: 2, parentId: 1, role: "Iframe", contextScopeId: "main" }),
node({
id: 3,
parentId: 2,
role: "StaticText",
name: "工具栏",
contextScopeId: "child",
}),
node({
id: 4,
parentId: 2,
role: "button",
name: "更多",
contextScopeId: "child",
}),
node({
id: 5,
parentId: 1,
role: "button",
name: "更多",
contextScopeId: "main",
}),
]),
);
expect(out.text).toContain('@e1 button "更多" [ctx: 工具栏]');
});
it("always emits a complete @vom single-layer document when there is no blocker", () => {
const out = renderVom(
scene([
@@ -157,7 +236,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("hunter2");
});
it("does not imply a sensitive textbox has a value when the value is missing", () => {
it("renders a sensitive filled state without retaining its clear text value", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Login" }),
@@ -168,6 +247,26 @@ describe("renderVom single-layer page", () => {
name: "密码",
tag: "input",
sensitive: true,
inputState: "filled",
}),
]),
);
expect(out.text).toContain('@e1 textbox "密码" [filled] ="•••"');
});
it("does not imply an empty sensitive textbox has a value", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Login" }),
node({
id: 2,
parentId: 1,
role: "textbox",
name: "密码",
tag: "input",
sensitive: true,
inputState: "empty",
}),
]),
);
@@ -389,6 +488,102 @@ describe("renderVom single-layer page", () => {
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("keeps paint-order comparisons inside their frame document", () => {
const out = renderVom(
{
...scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({
id: 2,
parentId: 1,
role: "button",
name: "Top-level action",
tag: "button",
rect: { x: 120, y: 120, w: 120, h: 40 },
paintOrder: 2,
frameId: "main",
}),
node({
id: 3,
parentId: 1,
role: "Iframe",
tag: "iframe",
rect: { x: 0, y: 0, w: 1000, h: 700 },
paintOrder: 1,
frameId: "main",
}),
node({
id: 4,
parentId: 3,
role: "generic",
rect: { x: 0, y: 0, w: 1000, h: 700 },
paintOrder: 100,
position: "fixed",
frameId: "child",
}),
node({
id: 5,
parentId: 4,
role: "button",
name: "Child action",
tag: "button",
rect: { x: 120, y: 120, w: 120, h: 40 },
paintOrder: 101,
frameId: "child",
}),
]),
rootFrameId: "main",
},
{ activeRegionPolicy: true },
);
expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('button "Top-level action"');
expect(out.text).toContain('button "Child action"');
});
it("uses the iframe owner when a parent-frame region covers child content", () => {
const out = renderVom(
{
...scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({
id: 2,
parentId: 1,
role: "Iframe",
tag: "iframe",
rect: { x: 100, y: 100, w: 400, h: 300 },
paintOrder: 1,
frameId: "main",
}),
node({
id: 3,
parentId: 2,
role: "button",
name: "Covered child action",
tag: "button",
rect: { x: 150, y: 150, w: 120, h: 40 },
paintOrder: 100,
frameId: "child",
}),
node({
id: 4,
parentId: 1,
role: "generic",
rect: { x: 120, y: 120, w: 200, h: 120 },
paintOrder: 2,
position: "fixed",
frameId: "main",
}),
]),
rootFrameId: "main",
},
{ activeRegionPolicy: true },
);
expect(out.text).not.toContain("Covered child action");
});
it("renders conditional surface items inline on the trigger", () => {
const out = renderVom({
...scene([
packages/vom/src/index.ts
+1
View File
@@ -8,6 +8,7 @@ export type {
Viewport,
VomNode,
VomOptions,
VomRef,
VomResult,
VomScene,
} from "./types";
packages/vom/src/layers.ts
+12 -2
View File
@@ -50,10 +50,18 @@ function classifyLayer(
return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal";
}
export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLayer | null {
export function detectBlockingLayer(
nodes: VomNode[],
vp: Viewport,
rootFrameId?: string,
): BlockingLayer | null {
let blocker: { node: VomNode; coverage: number } | null = null;
for (const node of nodes) {
// A document's paint order is local to its iframe stacking context. Only
// the root document can establish a page-level blocking layer; child
// documents are constrained by their iframe owner in the parent document.
if (rootFrameId !== undefined && node.frameId !== rootFrameId) continue;
if (!isBlockingCandidate(node)) continue;
const cov = coverage(node.rect, vp);
const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15);
@@ -72,7 +80,9 @@ export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLay
const threshold = blocker.node.paintOrder;
const members = new Set<number>();
for (const node of nodes) {
if (node.paintOrder >= threshold) members.add(node.id);
if (node.frameId === blocker.node.frameId && node.paintOrder >= threshold) {
members.add(node.id);
}
}
return {
packages/vom/src/render.ts
+73 -11
View File
@@ -6,6 +6,7 @@ import type {
Rect,
VomNode,
VomOptions,
VomRef,
VomResult,
VomScene,
} from "./types";
@@ -503,12 +504,18 @@ function isContextBoundary(node: VomNode): boolean {
);
}
function sharesContextScope(node: VomNode, candidate: VomNode): boolean {
return node.contextScopeId === candidate.contextScopeId;
}
function collectWeakLabelsFromSubtree(
node: VomNode,
target: VomNode,
nodeName: string,
state: RenderState,
labels: string[],
): void {
if (!sharesContextScope(target, node)) return;
if (isVomReferenceNode(node)) return;
const label = weakLabelContext(node, nodeName);
if (label) {
@@ -516,7 +523,7 @@ function collectWeakLabelsFromSubtree(
return;
}
for (const child of state.children.get(node.id) ?? []) {
collectWeakLabelsFromSubtree(child, nodeName, state, labels);
collectWeakLabelsFromSubtree(child, target, nodeName, state, labels);
if (labels.length >= MAX_HANDLE_CONTEXT_ITEMS) return;
}
}
@@ -532,6 +539,7 @@ function collectOwnedContext(
while (current !== null && guard <= state.parentMap.size) {
const parent = state.nodesById.get(current);
if (!parent) break;
if (!sharesContextScope(node, parent)) break;
const label = contextLabel(parent, nodeName);
if (label) contexts.push({ text: label, source: "owned", order: contexts.length });
current = state.parentMap.get(current) ?? null;
@@ -553,12 +561,12 @@ function collectSameContainerContext(
const siblings = state.children.get(parentId) ?? [];
for (const sibling of siblings) {
if (sibling.id === childId) break;
collectWeakLabelsFromSubtree(sibling, nodeName, state, labels);
collectWeakLabelsFromSubtree(sibling, node, nodeName, state, labels);
while (labels.length > MAX_HANDLE_CONTEXT_ITEMS) labels.shift();
}
const parent = state.nodesById.get(parentId);
if (!parent || isContextBoundary(parent)) break;
if (!parent || !sharesContextScope(node, parent) || isContextBoundary(parent)) break;
childId = parentId;
parentId = state.parentMap.get(parentId) ?? null;
guard += 1;
@@ -578,6 +586,7 @@ function collectDomContext(
for (const candidate of state.nodesById.values()) {
if (candidate.id === node.id) continue;
if (!sharesContextScope(node, candidate)) continue;
if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue;
const parentId = candidate.domParentId;
if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue;
@@ -627,8 +636,10 @@ function renderNodeLine(
}
const placeholder = cleaned(node.placeholder);
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
const sensitiveHasValue =
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
const value = node.sensitive
? rawValue !== undefined
? sensitiveHasValue
? SENSITIVE_MASK
: undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
@@ -684,7 +695,7 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
interface RenderState {
lines: string[];
refs: Array<{ ref: string; backendNodeId: number }>;
refs: VomRef[];
nextRef: number;
tokens: number;
maxDepth: number;
@@ -764,7 +775,11 @@ function renderTree(
state.lines.push(line);
state.tokens = nextTokens;
if (ref) {
state.refs.push({ ref, backendNodeId: node.id });
state.refs.push({
ref,
backendNodeId: node.backendNodeId ?? node.id,
...(node.frameId ? { frameId: node.frameId } : {}),
});
state.nextRef += 1;
}
@@ -855,29 +870,75 @@ function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number):
return 1;
}
function paintLineageByFrame(
node: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
): Map<string | undefined, VomNode> {
const lineage = new Map<string | undefined, VomNode>();
let current: VomNode | undefined = node;
let guard = 0;
while (current && guard <= parentMap.size) {
if (!lineage.has(current.frameId)) lineage.set(current.frameId, current);
const parentId: number | null = parentMap.get(current.id) ?? null;
current = parentId === null ? undefined : nodesById.get(parentId);
guard += 1;
}
return lineage;
}
/** Resolve both nodes to the nearest document where their paint order is comparable. */
function comparablePaintNodes(
target: VomNode,
blocker: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
): { target: VomNode; blocker: VomNode } | null {
if (target.frameId === blocker.frameId) return { target, blocker };
const targetLineage = paintLineageByFrame(target, parentMap, nodesById);
let blockerInFrame: VomNode | undefined = blocker;
let guard = 0;
while (blockerInFrame && guard <= parentMap.size) {
const targetInFrame = targetLineage.get(blockerInFrame.frameId);
if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame };
const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null;
blockerInFrame = parentId === null ? undefined : nodesById.get(parentId);
guard += 1;
}
return null;
}
function isBlockedByRegion(
target: VomNode,
blocker: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
viewportCoverage: number,
): boolean {
if (target.id === blocker.id) return false;
if (isAncestorOf(parentMap, blocker.id, target.id)) return false;
if (isAncestorOf(parentMap, target.id, blocker.id)) return false;
if (!target.rect || !blocker.rect) return false;
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById);
if (!paintNodes) return false;
const points = interactionPoints(target.rect);
if (
viewportCoverage < 0.9 &&
target.paintOrder >= blocker.paintOrder &&
paintNodes.target.paintOrder >= paintNodes.blocker.paintOrder &&
points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y))
) {
return false;
}
if (blocker.paintOrder < target.paintOrder) return false;
if (blocker.paintOrder === target.paintOrder && blocker.id <= target.id) return false;
if (paintNodes.blocker.paintOrder < paintNodes.target.paintOrder) return false;
if (
paintNodes.blocker.paintOrder === paintNodes.target.paintOrder &&
paintNodes.blocker.id <= paintNodes.target.id
) {
return false;
}
if (viewportCoverage >= 0.9) return true;
return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y));
@@ -901,6 +962,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<numb
function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
const parentMap = buildParentMap(nodes);
const nodesById = new Map(nodes.map((node) => [node.id, node]));
const candidates = nodes
.filter(isPositionedRegionCandidate)
.map((node) => {
@@ -917,7 +979,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
for (const target of nodes) {
if (!isVomReferenceNode(target)) continue;
const blocked = candidates.some((candidate) =>
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage),
);
if (blocked) blockedRoots.add(target.id);
}
@@ -972,7 +1034,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
const nodes = applyVomInteractionRecovery(scene.nodes);
const renderScene = { ...scene, nodes };
const layer = detectBlockingLayer(nodes, scene.viewport);
const layer = detectBlockingLayer(nodes, scene.viewport, scene.rootFrameId);
if (layer) return renderDoubleLayer(renderScene, layer, options);
const visibleNodes = options.activeRegionPolicy ? applyActiveRegionPolicy(nodes, scene) : nodes;
packages/vom/src/types.ts
+12 -1
View File
@@ -21,6 +21,9 @@ export type LayerKind = "page" | "modal" | "mask";
export interface VomNode {
id: number;
parentId: number | null;
backendNodeId?: number;
frameId?: string;
contextScopeId?: string;
role?: string;
name?: string;
@@ -52,6 +55,8 @@ export interface VomNode {
export interface VomScene {
viewport: Viewport;
nodes: VomNode[];
/** Root document whose paint order defines page-level blocking layers. */
rootFrameId?: string;
surfaces?: CondSurface[];
activeScopeBlocks?: ActiveScopeBlock[];
}
@@ -79,9 +84,15 @@ export interface ActiveScopeBlock {
lines: string[];
}
export interface VomRef {
ref: string;
backendNodeId: number;
frameId?: string;
}
export interface VomResult {
text: string;
refs: Array<{ ref: string; backendNodeId: number }>;
refs: VomRef[];
truncated: boolean;
}