mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): canvas recognition
This commit is contained in:
@@ -29,7 +29,7 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("renders visual surfaces as screenshot-only refs and low-confidence counts as summaries", () => {
|
||||
it("renders visual surfaces as screenshot-only refs", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
|
||||
@@ -40,19 +40,15 @@ describe("renderVom single-layer page", () => {
|
||||
backendNodeId: 99,
|
||||
frameId: "child",
|
||||
renderingKind: "canvas",
|
||||
rect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
label: "Data grid",
|
||||
memberCount: 2,
|
||||
},
|
||||
];
|
||||
input.visualSurfaceSummaries = [{ parentId: 2, frameId: "child", count: 3 }];
|
||||
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.text).toContain("[3 additional visual surfaces are not represented]");
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
|
||||
@@ -19,5 +19,4 @@ export type {
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
VomVisualSurfaceSummary,
|
||||
} from "./types";
|
||||
|
||||
@@ -10,7 +10,6 @@ import type {
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
VomVisualSurfaceSummary,
|
||||
} from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
@@ -796,7 +795,6 @@ interface RenderState {
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
visualSurfaces: Map<number | null, VomVisualSurface[]>;
|
||||
visualSurfaceSummaries: Map<number | null, VomVisualSurfaceSummary[]>;
|
||||
}
|
||||
|
||||
function groupedByParent<T extends { parentId: number | null }>(
|
||||
@@ -842,23 +840,6 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render
|
||||
});
|
||||
state.nextRef += 1;
|
||||
}
|
||||
|
||||
for (const summary of state.visualSurfaceSummaries.get(parentId) ?? []) {
|
||||
if (depth > state.maxDepth) {
|
||||
state.truncated = true;
|
||||
continue;
|
||||
}
|
||||
const noun = summary.count === 1 ? "surface is" : "surfaces are";
|
||||
const line = `${" ".repeat(depth)}[${summary.count} additional visual ${noun} not represented]`;
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
}
|
||||
}
|
||||
|
||||
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
||||
@@ -965,7 +946,6 @@ function renderNodes(
|
||||
surfaces: CondSurface[] = [],
|
||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||
visualSurfaces: VomVisualSurface[] = [],
|
||||
visualSurfaceSummaries: VomVisualSurfaceSummary[] = [],
|
||||
): RenderState {
|
||||
const children = buildChildren(nodes);
|
||||
const state: RenderState = {
|
||||
@@ -986,7 +966,6 @@ function renderNodes(
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
visualSurfaces: groupedByParent(visualSurfaces),
|
||||
visualSurfaceSummaries: groupedByParent(visualSurfaceSummaries),
|
||||
};
|
||||
|
||||
renderTree(children, state);
|
||||
@@ -1207,7 +1186,6 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||
scene.visualSurfaceSummaries?.filter((summary) => visibleSurface(summary.parentId)),
|
||||
);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
@@ -1237,7 +1215,6 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces,
|
||||
scene.visualSurfaceSummaries,
|
||||
);
|
||||
|
||||
return {
|
||||
|
||||
@@ -63,7 +63,6 @@ export interface VomScene {
|
||||
activeScopeBlocks?: ActiveScopeBlock[];
|
||||
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
|
||||
visualSurfaces?: VomVisualSurface[];
|
||||
visualSurfaceSummaries?: VomVisualSurfaceSummary[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
@@ -108,18 +107,10 @@ export interface VomVisualSurface {
|
||||
backendNodeId: number;
|
||||
frameId: string;
|
||||
renderingKind: "canvas";
|
||||
rect: Rect;
|
||||
localRect?: Rect;
|
||||
label?: string;
|
||||
memberCount: number;
|
||||
}
|
||||
|
||||
export interface VomVisualSurfaceSummary {
|
||||
parentId: number | null;
|
||||
frameId: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface RenderedRef extends VomRef {
|
||||
role?: string;
|
||||
name?: string;
|
||||
|
||||
Reference in New Issue
Block a user