mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): add new texts and fix bugs
This commit is contained in:
@@ -49,7 +49,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain("[visual surfaces]");
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
'@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; screenshot-only; requires=image-understanding]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
@@ -78,7 +78,7 @@ describe("renderVom single-layer page", () => {
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
'@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; screenshot-only; requires=image-understanding]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
|
||||
@@ -20,3 +20,7 @@ export type {
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
export {
|
||||
compareVisualSurfacePriority,
|
||||
type VisualSurfacePriorityInput,
|
||||
} from "./visual-surface-priority";
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
import { compareVisualSurfacePriority } from "./visual-surface-priority";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
|
||||
@@ -809,29 +810,12 @@ function tryAppendLine(state: RenderState, line: string, tokenLimit = state.maxT
|
||||
return true;
|
||||
}
|
||||
|
||||
function visualSurfaceArea(surface: VomVisualSurface): number {
|
||||
return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
|
||||
}
|
||||
|
||||
function compareVisualSurfaces(a: VomVisualSurface, b: VomVisualSurface): number {
|
||||
const aLabeled = cleaned(a.label) !== undefined;
|
||||
const bLabeled = cleaned(b.label) !== undefined;
|
||||
return (
|
||||
Number(bLabeled) - Number(aLabeled) ||
|
||||
visualSurfaceArea(b) - visualSurfaceArea(a) ||
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.visibleRect.y - b.visibleRect.y ||
|
||||
a.visibleRect.x - b.visibleRect.x ||
|
||||
a.backendNodeId - b.backendNodeId
|
||||
);
|
||||
}
|
||||
|
||||
function visualSurfaceLine(surface: VomVisualSurface, ref: string): string {
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||
const { x, y, w, h } = surface.visibleRect;
|
||||
const bounds = [x, y, w, h].map(Math.round).join(",");
|
||||
return ` @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
||||
return ` @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; screenshot-only; requires=image-understanding]`;
|
||||
}
|
||||
|
||||
function selectVisualSurfaces(
|
||||
@@ -847,7 +831,7 @@ function selectVisualSurfaces(
|
||||
|
||||
selected.push(surface);
|
||||
}
|
||||
selected.sort(compareVisualSurfaces);
|
||||
selected.sort(compareVisualSurfacePriority);
|
||||
return { selected, eligibleCount };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
export interface VisualSurfacePriorityInput {
|
||||
label?: string;
|
||||
visibleRect: { x: number; y: number; w: number; h: number };
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
}
|
||||
|
||||
function hasLabel(label: string | undefined): boolean {
|
||||
return (label?.trim().length ?? 0) > 0;
|
||||
}
|
||||
|
||||
function area(surface: VisualSurfacePriorityInput): number {
|
||||
return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
|
||||
}
|
||||
|
||||
/** Negative means `a` should be emitted before `b`. */
|
||||
export function compareVisualSurfacePriority(
|
||||
a: VisualSurfacePriorityInput,
|
||||
b: VisualSurfacePriorityInput,
|
||||
): number {
|
||||
return (
|
||||
Number(hasLabel(b.label)) - Number(hasLabel(a.label)) ||
|
||||
area(b) - area(a) ||
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.visibleRect.y - b.visibleRect.y ||
|
||||
a.visibleRect.x - b.visibleRect.x ||
|
||||
a.backendNodeId - b.backendNodeId
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user