figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): two channels to deal with canvas

Browse Source
This commit is contained in:
Ljy-0827
2026-08-31 12:32:20 +08:00
parent a0773f3551
commit d660da87f3
6 changed files with 14 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/dsh-plugin-browserskill/src/tools.ts
+8 -4
View File
@@ -489,8 +489,9 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
"carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
"probes) with @eN refs for browser_interact actions. Visual surface refs are screenshot-only; " +
"DOM refs support interaction. Read-only: never submits input.";
"probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " +
"understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " +
"alongside them. Read-only: never submits input.";
register(
defineTool({
name,
@@ -753,8 +754,11 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
name: "inspect.screenshot",
description:
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
"This is the supported operation for visual surface refs emitted by browser_observe. Returns " +
"the image itself when the deployment supports image input, otherwise a file path.",
"This is the supported operation for visual surface refs emitted by inspect.observe. Only use " +
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
"not guess content or coordinates. " +
"Returns the image itself when the deployment supports image input, otherwise a file path.",
parameters: {
session: SESSION_PARAM,
ref: {
packages/vom/src/__tests__/render.test.ts
+2 -2
View File
@@ -48,7 +48,7 @@ describe("renderVom single-layer page", () => {
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]',
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
@@ -77,7 +77,7 @@ describe("renderVom single-layer page", () => {
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]',
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
packages/vom/src/render.ts
+1 -1
View File
@@ -763,7 +763,7 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render
const ref = `e${state.nextRef}`;
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; use: bsk screenshot --ref @${ref}]`;
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
const nextTokens = state.tokens + estimateTokens(line);
if (nextTokens > state.maxTokens) {
state.truncated = true;