From 905ba6a8a680b87320bd26211c4ca71c83493a07 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 28 Aug 2026 15:10:36 +0800 Subject: [PATCH] feat(vom): two channels to deal with canvas --- crates/bsk-cli/skill/SKILL.md | 13 +++++++---- crates/bsk-cli/src/cli/render_error.rs | 2 +- packages/dsh-plugin-browserskill/src/tools.ts | 12 ++++++---- packages/vom/src/__tests__/render.test.ts | 4 ++-- packages/vom/src/render.ts | 22 ++++++++++++++----- skill/SKILL.md | 13 +++++++---- 6 files changed, 45 insertions(+), 21 deletions(-) diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 5e72708..c6ba9e6 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -73,10 +73,15 @@ CSS-only hover menu looks like from here. It hovers a bounded set of likely trig few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` is cheaper and more precise. -When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered -canvas surface, not an interactable DOM control. Use -`bsk screenshot --ref @eN --session ` to inspect its visible crop. Do not pass a visual surface -ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. +When `bsk observe` renders +`@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered +canvas content that is not represented by the text observation, not an interactable DOM control. +Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually +inspect image output, use `bsk screenshot --ref @eN --session ` to obtain the visible crop. If +you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its +contents, and never guess coordinates; tell the user that they need to switch to a model with +image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; +those commands intentionally reject screenshot-only refs. Escalate page reading only as needed: diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index 04984f1..7f7aee5 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -303,7 +303,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo { summary: "snapshot ref does not support this operation", hint: Some( - "visual surface refs are screenshot-only; run `bsk screenshot --ref --session `", + "visual surface refs are screenshot-only; inspect `bsk screenshot --ref --session ` with image-understanding capability", ), exit_code: base.exit_code, }, diff --git a/packages/dsh-plugin-browserskill/src/tools.ts b/packages/dsh-plugin-browserskill/src/tools.ts index 5fc4101..be99780 100644 --- a/packages/dsh-plugin-browserskill/src/tools.ts +++ b/packages/dsh-plugin-browserskill/src/tools.ts @@ -489,8 +489,9 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): ? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " + "carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view." : "Produce a semantic VOM observation of the session's active tab (roles, states, perception " + - "probes) with @eN refs for browser_interact actions. Visual surface refs are screenshot-only; " + - "DOM refs support interaction. Read-only: never submits input."; + "probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " + + "understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " + + "alongside them. Read-only: never submits input."; register( defineTool({ name, @@ -755,8 +756,11 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): name: "inspect.screenshot", description: "Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " + - "This is the supported operation for visual surface refs emitted by browser_observe. Returns " + - "the image itself when the deployment supports image input, otherwise a file path.", + "This is the supported operation for visual surface refs emitted by inspect.observe. Only use " + + "it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " + + "or image-reading capability, tell the user to switch to a model with image understanding; do " + + "not guess content or coordinates. " + + "Returns the image itself when the deployment supports image input, otherwise a file path.", parameters: { session: SESSION_PARAM, ref: { diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 2039837..85bbcf9 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -48,7 +48,7 @@ describe("renderVom single-layer page", () => { const out = renderVom(input); expect(out.text).toContain( - '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]', + '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]', ); expect(out.refs).toContainEqual( expect.objectContaining({ @@ -77,7 +77,7 @@ describe("renderVom single-layer page", () => { const out = renderVom(input); expect(out.text).toContain( - '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]', + '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]', ); expect(out.refs).toContainEqual( expect.objectContaining({ diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 87a0fcb..555e42b 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -819,7 +819,7 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render const ref = `e${state.nextRef}`; const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`; const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : ""; - const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; use: bsk screenshot --ref @${ref}]`; + const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`; const nextTokens = state.tokens + estimateTokens(line); if (nextTokens > state.maxTokens) { state.truncated = true; @@ -872,24 +872,34 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren } } +type RenderTask = + | { kind: "node"; node: VomNode; depth: number } + | { kind: "visual"; parentId: number | null; depth: number }; + function pushRenderChildren( - stack: Array<{ node: VomNode; depth: number }>, + stack: RenderTask[], children: readonly VomNode[], depth: number, ): void { for (let index = children.length - 1; index >= 0; index -= 1) { - stack.push({ node: children[index], depth }); + stack.push({ kind: "node", node: children[index], depth }); } } function renderTree(children: Map, state: RenderState): void { - const stack: Array<{ node: VomNode; depth: number }> = []; + const stack: RenderTask[] = [{ kind: "visual", parentId: null, depth: 1 }]; pushRenderChildren(stack, children.get(null) ?? [], 1); while (stack.length > 0 && !state.stopped) { - const { node, depth } = stack.pop() as { node: VomNode; depth: number }; + const task = stack.pop() as RenderTask; + if (task.kind === "visual") { + emitVisualEntries(task.parentId, task.depth, state); + continue; + } + const { node, depth } = task; if (!shouldRender(node)) { + stack.push({ kind: "visual", parentId: node.id, depth }); pushRenderChildren(stack, children.get(node.id) ?? [], depth); continue; } @@ -935,9 +945,9 @@ function renderTree(children: Map, state: RenderState) emitVisualEntries(node.id, depth + 1, state); continue; } + stack.push({ kind: "visual", parentId: node.id, depth: depth + 1 }); pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1); } - emitVisualEntries(parentId, depth, state); } function renderNodes( diff --git a/skill/SKILL.md b/skill/SKILL.md index 5e72708..c6ba9e6 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -73,10 +73,15 @@ CSS-only hover menu looks like from here. It hovers a bounded set of likely trig few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` is cheaper and more precise. -When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered -canvas surface, not an interactable DOM control. Use -`bsk screenshot --ref @eN --session ` to inspect its visible crop. Do not pass a visual surface -ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. +When `bsk observe` renders +`@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered +canvas content that is not represented by the text observation, not an interactable DOM control. +Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually +inspect image output, use `bsk screenshot --ref @eN --session ` to obtain the visible crop. If +you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its +contents, and never guess coordinates; tell the user that they need to switch to a model with +image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; +those commands intentionally reject screenshot-only refs. Escalate page reading only as needed: