diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 2b870c2..e098730 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -80,7 +80,7 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. -When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session ` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. +When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session ` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. ## Observation priority diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index bd5db93..d7b2992 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -232,7 +232,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo { summary: "snapshot ref does not support this operation", hint: Some( - "visual surface refs are screenshot-only; run `bsk screenshot --ref --session `", + "visual surface refs are screenshot-only; inspect `bsk screenshot --ref --session ` with image-understanding capability", ), exit_code: base.exit_code, }, diff --git a/packages/dsh-plugin-browserskill/src/tools.ts b/packages/dsh-plugin-browserskill/src/tools.ts index b848b6b..cf1ba68 100644 --- a/packages/dsh-plugin-browserskill/src/tools.ts +++ b/packages/dsh-plugin-browserskill/src/tools.ts @@ -489,8 +489,9 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): ? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " + "carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view." : "Produce a semantic VOM observation of the session's active tab (roles, states, perception " + - "probes) with @eN refs for browser_interact actions. Visual surface refs are screenshot-only; " + - "DOM refs support interaction. Read-only: never submits input."; + "probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " + + "understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " + + "alongside them. Read-only: never submits input."; register( defineTool({ name, @@ -753,8 +754,11 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): name: "inspect.screenshot", description: "Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " + - "This is the supported operation for visual surface refs emitted by browser_observe. Returns " + - "the image itself when the deployment supports image input, otherwise a file path.", + "This is the supported operation for visual surface refs emitted by inspect.observe. Only use " + + "it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " + + "or image-reading capability, tell the user to switch to a model with image understanding; do " + + "not guess content or coordinates. " + + "Returns the image itself when the deployment supports image input, otherwise a file path.", parameters: { session: SESSION_PARAM, ref: { diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 08f7350..84a6731 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -48,7 +48,7 @@ describe("renderVom single-layer page", () => { const out = renderVom(input); expect(out.text).toContain( - '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]', + '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]', ); expect(out.refs).toContainEqual( expect.objectContaining({ @@ -77,7 +77,7 @@ describe("renderVom single-layer page", () => { const out = renderVom(input); expect(out.text).toContain( - '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]', + '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]', ); expect(out.refs).toContainEqual( expect.objectContaining({ diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 53076d0..92a151a 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -763,7 +763,7 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render const ref = `e${state.nextRef}`; const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`; const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : ""; - const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; use: bsk screenshot --ref @${ref}]`; + const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`; const nextTokens = state.tokens + estimateTokens(line); if (nextTokens > state.maxTokens) { state.truncated = true; diff --git a/skill/SKILL.md b/skill/SKILL.md index 2b870c2..e098730 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -80,7 +80,7 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. -When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session ` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. +When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session ` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. ## Observation priority