From 641d934337fa0dfa37fe4d960091a744e6d35b04 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Tue, 1 Sep 2026 14:55:36 +0800 Subject: [PATCH 1/3] perf(vom): optimize rendering algorithms --- packages/vom/src/__tests__/render.test.ts | 105 ++++++++++++ packages/vom/src/render.ts | 200 +++++++++++++++------- 2 files changed, 239 insertions(+), 66 deletions(-) diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index fa02ae5..a064df6 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -298,6 +298,34 @@ describe("renderVom single-layer page", () => { expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); + it("renders through deeply nested transparent wrappers without exhausting the call stack", () => { + const wrapperCount = 10_000; + const nodes = [node({ id: 1, role: "RootWebArea", name: "Doc" })]; + for (let index = 0; index < wrapperCount; index += 1) { + nodes.push( + node({ + id: index + 2, + parentId: index + 1, + role: "generic", + }), + ); + } + nodes.push( + node({ + id: wrapperCount + 2, + parentId: wrapperCount + 1, + role: "button", + name: "Deep action", + tag: "button", + }), + ); + + const out = renderVom(scene(nodes)); + + expect(out.text).toContain('@e1 button "Deep action"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: wrapperCount + 2 }]); + }); + it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => { const out = renderVom( scene([ @@ -785,6 +813,55 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain("北京市小客车指标调控管理信息系统]"); }); + it("derives duplicate-label context from indexed DOM ancestry without semantic proximity", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc", contextScopeId: "root" }), + node({ + id: 3, + parentId: 1, + role: "button", + name: "View", + tag: "button", + domParentId: 2, + domAncestorIds: [2, 1], + contextScopeId: "root", + }), + node({ + id: 5, + parentId: 1, + role: "button", + name: "View", + tag: "button", + domParentId: 4, + domAncestorIds: [4, 1], + contextScopeId: "root", + }), + node({ + id: 2, + parentId: 1, + role: "group", + name: "Project Alpha", + domParentId: 1, + domAncestorIds: [1], + contextScopeId: "root", + }), + node({ + id: 4, + parentId: 1, + role: "group", + name: "Project Beta", + domParentId: 1, + domAncestorIds: [1], + contextScopeId: "root", + }), + ]), + ); + + expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]'); + expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]'); + }); + it("does not add low-value root context to duplicated brand links", () => { const out = renderVom( scene([ @@ -833,6 +910,34 @@ describe("renderVom single-layer page", () => { }); describe("renderVom double-layer page", () => { + it("collects blocking-layer descendants even when children precede parents in source order", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 3, + parentId: 2, + role: "button", + name: "Close", + tag: "button", + paintOrder: 0, + }), + node({ + id: 2, + parentId: 1, + role: "dialog", + name: "Modal", + rect: { x: 0, y: 0, w: 1280, h: 720 }, + paintOrder: 10, + position: "fixed", + }), + ]), + ); + + expect(out.text).toContain('@e1 button "Close"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]); + }); + it("renders a blocking modal first and folds the base page into an occluded summary", () => { const out = renderVom( scene([ diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index b943dbd..c7c4164 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -421,6 +421,41 @@ function buildChildren(nodes: VomNode[]): Map { return children; } +const CONTEXT_LABEL_ROLES = new Set([ + "heading", + "article", + "section", + "group", + "region", + "row", + "form", +]); + +interface DomContextIndex { + nonBoundaryByScopeParent: Map>; + sourceOrderById: Map; +} + +function buildDomContextIndex(nodes: VomNode[]): DomContextIndex { + const nonBoundaryByScopeParent = new Map>(); + const sourceOrderById = new Map(); + for (const [sourceOrder, node] of nodes.entries()) { + sourceOrderById.set(node.id, sourceOrder); + if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node)) || isContextBoundary(node)) continue; + const parentId = node.domParentId; + if (parentId === undefined || parentId === null) continue; + let byParent = nonBoundaryByScopeParent.get(node.contextScopeId); + if (!byParent) { + byParent = new Map(); + nonBoundaryByScopeParent.set(node.contextScopeId, byParent); + } + const siblings = byParent.get(parentId) ?? []; + siblings.push(node); + byParent.set(parentId, siblings); + } + return { nonBoundaryByScopeParent, sourceOrderById }; +} + function duplicateReferenceNames(nodes: VomNode[]): Set { const counts = new Map(); for (const node of nodes) { @@ -440,13 +475,7 @@ function needsHandleContext(nodeName: string, duplicateRefNames: Set): b } function contextLabel(node: VomNode, targetName: string): string | undefined { - if ( - !["heading", "article", "section", "group", "region", "row", "form"].includes( - normalizedRole(node), - ) - ) { - return undefined; - } + if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node))) return undefined; return weakContextText( cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), targetName, @@ -607,19 +636,45 @@ function collectDomContext( ): ContextCandidate[] { const ancestorIds = node.domAncestorIds ?? []; if (ancestorIds.length === 0) return []; - const ancestors = new Set(ancestorIds); const contexts: ContextCandidate[] = []; + const ancestorRank = new Map(); + for (const [rank, ancestorId] of ancestorIds.entries()) { + if (!ancestorRank.has(ancestorId)) ancestorRank.set(ancestorId, rank); + } - for (const candidate of state.nodesById.values()) { + const candidates = new Map(); + const nonBoundaryByParent = state.domContextIndex.nonBoundaryByScopeParent.get( + node.contextScopeId, + ); + for (const ancestorId of ancestorRank.keys()) { + const boundary = state.nodesById.get(ancestorId); + const boundaryParentId = boundary?.domParentId; + if ( + boundary && + sharesContextScope(node, boundary) && + isContextBoundary(boundary) && + boundaryParentId !== undefined && + boundaryParentId !== null && + ancestorRank.has(boundaryParentId) + ) { + candidates.set(boundary.id, boundary); + } + for (const candidate of nonBoundaryByParent?.get(ancestorId) ?? []) { + candidates.set(candidate.id, candidate); + } + } + + const sourceOrderById = state.domContextIndex.sourceOrderById; + const orderedCandidates = [...candidates.values()].sort( + (a, b) => (sourceOrderById.get(a.id) ?? 0) - (sourceOrderById.get(b.id) ?? 0), + ); + for (const candidate of orderedCandidates) { if (candidate.id === node.id) continue; - if (!sharesContextScope(node, candidate)) continue; - if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue; const parentId = candidate.domParentId; - if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue; + if (parentId === undefined || parentId === null) continue; const label = contextLabel(candidate, nodeName); if (!label) continue; - const rank = ancestorIds.indexOf(parentId); - contexts.push({ text: label, source: "dom", order: rank < 0 ? contexts.length : rank }); + contexts.push({ text: label, source: "dom", order: ancestorRank.get(parentId) ?? 0 }); } return contexts; @@ -734,6 +789,7 @@ interface RenderState { children: Map; parentMap: Map; nodesById: Map; + domContextIndex: DomContextIndex; duplicateRefNames: Set; surfaceMap: Map; scopeMap: Map; @@ -768,20 +824,25 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren } } -function renderTree( - children: Map, - parentId: number | null, +function pushRenderChildren( + stack: Array<{ node: VomNode; depth: number }>, + children: readonly VomNode[], depth: number, - state: RenderState, ): void { - if (state.stopped) return; + for (let index = children.length - 1; index >= 0; index -= 1) { + stack.push({ node: children[index], depth }); + } +} - const nodes = children.get(parentId) ?? []; - for (const node of nodes) { - if (state.stopped) return; +function renderTree(children: Map, state: RenderState): void { + const stack: Array<{ node: VomNode; depth: number }> = []; + pushRenderChildren(stack, children.get(null) ?? [], 1); + + while (stack.length > 0 && !state.stopped) { + const { node, depth } = stack.pop() as { node: VomNode; depth: number }; if (!shouldRender(node)) { - renderTree(children, node.id, depth, state); + pushRenderChildren(stack, children.get(node.id) ?? [], depth); continue; } @@ -798,7 +859,7 @@ function renderTree( if (nextTokens > state.maxTokens) { state.truncated = true; state.stopped = true; - return; + break; } state.lines.push(line); @@ -823,7 +884,7 @@ function renderTree( if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) { continue; } - renderTree(children, node.id, depth + 1, state); + pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1); } } @@ -848,12 +909,13 @@ function renderNodes( children, parentMap: buildParentMap(nodes), nodesById: new Map(nodes.map((node) => [node.id, node])), + domContextIndex: buildDomContextIndex(nodes), duplicateRefNames: duplicateReferenceNames(nodes), surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])), scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])), }; - renderTree(children, null, 1, state); + renderTree(children, state); return state; } @@ -922,23 +984,33 @@ function paintLineageByFrame( return lineage; } +function cachedPaintLineageByFrame( + node: VomNode, + parentMap: Map, + nodesById: Map, + cache: Map>, +): Map { + const cached = cache.get(node.id); + if (cached) return cached; + const lineage = paintLineageByFrame(node, parentMap, nodesById); + cache.set(node.id, lineage); + return lineage; +} + /** Resolve both nodes to the nearest document where their paint order is comparable. */ function comparablePaintNodes( target: VomNode, blocker: VomNode, parentMap: Map, nodesById: Map, + lineageCache: Map>, ): { target: VomNode; blocker: VomNode } | null { if (target.frameId === blocker.frameId) return { target, blocker }; - const targetLineage = paintLineageByFrame(target, parentMap, nodesById); - let blockerInFrame: VomNode | undefined = blocker; - let guard = 0; - while (blockerInFrame && guard <= parentMap.size) { - const targetInFrame = targetLineage.get(blockerInFrame.frameId); + const targetLineage = cachedPaintLineageByFrame(target, parentMap, nodesById, lineageCache); + const blockerLineage = cachedPaintLineageByFrame(blocker, parentMap, nodesById, lineageCache); + for (const [frameId, blockerInFrame] of blockerLineage) { + const targetInFrame = targetLineage.get(frameId); if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame }; - const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null; - blockerInFrame = parentId === null ? undefined : nodesById.get(parentId); - guard += 1; } return null; } @@ -949,12 +1021,13 @@ function isBlockedByRegion( parentMap: Map, nodesById: Map, viewportCoverage: number, + lineageCache: Map>, ): boolean { if (target.id === blocker.id) return false; if (isAncestorOf(parentMap, blocker.id, target.id)) return false; if (isAncestorOf(parentMap, target.id, blocker.id)) return false; if (!target.rect || !blocker.rect) return false; - const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById); + const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById, lineageCache); if (!paintNodes) return false; const points = interactionPoints(target.rect); @@ -979,17 +1052,22 @@ function isBlockedByRegion( return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)); } -function collectDescendantsOfIds(nodes: VomNode[], roots: Set): Set { +function collectDescendants(nodes: VomNode[], roots: Set): Set { + const children = new Map(); + for (const node of nodes) { + if (node.parentId === null) continue; + const siblings = children.get(node.parentId) ?? []; + siblings.push(node.id); + children.set(node.parentId, siblings); + } + const included = new Set(roots); - let changed = true; - while (changed) { - changed = false; - for (const node of nodes) { - if (included.has(node.id)) continue; - if (node.parentId !== null && included.has(node.parentId)) { - included.add(node.id); - changed = true; - } + const queue = [...roots]; + for (let index = 0; index < queue.length; index += 1) { + for (const childId of children.get(queue[index]) ?? []) { + if (included.has(childId)) continue; + included.add(childId); + queue.push(childId); } } return included; @@ -998,6 +1076,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set): Set [node.id, node])); + const lineageCache = new Map>(); const candidates = nodes .filter(isPositionedRegionCandidate) .map((node) => { @@ -1014,40 +1093,29 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] { for (const target of nodes) { if (!isVomReferenceNode(target)) continue; const blocked = candidates.some((candidate) => - isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage), + isBlockedByRegion( + target, + candidate.node, + parentMap, + nodesById, + candidate.viewportCoverage, + lineageCache, + ), ); if (blocked) blockedRoots.add(target.id); } if (blockedRoots.size === 0) return nodes; - const blocked = collectDescendantsOfIds(nodes, blockedRoots); + const blocked = collectDescendants(nodes, blockedRoots); return nodes.filter((node) => !blocked.has(node.id)); } -function collectDescendantsOfMembers(nodes: VomNode[], members: Set): Set { - const included = new Set(members); - let changed = true; - - while (changed) { - changed = false; - for (const node of nodes) { - if (included.has(node.id)) continue; - if (node.parentId !== null && included.has(node.parentId)) { - included.add(node.id); - changed = true; - } - } - } - - return included; -} - function countRenderable(nodes: VomNode[]): number { return nodes.filter(shouldRender).length; } function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult { - const included = collectDescendantsOfMembers(scene.nodes, layer.members); + const included = collectDescendants(scene.nodes, layer.members); const visibleNodes = scene.nodes.filter((node) => included.has(node.id)); const hiddenCount = countRenderable(scene.nodes.filter((node) => !included.has(node.id))); const header = [ From 536d3a9844c4347e5f12195c8d9db446ab3c6383 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Tue, 1 Sep 2026 18:39:06 +0800 Subject: [PATCH 2/3] chore: ignore local pnpm store and git worktrees Both directories are local scratch space that kept showing up as untracked noise in `git status`. Co-authored-by: Cursor --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 744b61c..3a8a21f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ dist/ .output/ .wxt/ .web-extension-id +.pnpm-store/ + +# Git worktrees +.worktrees/ # Logs *.log From 59d6533eff4f9a15504143c33670ab7ba1ff8e9d Mon Sep 17 00:00:00 2001 From: drakezhang Date: Tue, 1 Sep 2026 18:39:19 +0800 Subject: [PATCH 3/3] docs(skill): trim agent instructions to a leaner core The CLI skill had grown to 20.8 KB of command tables and flag catalogs that `bsk --help` already answers at runtime, so every agent paid for them on every task. Rewrite both skills around the durable rules an agent cannot rediscover -- session lifecycle, the observe/act/observe loop, the Agent Window boundary, and the stop-on-success contract -- and delegate flag details to the CLI's own help output. The CLI skill drops to 6.9 KB (-67%) and the DSH skill to 5.1 KB. Keep the parts that help discovery has no answer for: the literal `[hover first: ...]` observation marker, the `request-help` outcome values, the complete command-name inventory, and the required flags agents guess wrong (`fill --value`, `select --value`, `screenshot --out`, `emulate --device`). Also bind `--no-focus` explicitly to `bsk session start`, since a bare mention led agents to pass it to `bsk navigate`. Validated with 36 agent runs of the `core` eval suite against a real browser, comparing the old and new instructions with everything else held constant. Pass rate did not regress (the lean version was better on `form-controls`) and token cost per task came out even. The inventory and flag signatures cut `--help` round-trips from 1.7 to 0.7 per task and removed the invented `bsk back` / `bsk forward` calls the first draft produced. Tighten the DSH size assertion to a range so the budget is bounded from both sides and accidental truncation fails the test. Co-authored-by: Cursor --- crates/bsk-cli/skill/SKILL.md | 381 +++++------------- .../dsh-plugin-browserskill/skill/SKILL.md | 175 +++----- .../tests/skill.test.ts | 5 +- skill/SKILL.md | 381 +++++------------- 4 files changed, 266 insertions(+), 676 deletions(-) diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 42bb452..f310516 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -1,327 +1,142 @@ --- name: browser-skill description: | - Use when the user asks to perform browser automation tasks against their - logged-in browser: visit and read pages, fill forms, scrape data, click - through a flow, regression-test a PR's UI, validate a deployed page. - Requires the bsk CLI installed and the browser-skill extension loaded. + Use when the user asks to automate their logged-in Chromium browser: visit + and read pages, fill forms, scrape data, click through flows, regression-test + a PR's UI, validate a deployed page, or operate a tab they identify. Requires + the bsk CLI and browser extension. --- # browser-skill -Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab. +Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent +Window** with the user's existing logins and cookies. User-window tabs remain protected unless they +are explicitly borrowed. -## When to use +Do not use this skill for tasks with no browser, for extension installation, or when the user only +wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages. -- Open pages, read titles/text, scrape structured data from sites the user can already access -- Fill forms, click through multi-step flows, smoke-test a UI change -- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient -- Operate on a specific user tab they point you at (after `bsk tab borrow`) +## Required lifecycle -## When NOT to use +Every browser task owns a bounded session: -- Tasks with **no browser** involved (files, APIs, databases only) -- Installing or configuring the extension (point the user to setup docs instead) -- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets -- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session -- Replacing the user's manual browsing when they only wanted an explanation - -## Prerequisites - -1. `bsk` on `PATH` (Rust CLI from browser-skill) -2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue) -3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails - -## Mandatory workflow - -Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes). - -``` -1. bsk session start → capture the 4-letter session id printed on stdout -2. … every tool command … → always pass --session -3. bsk session stop → REQUIRED when done (even on error paths) +```text +1. bsk session start # retain the printed 4-letter session id +2. bsk ... --session # pass it to every session-scoped command +3. bsk session stop # always run on success and error paths ``` -Optional: `bsk session start --browser ` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window. +Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the +user explicitly asks to keep it open. Stopping also returns borrowed tabs. -Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand. +When multiple browsers are connected, use `bsk browsers` and start with +`bsk session start --browser `. Add `--no-focus` to that same start command when the +Agent Window does not need to interrupt the user's current work; it is not a flag on other commands. +Run `bsk doctor` when startup or transport problems persist after one retry. -## Stop when the goal is met +## Work toward one observable goal -Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. +- Derive a concrete success condition from the user's request or a supplied trace. +- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an + ambiguous result. +- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks. +- If a human-only step appears or two attempts make no progress, request help instead of + brute-forcing. -1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). -2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. -3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). -4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. -5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. -6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. +With a trace, follow its semantic target information and values in order, but treat its refs as +record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the +task; it does not expand the user's goal or authorize additional actions. -**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. +## Observe, act, observe -**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. +Use this default loop: -## Core interaction loop - -Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). - -``` +```text bsk navigate --session -bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces -bsk snapshot --session → static aria tree fallback when VOM is insufficient -bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking -bsk click @e4 --session → or bsk fill, bsk select, bsk press -bsk observe --session → again after navigation / DOM change +bsk observe --session +bsk click|hover|fill|select|press ... --session +bsk observe --session # after navigation or a meaningful DOM change ``` -**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. +Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also +make them stale. Observe again before the next interaction. -Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. +The listed items are labels, not usable refs: hover the trigger, observe again, then act on the +revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. -When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. +Escalate page reading only as needed: -## Observation priority +1. `bsk observe` for normal semantic understanding, text, controls, and refs. +2. `bsk snapshot` when a stricter static accessibility tree is more useful. +3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide. +4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence. -Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is +needed, obtain a fresh observation before acting on screenshot or HTML findings. -1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery -2. `bsk snapshot` — strict static accessibility tree fallback -3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +## Respect the Agent Window boundary -Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. +Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with +`bsk tab list --scope user --session `, then `bsk tab borrow `. Return it immediately +after the relevant step with `bsk tab return `; never invent a tab id or keep a personal tab +borrowed across unrelated work. -## Sandbox rules +## Ask the human when needed -| Rule | Detail | -|------|--------| -| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default | -| User tabs | Read-only until borrowed: `bsk tab list --session --scope user` then `bsk tab borrow --session ` | -| Return borrowed tabs | Call `bsk tab return --session ` when finished; unreturned tabs are **auto-returned** on `bsk session stop` | -| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first | +Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the +user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete +controls can be highlighted. Use completion criteria only when the page has a clear stable success +signal. -## Global flags +The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled` +(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after +`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a +blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before +reasoning about the page or using refs. -| Flag | Purpose | -|------|---------| -| `--json` | Machine-readable JSON on stdout (errors too) | -| `--quiet` | Suppress informational stderr | -| `-v` / `-vv` | More verbose logging | +## Command inventory -Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`. +This list of names is complete. Never invent a command outside it; read +`bsk --help` for flags instead of guessing them. -Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk --help`** - -## CLI command reference (one line each) - -Details and flags: **`bsk --help`** - -### Diagnostics - -| Command | Summary | -|---------|---------| -| `bsk status` | Connection health, connected browsers, active sessions | -| `bsk doctor` | Deep diagnostics and repair hints | -| `bsk browsers` | List connected browser instances (ids, labels, versions) | - -### Session - -| Command | Summary | -|---------|---------| -| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** | -| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus | -| `bsk session stop ` | End session, close Agent Window, auto-return borrowed tabs | -| `bsk session stop --all` | Stop every active session | -| `bsk session list` | List active sessions | - -### Window (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) | - -### Device emulation — `bsk emulate` (requires `--session `) - -Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour: - -```bash -bsk emulate --session --device iphone-14 -bsk emulate --session --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch -bsk emulate --session --off +```text +session start|stop|list browsers status doctor update logs +navigate navigate-back navigate-forward reload wait-for-navigation wait-ms +observe snapshot get-html screenshot console network +click hover fill select press evaluate +tab list|create|close|select|borrow|return window resize emulate +request-help record start|stop ``` -- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off. -- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.) -- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment. -- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides). -- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams. +Required flags that are easy to get wrong: -### Tabs (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) | -| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) | -| `bsk tab close ` | Close an agent tab | -| `bsk tab select ` | Focus an agent tab | -| `bsk tab borrow ` | Move a user tab into the Agent Window | -| `bsk tab return ` | Return a borrowed tab to its original window | - -### Observation (require `--session` unless noted) - -| Command | Summary | -|---------|---------| -| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | -| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | -| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | - -### Console & network debugging (read-only; require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) | -| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) | - -Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since ` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured. - -### Navigation - -| Command | Summary | -|---------|---------| -| `bsk navigate ` | Go to URL in agent tab (`--wait-until`, `--timeout`) | -| `bsk navigate-back` | History back one step | -| `bsk navigate-forward` | History forward one step | -| `bsk reload` | Reload current tab (`--hard` bypass cache) | - -(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.) - -### Interaction - -| Command | Summary | -|---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | -| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | -| `bsk fill --value ` | Clear and type into input | -| `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | -| `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) | - -### Scripting & timing - -| Command | Summary | -|---------|---------| -| `bsk evaluate ` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors | -| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) | -| `bsk wait-ms ` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) | - -### Ask the human for help — `bsk request-help` - -When a step needs a human (captcha, login, OTP) or you want the user to -confirm an important action, pause and ask: - - bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ - --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - -- `--prompt` (required): what the user should do. -- `--title` (optional): custom title for the overlay panel. When omitted, - the extension shows its default localized title. -- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector - (`#submit`) to scroll to and flash-highlight. **Strongly recommended** — - whenever the prompt refers to a concrete element (a button to click, a - field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the - user is guided straight to the right spot instead of hunting for it. For - interaction scenarios, always include the relevant target(s); reserve a - prompt with no `--target` for cases where there is genuinely no specific - element to point at (e.g. "wait for the page to finish loading"). -- `--timeout` (default `5m`): how long to wait. -- `--completion-criteria` (optional): JSON success detector. Use it only - when there is a concrete post-help success signal, e.g. - `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. - -The target tab is brought to the foreground; the page stays interactive -while the agent control mask is hidden. The call blocks until the user -explicitly acts, the timeout expires, cancellation arrives, or explicit -completion criteria match. Page reloads, SPA route changes, and captcha -refreshes do not return control by themselves. The result `outcome` is one of: - -- `continued` — the user finished and clicked Done / return control (treat as confirm). -- `cancelled` — the user clicked Cancel (treat as reject/abort). -- `timed_out` — nobody acted within the timeout. -- `completed` — the explicit `--completion-criteria` matched while the user had control. -- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. - -`note` carries any text the user typed back. `resolved_targets` reports -which refs/selectors matched a live element. - -`request-help` does not refresh the page model after the user returns -control. After a `continued` or `completed` result, issue a separate -observation tool call (usually `bsk snapshot --session `) before using -new refs or reasoning about the post-help page state. -#### Disabling request-help (unattended mode) -Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then -returns immediately with `outcome="disabled"` (no overlay, no waiting, -exit 0). Any other value keeps it enabled. If you get `disabled`, do not -retry — complete the task autonomously or stop gracefully. - -### Recording — `bsk record` - -Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`. - -```bash -bsk record start --browser \ - [--url https://…] [--purpose "publish a wiki doc"] \ - [--max-page-tokens 3000] [--redact-values] \ - [--output trace] -# `--url` is optional; default https://example.com/ when omitted (must be http(s)). -# Blocks until the user clicks Finish in the recording panel, then writes: -# trace/trace.json — action chain (+ state index when v3) -# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state - -bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable +```text +bsk fill --value bsk select --value +bsk screenshot --out bsk emulate --device ``` -- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`. -- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles. -- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer). -- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead. -- `--purpose` is optional context metadata; it does **not** change what gets captured. -- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`. -- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. +`select` matches an option's `value` attribute, not its visible label. Device preset ids are +lowercase and hyphenated, such as `iphone-14`. -## Error handling +- `console` and `network` provide bounded, read-only debugging evidence. +- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit + them. Use `--off` to restore the real environment. +- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With + `--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC + succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data. +- `record` captures a user's actions for later replay. Read `bsk record start --help` before use, + and never record banking, SSO, password-manager, or other sensitive pages. -### Exit codes (`echo $?` after `bsk …`) +## Recover without wandering -| Code | Meaning | What to do | -|------|---------|------------| -| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success | -| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot | -| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command | -| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open | -| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` | -| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. | +- Stale ref: observe again and retry the intended action once. +- Unknown tab or session: list current tabs/sessions; never guess identifiers. +- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful. +- Unsupported command: continue with available capabilities; suggest updating only when the missing + command is necessary. +- Unrecoverable failure: report the blocker and stop the session in a finally-style path. -Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`. - -### When to run diagnostics - -| Situation | Command | -|-----------|---------| -| Before first task in a session | `bsk status` — extension connected? | -| Any failure you cannot fix in one retry | `bsk doctor` | -| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser ` | - -Always **`bsk session stop `** in a `finally`-style path so the Agent Window closes and borrowed tabs return. - -## Red lines - -1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration. -2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. -3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. -4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. -5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual. -6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. - ---- - -**More detail for any command:** `bsk --help` +The CLI's current help and error hints are authoritative for flags, parameters, and recovery +details.