diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 32cd5ed..ed10b0b 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -344,6 +344,11 @@ export default defineContentScript({ renderControlOverlay(); } + async function waitForRenderedOverlayUpdate(): Promise { + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + } + function clearCurrentAgentSession(): void { const sessionId = overlays.snapshot().activeSessionId; if (!sessionId) return; @@ -528,6 +533,7 @@ export default defineContentScript({ }; overlays.clearAgentHelpRequest(requestId); renderAll(); + await waitForRenderedOverlayUpdate(); await chrome.runtime.sendMessage(msg).catch((err) => { console.debug("[bsk overlay] help finish failed", err); }); diff --git a/apps/extension/src/tools/__tests__/human-loop.test.ts b/apps/extension/src/tools/__tests__/human-loop.test.ts index 9f6e9c7..960758b 100644 --- a/apps/extension/src/tools/__tests__/human-loop.test.ts +++ b/apps/extension/src/tools/__tests__/human-loop.test.ts @@ -107,6 +107,30 @@ describe("handleRequestHelp", () => { expect(res).toMatchObject({ outcome: "continued", note: "ok", tab_id: 5 }); }); + it("does not refresh observations after the user returns control", async () => { + const cdpSend = vi.fn(async () => ({})); + const deps = baseDeps({ + cdp: { send: cdpSend } as unknown as RequestHelpDeps["cdp"], + }); + const res = await handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ tab_id: 5 }), + deps, + ); + + expect(res).toMatchObject({ outcome: "continued", tab_id: 5 }); + expect(cdpSend).not.toHaveBeenCalledWith( + expect.any(Number), + "Accessibility.getFullAXTree", + expect.anything(), + ); + expect(cdpSend).not.toHaveBeenCalledWith( + expect.any(Number), + "DOMSnapshot.captureSnapshot", + expect.anything(), + ); + }); + it("forwards title into the help request message when provided", async () => { const deps = baseDeps(); await handleRequestHelp( diff --git a/apps/extension/src/tools/human-loop.ts b/apps/extension/src/tools/human-loop.ts index 9b65325..5c05f04 100644 --- a/apps/extension/src/tools/human-loop.ts +++ b/apps/extension/src/tools/human-loop.ts @@ -304,11 +304,12 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise { ); } -function finishHelp( +async function finishHelp( help: ActiveHelpRequest, value: RequestHelpResult | RpcError, notifyContent = true, -): void { + waitForCleanup = false, +): Promise { if (help.settled) return; help.settled = true; activeHelpRequests.delete(help.requestId); @@ -317,7 +318,11 @@ function finishHelp( if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler); clearRearmTimer(help.primaryTabId); for (const tabId of help.overlayTabIds) clearRearmTimer(tabId); - if (notifyContent) void cleanupHelp(help); + if (notifyContent) { + const cleanup = cleanupHelp(help); + if (waitForCleanup) await cleanup; + else void cleanup; + } help.resolve(value); } @@ -386,12 +391,17 @@ async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise void { @@ -639,7 +654,7 @@ function startCompletionPolling(help: ActiveHelpRequest): void { } help.completionMatchedSince ??= now; if (now - help.completionMatchedSince < stableMs) return; - finishHelp(help, { + void finishHelp(help, { outcome: "completed", completed_by: "system", tab_id: help.primaryTabId, @@ -698,7 +713,7 @@ export async function handleRequestHelp( return new Promise((resolve) => { const timeoutTimer = setTimeout(() => { - finishHelp(help, { + void finishHelp(help, { outcome: "timed_out", tab_id: help.primaryTabId, resolved_targets: help.resolvedTargets, @@ -727,7 +742,8 @@ export async function handleRequestHelp( abortHandler: null, }; - const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" }); + const onAbort = () => + void finishHelp(help, { code: "cancelled", message: "request_help aborted" }); help.abortHandler = onAbort; activeHelpRequests.set(requestId, help); if (deps.signal?.aborted) { @@ -738,14 +754,19 @@ export async function handleRequestHelp( startCompletionPolling(help); void sendCurrentHelpOverlay(help) - .then((legacyFinish) => { + .then(async (legacyFinish) => { if (!legacyFinish || help.settled) return; - finishHelp(help, { - outcome: legacyFinish.outcome, - ...(legacyFinish.note ? { note: legacyFinish.note } : {}), - tab_id: help.primaryTabId, - resolved_targets: help.resolvedTargets, - }); + await finishHelp( + help, + { + outcome: legacyFinish.outcome, + ...(legacyFinish.note ? { note: legacyFinish.note } : {}), + tab_id: help.primaryTabId, + resolved_targets: help.resolvedTargets, + }, + true, + true, + ); }) .catch(() => { scheduleRearmForTab(tabId, deps); diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 3290d90..43d2417 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -215,6 +215,11 @@ refreshes do not return control by themselves. The result `outcome` is one of: `note` carries any text the user typed back. `resolved_targets` reports which refs/selectors matched a live element. +`request-help` does not refresh the page model after the user returns +control. After a `continued` or `completed` result, issue a separate +observation tool call (usually `bsk snapshot --session `) before using +new refs or reasoning about the post-help page state. + ### Recording — `bsk record` Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation: diff --git a/skill/SKILL.md b/skill/SKILL.md index 3e5599a..43d2417 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -68,6 +68,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe ``` bsk navigate --session bsk snapshot --session → aria tree with @e1, @e2, … refs +bsk observe --session → semantic VOM view; may reveal hover/focus surfaces bsk click @e3 --session → or bsk fill, bsk select, bsk press bsk snapshot --session → again after navigation / DOM change ``` @@ -78,11 +79,12 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / ## Observation priority -Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Only escalate when the latest snapshot cannot answer the question: +Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: -1. `bsk snapshot` — default for page understanding and interaction planning -2. `bsk get-html` — when hidden DOM, metadata, or markup details are required -3. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the snapshot. Use `--ref @eN` (from the latest snapshot) to crop to one element; omit `--ref` for the full visible tab. +1. `bsk snapshot` — strict static page understanding and interaction planning +2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery +3. `bsk get-html` — when hidden DOM, metadata, or markup details are required +4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. @@ -141,7 +143,8 @@ Details and flags: **`bsk --help`** | Command | Summary | |---------|---------| -| `bsk snapshot` | First-choice page understanding: accessibility tree with `@eN` element refs | +| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | +| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | | `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | @@ -212,6 +215,11 @@ refreshes do not return control by themselves. The result `outcome` is one of: `note` carries any text the user typed back. `resolved_targets` reports which refs/selectors matched a live element. +`request-help` does not refresh the page model after the user returns +control. After a `continued` or `completed` result, issue a separate +observation tool call (usually `bsk snapshot --session `) before using +new refs or reasoning about the post-help page state. + ### Recording — `bsk record` Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation: @@ -260,7 +268,7 @@ Always **`bsk session stop `** in a `finally`-style path so the Agent Window 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. 3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. 4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. -5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual. +5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual. 6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. ---