fix(vom): bug fix

apps/extension/src/entrypoints/content.ts

@@ -344,6 +344,11 @@ export default defineContentScript({
       renderControlOverlay();
     }
 
+    async function waitForRenderedOverlayUpdate(): Promise<void> {
+      await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
+      await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
+    }
+
     function clearCurrentAgentSession(): void {
       const sessionId = overlays.snapshot().activeSessionId;
       if (!sessionId) return;
@@ -528,6 +533,7 @@ export default defineContentScript({
       };
       overlays.clearAgentHelpRequest(requestId);
       renderAll();
+      await waitForRenderedOverlayUpdate();
       await chrome.runtime.sendMessage(msg).catch((err) => {
         console.debug("[bsk overlay] help finish failed", err);
       });

apps/extension/src/tools/__tests__/human-loop.test.ts

@@ -107,6 +107,30 @@ describe("handleRequestHelp", () => {
     expect(res).toMatchObject({ outcome: "continued", note: "ok", tab_id: 5 });
   });
 
+  it("does not refresh observations after the user returns control", async () => {
+    const cdpSend = vi.fn(async () => ({}));
+    const deps = baseDeps({
+      cdp: { send: cdpSend } as unknown as RequestHelpDeps["cdp"],
+    });
+    const res = await handleRequestHelp(
+      fakeManager("abcd", 99, 5),
+      baseParams({ tab_id: 5 }),
+      deps,
+    );
+
+    expect(res).toMatchObject({ outcome: "continued", tab_id: 5 });
+    expect(cdpSend).not.toHaveBeenCalledWith(
+      expect.any(Number),
+      "Accessibility.getFullAXTree",
+      expect.anything(),
+    );
+    expect(cdpSend).not.toHaveBeenCalledWith(
+      expect.any(Number),
+      "DOMSnapshot.captureSnapshot",
+      expect.anything(),
+    );
+  });
+
   it("forwards title into the help request message when provided", async () => {
     const deps = baseDeps();
     await handleRequestHelp(

apps/extension/src/tools/human-loop.ts

@@ -304,11 +304,12 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
   );
 }
 
-function finishHelp(
+async function finishHelp(
   help: ActiveHelpRequest,
   value: RequestHelpResult | RpcError,
   notifyContent = true,
-): void {
+  waitForCleanup = false,
+): Promise<void> {
   if (help.settled) return;
   help.settled = true;
   activeHelpRequests.delete(help.requestId);
@@ -317,7 +318,11 @@ function finishHelp(
   if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
   clearRearmTimer(help.primaryTabId);
   for (const tabId of help.overlayTabIds) clearRearmTimer(tabId);
-  if (notifyContent) void cleanupHelp(help);
+  if (notifyContent) {
+    const cleanup = cleanupHelp(help);
+    if (waitForCleanup) await cleanup;
+    else void cleanup;
+  }
   help.resolve(value);
 }
 
@@ -386,12 +391,17 @@ async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise<boolea
     try {
       const legacyFinish = await refreshAndSendHelpOverlay(help, tabId);
       if (legacyFinish && !help.settled) {
-        finishHelp(help, {
-          outcome: legacyFinish.outcome,
-          ...(legacyFinish.note ? { note: legacyFinish.note } : {}),
-          tab_id: help.primaryTabId,
-          resolved_targets: help.resolvedTargets,
-        });
+        await finishHelp(
+          help,
+          {
+            outcome: legacyFinish.outcome,
+            ...(legacyFinish.note ? { note: legacyFinish.note } : {}),
+            tab_id: help.primaryTabId,
+            resolved_targets: help.resolvedTargets,
+          },
+          true,
+          true,
+        );
       }
       return true;
     } catch {
@@ -501,12 +511,17 @@ async function finishHelpFromContent(
     const match = await findHelpForTab(tabId, deps);
     if (match !== help) return;
   }
-  finishHelp(help, {
-    outcome: message.outcome,
-    ...(message.note ? { note: message.note } : {}),
-    tab_id: help.primaryTabId,
-    resolved_targets: help.resolvedTargets,
-  });
+  await finishHelp(
+    help,
+    {
+      outcome: message.outcome,
+      ...(message.note ? { note: message.note } : {}),
+      tab_id: help.primaryTabId,
+      resolved_targets: help.resolvedTargets,
+    },
+    true,
+    true,
+  );
 }
 
 function attachHelpTabListener(deps: RequestHelpDeps): () => void {
@@ -639,7 +654,7 @@ function startCompletionPolling(help: ActiveHelpRequest): void {
     }
     help.completionMatchedSince ??= now;
     if (now - help.completionMatchedSince < stableMs) return;
-    finishHelp(help, {
+    void finishHelp(help, {
       outcome: "completed",
       completed_by: "system",
       tab_id: help.primaryTabId,
@@ -698,7 +713,7 @@ export async function handleRequestHelp(
 
   return new Promise<RequestHelpResult | RpcError>((resolve) => {
     const timeoutTimer = setTimeout(() => {
-      finishHelp(help, {
+      void finishHelp(help, {
         outcome: "timed_out",
         tab_id: help.primaryTabId,
         resolved_targets: help.resolvedTargets,
@@ -727,7 +742,8 @@ export async function handleRequestHelp(
       abortHandler: null,
     };
 
-    const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
+    const onAbort = () =>
+      void finishHelp(help, { code: "cancelled", message: "request_help aborted" });
     help.abortHandler = onAbort;
     activeHelpRequests.set(requestId, help);
     if (deps.signal?.aborted) {
@@ -738,14 +754,19 @@ export async function handleRequestHelp(
     startCompletionPolling(help);
 
     void sendCurrentHelpOverlay(help)
-      .then((legacyFinish) => {
+      .then(async (legacyFinish) => {
         if (!legacyFinish || help.settled) return;
-        finishHelp(help, {
-          outcome: legacyFinish.outcome,
-          ...(legacyFinish.note ? { note: legacyFinish.note } : {}),
-          tab_id: help.primaryTabId,
-          resolved_targets: help.resolvedTargets,
-        });
+        await finishHelp(
+          help,
+          {
+            outcome: legacyFinish.outcome,
+            ...(legacyFinish.note ? { note: legacyFinish.note } : {}),
+            tab_id: help.primaryTabId,
+            resolved_targets: help.resolvedTargets,
+          },
+          true,
+          true,
+        );
       })
       .catch(() => {
         scheduleRearmForTab(tabId, deps);

crates/bsk-cli/skill/SKILL.md

@@ -215,6 +215,11 @@ refreshes do not return control by themselves. The result `outcome` is one of:
 `note` carries any text the user typed back. `resolved_targets` reports
 which refs/selectors matched a live element.
 
+`request-help` does not refresh the page model after the user returns
+control. After a `continued` or `completed` result, issue a separate
+observation tool call (usually `bsk snapshot --session <id>`) before using
+new refs or reasoning about the post-help page state.
+
 ### Recording — `bsk record`
 
 Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:

skill/SKILL.md

@@ -68,6 +68,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
 ```
 bsk navigate <url> --session <id>
 bsk snapshot --session <id>          → aria tree with @e1, @e2, … refs
+bsk observe --session <id>           → semantic VOM view; may reveal hover/focus surfaces
 bsk click @e3 --session <id>          → or bsk fill, bsk select, bsk press
 bsk snapshot --session <id>            → again after navigation / DOM change
 ```
@@ -78,11 +79,12 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
 
 ## Observation priority
 
-Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Only escalate when the latest snapshot cannot answer the question:
+Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
 
-1. `bsk snapshot` — default for page understanding and interaction planning
-2. `bsk get-html` — when hidden DOM, metadata, or markup details are required
-3. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the snapshot. Use `--ref @eN` (from the latest snapshot) to crop to one element; omit `--ref` for the full visible tab.
+1. `bsk snapshot` — strict static page understanding and interaction planning
+2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
+3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
+4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
 
 Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
 
@@ -141,7 +143,8 @@ Details and flags: **`bsk <cmd> --help`**
 
 | Command | Summary |
 |---------|---------|
-| `bsk snapshot` | First-choice page understanding: accessibility tree with `@eN` element refs |
+| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
+| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
 | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
 | `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
 
@@ -212,6 +215,11 @@ refreshes do not return control by themselves. The result `outcome` is one of:
 `note` carries any text the user typed back. `resolved_targets` reports
 which refs/selectors matched a live element.
 
+`request-help` does not refresh the page model after the user returns
+control. After a `continued` or `completed` result, issue a separate
+observation tool call (usually `bsk snapshot --session <id>`) before using
+new refs or reasoning about the post-help page state.
+
 ### Recording — `bsk record`
 
 Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
@@ -260,7 +268,7 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
 3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
 4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
-5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
+5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
 6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
 
 ---