feat(extension): emit pending hover before actions

apps/extension/src/content/__tests__/record-capture.test.ts

@@ -470,6 +470,92 @@ describe("record-capture semantic", () => {
     });
   });
 
+  it("records hover before filling a field inside the hover surface", () => {
+    document.body.innerHTML = `
+      <button type="button" aria-label="Open user navigation menu">avatar</button>
+      <div class="user-menu dropdown-menu">
+        <label for="nickname">Nickname</label>
+        <input id="nickname" />
+      </div>
+    `;
+    const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step));
+    const trigger = document.querySelector("button")!;
+    const input = document.querySelector("input")!;
+    vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
+      x: 900,
+      y: 8,
+      top: 8,
+      left: 900,
+      right: 932,
+      bottom: 40,
+      width: 32,
+      height: 32,
+      toJSON: () => ({}),
+    });
+
+    trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
+    input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
+    input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
+    input.value = "Ada";
+    input.dispatchEvent(new Event("input", { bubbles: true }));
+    input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
+    capture.dispose();
+
+    expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]);
+    expect(steps[0]).toMatchObject({
+      op: "hover",
+      target: { role: "button", name: "Open user navigation menu" },
+    });
+    expect(steps[1]).toMatchObject({
+      op: "fill",
+      target: { tag: "input" },
+      value: "Ada",
+    });
+  });
+
+  it("records hover before selecting inside the hover surface", () => {
+    document.body.innerHTML = `
+      <button type="button" aria-label="Open user navigation menu">avatar</button>
+      <div class="user-menu dropdown-menu">
+        <select aria-label="Status">
+          <option value="online">Online</option>
+          <option value="away">Away</option>
+        </select>
+      </div>
+    `;
+    const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step));
+    const trigger = document.querySelector("button")!;
+    const select = document.querySelector("select")!;
+    vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
+      x: 900,
+      y: 8,
+      top: 8,
+      left: 900,
+      right: 932,
+      bottom: 40,
+      width: 32,
+      height: 32,
+      toJSON: () => ({}),
+    });
+
+    trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
+    select.value = "away";
+    select.dispatchEvent(new Event("change", { bubbles: true }));
+    capture.dispose();
+
+    expect(steps.map((s) => s.op)).toEqual(["hover", "select"]);
+    expect(steps[0]).toMatchObject({
+      op: "hover",
+      target: { role: "button", name: "Open user navigation menu" },
+    });
+    expect(steps[1]).toMatchObject({
+      op: "select",
+      target: { role: "combobox", name: "Status" },
+      values: ["away"],
+      labels: ["Away"],
+    });
+  });
+
   it("records hover before clicking an unlabelled positioned floating surface", () => {
     document.body.innerHTML = `
       <button type="button" aria-label="Open user navigation menu">avatar</button>

apps/extension/src/content/record-capture.ts

@@ -388,18 +388,18 @@ export function startRecordCapture(
     });
   };
 
-  const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => {
-    if (!pendingHover || !(clickTarget instanceof Element)) return;
+  const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
+    if (!pendingHover || !(actionTarget instanceof Element)) return;
     if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
       pendingHover = null;
       return;
     }
-    const clickElement = resolveClickableElement(clickTarget);
-    if (clickElement === pendingHover.element) {
+    const actionElement = resolveClickableElement(actionTarget);
+    if (actionElement === pendingHover.element) {
       return;
     }
-    const relationTarget = clickElement ?? clickTarget;
-    if (!clickElement && pendingHover.element.contains(relationTarget)) {
+    const relationTarget = actionElement ?? actionTarget;
+    if (!actionElement && pendingHover.element.contains(relationTarget)) {
       return;
     }
     const relation = decideHoverSurfaceRelation(
@@ -460,6 +460,7 @@ export function startRecordCapture(
 
     const fillable = fillableFromTarget(target);
     if (fillable) {
+      emitPendingHoverBeforeAction(fillable);
       ensureFillSession(fillable);
       return;
     }
@@ -467,6 +468,7 @@ export function startRecordCapture(
     if (target instanceof Element) {
       const nearbyFillable = nearbyFillableFromSearchChrome(target);
       if (nearbyFillable) {
+        emitPendingHoverBeforeAction(nearbyFillable);
         ensureFillSession(nearbyFillable);
         return;
       }
@@ -486,7 +488,7 @@ export function startRecordCapture(
 
     commitFillSession();
     if (target instanceof Element && target.closest("select")) return;
-    emitPendingHoverBeforeClick(target);
+    emitPendingHoverBeforeAction(target);
     emitClick(event);
   };
 
@@ -528,6 +530,7 @@ export function startRecordCapture(
     commitFillSession();
     const target = eventTarget(event);
     if (target instanceof HTMLSelectElement) {
+      emitPendingHoverBeforeAction(target);
       const values = Array.from(target.selectedOptions).map((opt) => opt.value);
       const labels = Array.from(target.selectedOptions).map((opt) =>
         (opt.label || opt.textContent || opt.value).trim(),
@@ -581,6 +584,7 @@ export function startRecordCapture(
     }
     const desc = describeEventTarget(target);
     if (!desc && !event.key) return;
+    emitPendingHoverBeforeAction(target);
     emitStep({
       op: "press",
       key: event.key,

apps/extension/src/tools/__tests__/dispatcher.test.ts

@@ -456,6 +456,60 @@ describe("ToolDispatcher", () => {
     );
   });
 
+  it("limits default-target hover reassertion to the active tab", async () => {
+    vi.stubGlobal("chrome", {
+      tabs: {
+        query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
+        get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
+        sendMessage: vi.fn(async () => undefined),
+      },
+    });
+    const { transport } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 4242),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    await sessions.start("aa11");
+    const cdp = {
+      send: vi.fn(async <T>() => ({}) as T),
+      detachSession: vi.fn(async () => {}),
+      ensureNetworkCapture: vi.fn(async () => {}),
+      networkEntriesSince: vi.fn(() => ({
+        tab_id: 9,
+        entries: [],
+        next_since: 0,
+        truncated: false,
+      })),
+      setDeviceMetricsOverride: vi.fn(async () => {}),
+      clearDeviceMetricsOverride: vi.fn(async () => {}),
+      setUserAgentOverride: vi.fn(async () => {}),
+      setTouchEmulationEnabled: vi.fn(async () => {}),
+    };
+    const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
+    const helpers = dispatcher as unknown as {
+      rememberHover: (sessionId: string, result: unknown) => unknown;
+      withHoverReassert: <T>(
+        params: { session_id: string; tab_id?: number },
+        work: () => Promise<T>,
+      ) => Promise<T>;
+    };
+
+    helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
+    helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
+
+    await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
+
+    expect(cdp.send).toHaveBeenCalledTimes(1);
+    expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
+      type: "mouseMoved",
+      x: 30,
+      y: 40,
+    });
+  });
+
   it("releases the current session hover latch before navigation-style work", async () => {
     vi.stubGlobal("chrome", {
       tabs: {

apps/extension/src/tools/dispatcher.ts

@@ -58,7 +58,7 @@ import {
   type SessionStartParams,
   type SessionStopParams,
 } from "./session";
-import { chromeTabsApi } from "./shared";
+import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
 import {
   type BorrowConfirmationApprover,
   handleTabBorrow,
@@ -90,6 +90,11 @@ interface HoverLatch {
   y: number;
 }
 
+interface HoverLatchScope {
+  session_id: string;
+  tab_id?: number;
+}
+
 export interface DispatcherDeps {
   transport: Transport;
   sessions: SessionManager;
@@ -341,23 +346,24 @@ export class ToolDispatcher {
             this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
           ),
         );
-      case "tool.observe":
-        return this.withHoverReassert(req.params as ObserveParams, () =>
+      case "tool.observe": {
+        const params = req.params as ObserveParams;
+        const hoverScope = await this.resolveHoverLatchScope(params);
+        return this.withHoverReassert(params, () =>
           handleObserve(
             this.sessions,
-            req.params as ObserveParams,
+            params,
             this.cdp
               ? {
                   cdp: this.cdp,
                   tabsApi: chromeTabsCaptureApi,
-                  conditionalSurfaceProbe: !this.hasHoverLatchForRequest(
-                    req.params as ObserveParams,
-                  ),
+                  conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
                   hoverProbeBypassOverlay: bypassOverlay,
                 }
               : undefined,
           ),
         );
+      }
       case "tool.get_html":
         return handleGetHtml(
           this.sessions,
@@ -551,8 +557,8 @@ export class ToolDispatcher {
     return result;
   }
 
-  private hasHoverLatchForRequest(params: { session_id: string; tab_id?: number }): boolean {
-    return this.hoverLatchesForRequest(params).length > 0;
+  private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
+    return this.hoverLatchesForRequest(scope).length > 0;
   }
 
   private async withHoverReassert<T>(
@@ -560,12 +566,13 @@ export class ToolDispatcher {
     work: () => Promise<T>,
     options: { releaseAfter?: boolean } = {},
   ): Promise<T> {
-    await this.reassertHover(params);
+    const scope = await this.resolveHoverLatchScope(params);
+    await this.reassertHover(scope);
     try {
       return await work();
     } finally {
       if (options.releaseAfter) {
-        await this.releaseHoverLatch(params.session_id, params.tab_id);
+        await this.releaseHoverLatch(scope.session_id, scope.tab_id);
       }
     }
   }
@@ -574,10 +581,23 @@ export class ToolDispatcher {
     params: { session_id: string; tab_id?: number },
     work: () => Promise<T>,
   ): Promise<T> {
-    await this.releaseHoverLatch(params.session_id, params.tab_id);
+    const scope = await this.resolveHoverLatchScope(params);
+    await this.releaseHoverLatch(scope.session_id, scope.tab_id);
     return work();
   }
 
+  private async resolveHoverLatchScope(params: {
+    session_id: string;
+    tab_id?: number;
+  }): Promise<HoverLatchScope> {
+    if (params.tab_id !== undefined) return params;
+    const ctx = lookupSession(this.sessions, params, "hover latch");
+    if (isRpcError(ctx)) return params;
+    const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
+    if (isRpcError(target)) return params;
+    return { session_id: params.session_id, tab_id: target.tabId };
+  }
+
   private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
     return [...this.hoverLatches.values()].filter((latch) => {
       if (latch.sessionId !== params.session_id) return false;