From d685c75d25a769f1883688a2f8108df241da08d5 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 17:47:19 +0800 Subject: [PATCH] feat(extension): emit pending hover before actions --- .../content/__tests__/record-capture.test.ts | 86 +++++++++++++++++++ apps/extension/src/content/record-capture.ts | 18 ++-- .../src/tools/__tests__/dispatcher.test.ts | 54 ++++++++++++ apps/extension/src/tools/dispatcher.ts | 44 +++++++--- 4 files changed, 183 insertions(+), 19 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index e59cda0..26280f5 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -470,6 +470,92 @@ describe("record-capture semantic", () => { }); }); + it("records hover before filling a field inside the hover surface", () => { + document.body.innerHTML = ` + +
+ + +
+ `; + const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const input = document.querySelector("input")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "Ada"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "fill", + target: { tag: "input" }, + value: "Ada", + }); + }); + + it("records hover before selecting inside the hover surface", () => { + document.body.innerHTML = ` + +
+ +
+ `; + const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const select = document.querySelector("select")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + select.value = "away"; + select.dispatchEvent(new Event("change", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "select"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "select", + target: { role: "combobox", name: "Status" }, + values: ["away"], + labels: ["Away"], + }); + }); + it("records hover before clicking an unlabelled positioned floating surface", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index c879916..b21edcb 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -388,18 +388,18 @@ export function startRecordCapture( }); }; - const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => { - if (!pendingHover || !(clickTarget instanceof Element)) return; + const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => { + if (!pendingHover || !(actionTarget instanceof Element)) return; if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { pendingHover = null; return; } - const clickElement = resolveClickableElement(clickTarget); - if (clickElement === pendingHover.element) { + const actionElement = resolveClickableElement(actionTarget); + if (actionElement === pendingHover.element) { return; } - const relationTarget = clickElement ?? clickTarget; - if (!clickElement && pendingHover.element.contains(relationTarget)) { + const relationTarget = actionElement ?? actionTarget; + if (!actionElement && pendingHover.element.contains(relationTarget)) { return; } const relation = decideHoverSurfaceRelation( @@ -460,6 +460,7 @@ export function startRecordCapture( const fillable = fillableFromTarget(target); if (fillable) { + emitPendingHoverBeforeAction(fillable); ensureFillSession(fillable); return; } @@ -467,6 +468,7 @@ export function startRecordCapture( if (target instanceof Element) { const nearbyFillable = nearbyFillableFromSearchChrome(target); if (nearbyFillable) { + emitPendingHoverBeforeAction(nearbyFillable); ensureFillSession(nearbyFillable); return; } @@ -486,7 +488,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; - emitPendingHoverBeforeClick(target); + emitPendingHoverBeforeAction(target); emitClick(event); }; @@ -528,6 +530,7 @@ export function startRecordCapture( commitFillSession(); const target = eventTarget(event); if (target instanceof HTMLSelectElement) { + emitPendingHoverBeforeAction(target); const values = Array.from(target.selectedOptions).map((opt) => opt.value); const labels = Array.from(target.selectedOptions).map((opt) => (opt.label || opt.textContent || opt.value).trim(), @@ -581,6 +584,7 @@ export function startRecordCapture( } const desc = describeEventTarget(target); if (!desc && !event.key) return; + emitPendingHoverBeforeAction(target); emitStep({ op: "press", key: event.key, diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index c4f7a86..36badac 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -456,6 +456,60 @@ describe("ToolDispatcher", () => { ); }); + it("limits default-target hover reassertion to the active tab", async () => { + vi.stubGlobal("chrome", { + tabs: { + query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]), + get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })), + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + await sessions.start("aa11"); + const cdp = { + send: vi.fn(async () => ({}) as T), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 9, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + }; + + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 }); + + await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({})); + + expect(cdp.send).toHaveBeenCalledTimes(1); + expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 30, + y: 40, + }); + }); + it("releases the current session hover latch before navigation-style work", async () => { vi.stubGlobal("chrome", { tabs: { diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 4ca8297..389c9d3 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -58,7 +58,7 @@ import { type SessionStartParams, type SessionStopParams, } from "./session"; -import { chromeTabsApi } from "./shared"; +import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared"; import { type BorrowConfirmationApprover, handleTabBorrow, @@ -90,6 +90,11 @@ interface HoverLatch { y: number; } +interface HoverLatchScope { + session_id: string; + tab_id?: number; +} + export interface DispatcherDeps { transport: Transport; sessions: SessionManager; @@ -341,23 +346,24 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, ), ); - case "tool.observe": - return this.withHoverReassert(req.params as ObserveParams, () => + case "tool.observe": { + const params = req.params as ObserveParams; + const hoverScope = await this.resolveHoverLatchScope(params); + return this.withHoverReassert(params, () => handleObserve( this.sessions, - req.params as ObserveParams, + params, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, - conditionalSurfaceProbe: !this.hasHoverLatchForRequest( - req.params as ObserveParams, - ), + conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope), hoverProbeBypassOverlay: bypassOverlay, } : undefined, ), ); + } case "tool.get_html": return handleGetHtml( this.sessions, @@ -551,8 +557,8 @@ export class ToolDispatcher { return result; } - private hasHoverLatchForRequest(params: { session_id: string; tab_id?: number }): boolean { - return this.hoverLatchesForRequest(params).length > 0; + private hasHoverLatchForScope(scope: HoverLatchScope): boolean { + return this.hoverLatchesForRequest(scope).length > 0; } private async withHoverReassert( @@ -560,12 +566,13 @@ export class ToolDispatcher { work: () => Promise, options: { releaseAfter?: boolean } = {}, ): Promise { - await this.reassertHover(params); + const scope = await this.resolveHoverLatchScope(params); + await this.reassertHover(scope); try { return await work(); } finally { if (options.releaseAfter) { - await this.releaseHoverLatch(params.session_id, params.tab_id); + await this.releaseHoverLatch(scope.session_id, scope.tab_id); } } } @@ -574,10 +581,23 @@ export class ToolDispatcher { params: { session_id: string; tab_id?: number }, work: () => Promise, ): Promise { - await this.releaseHoverLatch(params.session_id, params.tab_id); + const scope = await this.resolveHoverLatchScope(params); + await this.releaseHoverLatch(scope.session_id, scope.tab_id); return work(); } + private async resolveHoverLatchScope(params: { + session_id: string; + tab_id?: number; + }): Promise { + if (params.tab_id !== undefined) return params; + const ctx = lookupSession(this.sessions, params, "hover latch"); + if (isRpcError(ctx)) return params; + const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi); + if (isRpcError(target)) return params; + return { session_id: params.session_id, tab_id: target.tabId }; + } + private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] { return [...this.hoverLatches.values()].filter((latch) => { if (latch.sessionId !== params.session_id) return false;