diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index e59cda0..26280f5 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -470,6 +470,92 @@ describe("record-capture semantic", () => { }); }); + it("records hover before filling a field inside the hover surface", () => { + document.body.innerHTML = ` + +
+ `; + const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const input = document.querySelector("input")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "Ada"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "fill", + target: { tag: "input" }, + value: "Ada", + }); + }); + + it("records hover before selecting inside the hover surface", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const select = document.querySelector("select")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + select.value = "away"; + select.dispatchEvent(new Event("change", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "select"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "select", + target: { role: "combobox", name: "Status" }, + values: ["away"], + labels: ["Away"], + }); + }); + it("records hover before clicking an unlabelled positioned floating surface", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index c879916..b21edcb 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -388,18 +388,18 @@ export function startRecordCapture( }); }; - const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => { - if (!pendingHover || !(clickTarget instanceof Element)) return; + const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => { + if (!pendingHover || !(actionTarget instanceof Element)) return; if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { pendingHover = null; return; } - const clickElement = resolveClickableElement(clickTarget); - if (clickElement === pendingHover.element) { + const actionElement = resolveClickableElement(actionTarget); + if (actionElement === pendingHover.element) { return; } - const relationTarget = clickElement ?? clickTarget; - if (!clickElement && pendingHover.element.contains(relationTarget)) { + const relationTarget = actionElement ?? actionTarget; + if (!actionElement && pendingHover.element.contains(relationTarget)) { return; } const relation = decideHoverSurfaceRelation( @@ -460,6 +460,7 @@ export function startRecordCapture( const fillable = fillableFromTarget(target); if (fillable) { + emitPendingHoverBeforeAction(fillable); ensureFillSession(fillable); return; } @@ -467,6 +468,7 @@ export function startRecordCapture( if (target instanceof Element) { const nearbyFillable = nearbyFillableFromSearchChrome(target); if (nearbyFillable) { + emitPendingHoverBeforeAction(nearbyFillable); ensureFillSession(nearbyFillable); return; } @@ -486,7 +488,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; - emitPendingHoverBeforeClick(target); + emitPendingHoverBeforeAction(target); emitClick(event); }; @@ -528,6 +530,7 @@ export function startRecordCapture( commitFillSession(); const target = eventTarget(event); if (target instanceof HTMLSelectElement) { + emitPendingHoverBeforeAction(target); const values = Array.from(target.selectedOptions).map((opt) => opt.value); const labels = Array.from(target.selectedOptions).map((opt) => (opt.label || opt.textContent || opt.value).trim(), @@ -581,6 +584,7 @@ export function startRecordCapture( } const desc = describeEventTarget(target); if (!desc && !event.key) return; + emitPendingHoverBeforeAction(target); emitStep({ op: "press", key: event.key, diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index c4f7a86..36badac 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -456,6 +456,60 @@ describe("ToolDispatcher", () => { ); }); + it("limits default-target hover reassertion to the active tab", async () => { + vi.stubGlobal("chrome", { + tabs: { + query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]), + get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })), + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + await sessions.start("aa11"); + const cdp = { + send: vi.fn(async