feat(extension): emit pending hover before actions

This commit is contained in:
Ljy-0827 committed 2026-08-06 17:47:19 +08:00
1 parent 73724617f9
commit d685c75d25
4 files changed
+183 -19

No files matched your search

@@ -470,6 +470,92 @@ describe("record-capture semantic", () => {
}); });
}); });
it("records hover before filling a field inside the hover surface", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button>
<div class="user-menu dropdown-menu">
<label for="nickname">Nickname</label>
<input id="nickname" />
</div>
`;
const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const input = document.querySelector("input")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "Ada";
input.dispatchEvent(new Event("input", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
});
expect(steps[1]).toMatchObject({
op: "fill",
target: { tag: "input" },
value: "Ada",
});
});
it("records hover before selecting inside the hover surface", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button>
<div class="user-menu dropdown-menu">
<select aria-label="Status">
<option value="online">Online</option>
<option value="away">Away</option>
</select>
</div>
`;
const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const select = document.querySelector("select")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
select.value = "away";
select.dispatchEvent(new Event("change", { bubbles: true }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "select"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
});
expect(steps[1]).toMatchObject({
op: "select",
target: { role: "combobox", name: "Status" },
values: ["away"],
labels: ["Away"],
});
});
it("records hover before clicking an unlabelled positioned floating surface", () => { it("records hover before clicking an unlabelled positioned floating surface", () => {
document.body.innerHTML = ` document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button> <button type="button" aria-label="Open user navigation menu">avatar</button>
+11 -7
View File
@@ -388,18 +388,18 @@ export function startRecordCapture(
}); });
}; };
const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => { const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
if (!pendingHover || !(clickTarget instanceof Element)) return; if (!pendingHover || !(actionTarget instanceof Element)) return;
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
pendingHover = null; pendingHover = null;
return; return;
} }
const clickElement = resolveClickableElement(clickTarget); const actionElement = resolveClickableElement(actionTarget);
if (clickElement === pendingHover.element) { if (actionElement === pendingHover.element) {
return; return;
} }
const relationTarget = clickElement ?? clickTarget; const relationTarget = actionElement ?? actionTarget;
if (!clickElement && pendingHover.element.contains(relationTarget)) { if (!actionElement && pendingHover.element.contains(relationTarget)) {
return; return;
} }
const relation = decideHoverSurfaceRelation( const relation = decideHoverSurfaceRelation(
@@ -460,6 +460,7 @@ export function startRecordCapture(
const fillable = fillableFromTarget(target); const fillable = fillableFromTarget(target);
if (fillable) { if (fillable) {
emitPendingHoverBeforeAction(fillable);
ensureFillSession(fillable); ensureFillSession(fillable);
return; return;
} }
@@ -467,6 +468,7 @@ export function startRecordCapture(
if (target instanceof Element) { if (target instanceof Element) {
const nearbyFillable = nearbyFillableFromSearchChrome(target); const nearbyFillable = nearbyFillableFromSearchChrome(target);
if (nearbyFillable) { if (nearbyFillable) {
emitPendingHoverBeforeAction(nearbyFillable);
ensureFillSession(nearbyFillable); ensureFillSession(nearbyFillable);
return; return;
} }
@@ -486,7 +488,7 @@ export function startRecordCapture(
commitFillSession(); commitFillSession();
if (target instanceof Element && target.closest("select")) return; if (target instanceof Element && target.closest("select")) return;
emitPendingHoverBeforeClick(target); emitPendingHoverBeforeAction(target);
emitClick(event); emitClick(event);
}; };
@@ -528,6 +530,7 @@ export function startRecordCapture(
commitFillSession(); commitFillSession();
const target = eventTarget(event); const target = eventTarget(event);
if (target instanceof HTMLSelectElement) { if (target instanceof HTMLSelectElement) {
emitPendingHoverBeforeAction(target);
const values = Array.from(target.selectedOptions).map((opt) => opt.value); const values = Array.from(target.selectedOptions).map((opt) => opt.value);
const labels = Array.from(target.selectedOptions).map((opt) => const labels = Array.from(target.selectedOptions).map((opt) =>
(opt.label || opt.textContent || opt.value).trim(), (opt.label || opt.textContent || opt.value).trim(),
@@ -581,6 +584,7 @@ export function startRecordCapture(
} }
const desc = describeEventTarget(target); const desc = describeEventTarget(target);
if (!desc && !event.key) return; if (!desc && !event.key) return;
emitPendingHoverBeforeAction(target);
emitStep({ emitStep({
op: "press", op: "press",
key: event.key, key: event.key,
@@ -456,6 +456,60 @@ describe("ToolDispatcher", () => {
); );
}); });
it("limits default-target hover reassertion to the active tab", async () => {
vi.stubGlobal("chrome", {
tabs: {
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
sendMessage: vi.fn(async () => undefined),
},
});
const { transport } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
ensureNetworkCapture: vi.fn(async () => {}),
networkEntriesSince: vi.fn(() => ({
tab_id: 9,
entries: [],
next_since: 0,
truncated: false,
})),
setDeviceMetricsOverride: vi.fn(async () => {}),
clearDeviceMetricsOverride: vi.fn(async () => {}),
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
expect(cdp.send).toHaveBeenCalledTimes(1);
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 30,
y: 40,
});
});
it("releases the current session hover latch before navigation-style work", async () => { it("releases the current session hover latch before navigation-style work", async () => {
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
tabs: { tabs: {
+32 -12
View File
@@ -58,7 +58,7 @@ import {
type SessionStartParams, type SessionStartParams,
type SessionStopParams, type SessionStopParams,
} from "./session"; } from "./session";
import { chromeTabsApi } from "./shared"; import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
import { import {
type BorrowConfirmationApprover, type BorrowConfirmationApprover,
handleTabBorrow, handleTabBorrow,
@@ -90,6 +90,11 @@ interface HoverLatch {
y: number; y: number;
} }
interface HoverLatchScope {
session_id: string;
tab_id?: number;
}
export interface DispatcherDeps { export interface DispatcherDeps {
transport: Transport; transport: Transport;
sessions: SessionManager; sessions: SessionManager;
@@ -341,23 +346,24 @@ export class ToolDispatcher {
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
), ),
); );
case "tool.observe": case "tool.observe": {
return this.withHoverReassert(req.params as ObserveParams, () => const params = req.params as ObserveParams;
const hoverScope = await this.resolveHoverLatchScope(params);
return this.withHoverReassert(params, () =>
handleObserve( handleObserve(
this.sessions, this.sessions,
req.params as ObserveParams, params,
this.cdp this.cdp
? { ? {
cdp: this.cdp, cdp: this.cdp,
tabsApi: chromeTabsCaptureApi, tabsApi: chromeTabsCaptureApi,
conditionalSurfaceProbe: !this.hasHoverLatchForRequest( conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
req.params as ObserveParams,
),
hoverProbeBypassOverlay: bypassOverlay, hoverProbeBypassOverlay: bypassOverlay,
} }
: undefined, : undefined,
), ),
); );
}
case "tool.get_html": case "tool.get_html":
return handleGetHtml( return handleGetHtml(
this.sessions, this.sessions,
@@ -551,8 +557,8 @@ export class ToolDispatcher {
return result; return result;
} }
private hasHoverLatchForRequest(params: { session_id: string; tab_id?: number }): boolean { private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
return this.hoverLatchesForRequest(params).length > 0; return this.hoverLatchesForRequest(scope).length > 0;
} }
private async withHoverReassert<T>( private async withHoverReassert<T>(
@@ -560,12 +566,13 @@ export class ToolDispatcher {
work: () => Promise<T>, work: () => Promise<T>,
options: { releaseAfter?: boolean } = {}, options: { releaseAfter?: boolean } = {},
): Promise<T> { ): Promise<T> {
await this.reassertHover(params); const scope = await this.resolveHoverLatchScope(params);
await this.reassertHover(scope);
try { try {
return await work(); return await work();
} finally { } finally {
if (options.releaseAfter) { if (options.releaseAfter) {
await this.releaseHoverLatch(params.session_id, params.tab_id); await this.releaseHoverLatch(scope.session_id, scope.tab_id);
} }
} }
} }
@@ -574,10 +581,23 @@ export class ToolDispatcher {
params: { session_id: string; tab_id?: number }, params: { session_id: string; tab_id?: number },
work: () => Promise<T>, work: () => Promise<T>,
): Promise<T> { ): Promise<T> {
await this.releaseHoverLatch(params.session_id, params.tab_id); const scope = await this.resolveHoverLatchScope(params);
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
return work(); return work();
} }
private async resolveHoverLatchScope(params: {
session_id: string;
tab_id?: number;
}): Promise<HoverLatchScope> {
if (params.tab_id !== undefined) return params;
const ctx = lookupSession(this.sessions, params, "hover latch");
if (isRpcError(ctx)) return params;
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
if (isRpcError(target)) return params;
return { session_id: params.session_id, tab_id: target.tabId };
}
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] { private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
return [...this.hoverLatches.values()].filter((latch) => { return [...this.hoverLatches.values()].filter((latch) => {
if (latch.sessionId !== params.session_id) return false; if (latch.sessionId !== params.session_id) return false;