From eeab97bcddd25c5b11090b2cc9940852f769e485 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 7 Aug 2026 12:02:28 +0800 Subject: [PATCH] feat(extension): hover candidate pool --- .../content/__tests__/record-capture.test.ts | 237 ++++++++++++++++++ apps/extension/src/content/record-capture.ts | 171 +++++++++---- .../src/content/record-hover-surface.ts | 6 +- .../__tests__/hover-trigger-policy.test.ts | 13 + .../extension/src/lib/hover-trigger-policy.ts | 20 +- 5 files changed, 391 insertions(+), 56 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 26280f5..16059d0 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -8,6 +8,44 @@ vi.stubGlobal("chrome", { }, }); +function mockRect( + el: Element, + rect: { left: number; top: number; width: number; height: number }, +): void { + vi.spyOn(el, "getBoundingClientRect").mockReturnValue({ + x: rect.left, + y: rect.top, + top: rect.top, + left: rect.left, + right: rect.left + rect.width, + bottom: rect.top + rect.height, + width: rect.width, + height: rect.height, + toJSON: () => ({}), + }); +} + +function mockHoverStyle(pointerElements: Element[]): void { + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: pointerElements.includes(el) ? "pointer" : "", + display: "block", + pointerEvents: "auto", + position: "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); +} + +function mouseOver(el: Element): void { + el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); +} + +function click(el: Element): void { + el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); +} + describe("handleRecordContentMessage stop/cancel", () => { it("ignores STOP when no recording is active", () => { const dispose = vi.fn(); @@ -381,6 +419,205 @@ describe("record-capture semantic", () => { }); }); + it("keeps the menu opener hover when moving over an accepted menu item", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("li")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "li", name: "文档C+D" }, + }); + }); + + it("does not latch a plain div action item inside a hover surface", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("keeps the opener hover when a surface action is clicked after the short latch window", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T03:43:36.000Z")); + try { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu-div-action-slow", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + vi.setSystemTime(new Date("2026-08-07T03:43:47.000Z")); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("infers a non-policy opener hover from the later surface action", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("does not record a strong-looking surface item when clicking that item itself", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("li")!; + const itemInner = document.querySelector("li div")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item, itemInner]); + + mouseOver(trigger); + mouseOver(item); + click(itemInner); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("records cascaded hover triggers before clicking inside the final surface", () => { + document.body.innerHTML = ` + + + + `; + const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step)); + const trigger = document.querySelector("button[aria-label]")!; + const nestedTrigger = document.querySelector("li")!; + const finalAction = document.querySelector(".template-submenu button")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mockHoverStyle([nestedTrigger, finalAction]); + + mouseOver(trigger); + mouseOver(nestedTrigger); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "更多模板" }, + }); + expect(steps[2]).toMatchObject({ + op: "click", + target: { role: "button", name: "Blank doc" }, + }); + }); + it("records avatar div hover with a semantic image target", () => { document.body.innerHTML = `
diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 77123d6..530085b 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -9,6 +9,8 @@ import { evaluateHoverTrigger, type HoverTriggerDecision, type HoverTriggerRect, + hasDirectHoverInteractiveSignal, + hasStrongHoverExpansionSignal, } from "@/lib/hover-trigger-policy"; import { isRecordCancelMessage, @@ -26,7 +28,7 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; -import { decideHoverSurfaceRelation } from "./record-hover-surface"; +import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -61,18 +63,21 @@ interface FillSession { lastValue: string; } -interface PendingHover { +interface HoverCandidate { element: Element; target: TargetDescriptor; recordedAt: number; score: number; - reasons: string[]; + hasExpansionSignal: boolean; + eligible: boolean; } type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; +const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000; const HOVER_REPLACE_SCORE_MARGIN = 50; +const HOVER_CANDIDATE_LIMIT = 24; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -213,13 +218,10 @@ function normalizeHoverTarget( return desc; } -function evaluateHoverTriggerElement( - el: Element, - desc: TargetDescriptor, -): HoverTriggerDecision | null { +function hoverTriggerSignals(el: Element, desc: TargetDescriptor) { if (!(el instanceof HTMLElement)) return null; const style = hoverTriggerStyle(el); - return evaluateHoverTrigger({ + return { tag: el.tagName.toLowerCase(), role: desc.role, label: desc.name, @@ -228,29 +230,35 @@ function evaluateHoverTriggerElement( cursor: style.cursor, pointerEvents: style.pointerEvents, hasGraphicDescendant: hasHoverGraphicDescendant(el), - }); + }; } -function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { +function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null { if (!(target instanceof Element)) return null; const element = resolveHoverElement(target); if (!element) return null; const desc = describeTarget(element); - const decision = evaluateHoverTriggerElement(element, desc); - if (!decision?.eligible) return null; + const signals = hoverTriggerSignals(element, desc); + if (!signals) return null; + const decision = evaluateHoverTrigger(signals); + const hasExpansionSignal = hasStrongHoverExpansionSignal(signals); + const hasDirectSignal = hasDirectHoverInteractiveSignal(signals); + if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null; + if (!decision.eligible && !desc.name && !desc.role) return null; const normalizedTarget = normalizeHoverTarget(element, desc, decision); return { element, target: normalizedTarget, recordedAt: Date.now(), score: decision.score, - reasons: decision.reasons, + hasExpansionSignal, + eligible: decision.eligible, }; } -function shouldReplacePendingHover( - current: PendingHover, - next: PendingHover, +function shouldReplaceHoverCandidate( + current: HoverCandidate, + next: HoverCandidate, now: number, ): boolean { if (next.element === current.element) return false; @@ -310,7 +318,9 @@ export function startRecordCapture( let composing = false; let lastUrl = location.href; let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined; - let pendingHover: PendingHover | null = null; + let pendingHover: HoverCandidate | null = null; + const recentHoverCandidates: HoverCandidate[] = []; + const emittedHoverElements = new WeakSet(); let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -376,6 +386,68 @@ export function startRecordCapture( }); }; + const rememberHoverCandidate = (hover: HoverCandidate) => { + const duplicate = recentHoverCandidates.findIndex( + (candidate) => candidate.element === hover.element, + ); + if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1); + recentHoverCandidates.push(hover); + if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) { + recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT); + } + }; + + const actionIsOnHoverTriggerItself = ( + hover: HoverCandidate, + actionElement: Element | null, + relationTarget: Element, + ): boolean => { + const target = actionElement ?? relationTarget; + if (target === hover.element) return true; + if (!hover.element.contains(target)) return false; + const surface = closestRecognizedHoverSurface(target); + return !surface || surface === hover.element || !hover.element.contains(surface); + }; + + const hoverRelatedToAction = ( + hover: HoverCandidate, + relationTarget: Element, + now: number, + ): boolean => { + if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false; + return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related; + }; + + const emitHoverStep = (hover: HoverCandidate) => { + if (emittedHoverElements.has(hover.element)) return; + emitStep({ + op: "hover", + target: hover.target, + }); + emittedHoverElements.add(hover.element); + }; + + const findHoverForAction = ( + actionElement: Element | null, + relationTarget: Element, + now: number, + ): HoverCandidate | null => { + const candidates = pendingHover + ? [ + ...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element), + pendingHover, + ] + : recentHoverCandidates; + for (let index = candidates.length - 1; index >= 0; index -= 1) { + const hover = candidates[index]; + if (!hover) continue; + if (emittedHoverElements.has(hover.element)) continue; + if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue; + if (hoverRelatedToAction(hover, relationTarget, now)) return hover; + } + return null; + }; + const emitClick = (event: MouseEvent) => { // Only record clicks an LLM can re-identify (named interactive controls). const target = describeEventTarget(eventTarget(event)); @@ -388,51 +460,52 @@ export function startRecordCapture( }); }; - const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => { - if (!pendingHover || !(actionTarget instanceof Element)) return; - if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { - pendingHover = null; - return; - } + const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => { + if (!(actionTarget instanceof Element)) return; const actionElement = resolveClickableElement(actionTarget); - if (actionElement === pendingHover.element) { - return; - } const relationTarget = actionElement ?? actionTarget; - if (!actionElement && pendingHover.element.contains(relationTarget)) { - return; - } - const relation = decideHoverSurfaceRelation( - { - triggerElement: pendingHover.element, - }, - relationTarget, - ); - if (!relation.related) { + const hover = findHoverForAction(actionElement, relationTarget, Date.now()); + if (!hover) { pendingHover = null; return; } - emitStep({ - op: "hover", - target: pendingHover.target, - }); + emitHoverStep(hover); pendingHover = null; }; const onMouseOver = (event: MouseEvent) => { const target = eventTarget(event); if (isOverlayTarget(target)) return; + const now = Date.now(); if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) { - if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return; + if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return; pendingHover = null; } - const hover = hoverTargetFromEvent(target); + const candidate = hoverCandidateFromEvent(target); + if (candidate) rememberHoverCandidate(candidate); + const hover = candidate?.eligible ? candidate : null; + if (pendingHover && target instanceof Element) { + const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target); + if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) { + if (!hover || !hover.hasExpansionSignal) return; + if ( + hover.element === pendingHover.element || + pendingHover.element.contains(hover.element) + ) { + return; + } + emitHoverStep(pendingHover); + pendingHover = hover; + rememberHoverCandidate(hover); + return; + } + } if (!hover) return; - const now = Date.now(); - if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) { + if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) { return; } pendingHover = hover; + rememberHoverCandidate(hover); }; const onClick = (event: MouseEvent) => { @@ -464,7 +537,7 @@ export function startRecordCapture( const fillable = fillableFromTarget(target); if (fillable) { - emitPendingHoverBeforeAction(fillable); + emitHoverCandidateBeforeAction(fillable); ensureFillSession(fillable); return; } @@ -472,7 +545,7 @@ export function startRecordCapture( if (target instanceof Element) { const nearbyFillable = nearbyFillableFromSearchChrome(target); if (nearbyFillable) { - emitPendingHoverBeforeAction(nearbyFillable); + emitHoverCandidateBeforeAction(nearbyFillable); ensureFillSession(nearbyFillable); return; } @@ -492,7 +565,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; - emitPendingHoverBeforeAction(target); + emitHoverCandidateBeforeAction(target); emitClick(event); }; @@ -534,7 +607,7 @@ export function startRecordCapture( commitFillSession(); const target = eventTarget(event); if (target instanceof HTMLSelectElement) { - emitPendingHoverBeforeAction(target); + emitHoverCandidateBeforeAction(target); const values = Array.from(target.selectedOptions).map((opt) => opt.value); const labels = Array.from(target.selectedOptions).map((opt) => (opt.label || opt.textContent || opt.value).trim(), @@ -588,7 +661,7 @@ export function startRecordCapture( } const desc = describeEventTarget(target); if (!desc && !event.key) return; - emitPendingHoverBeforeAction(target); + emitHoverCandidateBeforeAction(target); emitStep({ op: "press", key: event.key, diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index c00d6fc..c6fbd43 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -50,8 +50,8 @@ function isHoverSurfaceElement(el: Element): boolean { return false; } -function closestHoverSurface(clickElement: Element): Element | null { - let node: Element | null = clickElement; +export function closestRecognizedHoverSurface(element: Element): Element | null { + let node: Element | null = element; let depth = 0; while (node && node !== document.body && node !== document.documentElement && depth < 10) { if (isHoverSurfaceElement(node)) return node; @@ -115,7 +115,7 @@ export function decideHoverSurfaceRelation( context: HoverSurfaceContext, clickElement: Element, ): HoverSurfaceDecision { - const clickSurface = closestHoverSurface(clickElement); + const clickSurface = closestRecognizedHoverSurface(clickElement); if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) { return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; } diff --git a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts index d3b3dce..dd73c24 100644 --- a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts +++ b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts @@ -77,6 +77,19 @@ describe("evaluateHoverTrigger", () => { expect(decision.eligible).toBe(false); }); + it("rejects labelled dropdown list items as hover triggers", () => { + const decision = evaluateHoverTrigger({ + tag: "li", + label: "文档C+D", + attrs: { class: "dropdown-item", tabindex: "0" }, + rect: { x: 820, y: 48, w: 160, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + it("rejects ordinary controls without hover popup signals", () => { const decision = evaluateHoverTrigger({ tag: "button", diff --git a/apps/extension/src/lib/hover-trigger-policy.ts b/apps/extension/src/lib/hover-trigger-policy.ts index 8701c89..23922b4 100644 --- a/apps/extension/src/lib/hover-trigger-policy.ts +++ b/apps/extension/src/lib/hover-trigger-policy.ts @@ -26,6 +26,7 @@ export interface HoverTriggerDecision { const HOVER_TRIGGER_MIN_SCORE = 45; const HOVER_POPUP_SIGNAL_RE = /(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/; +const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/; function attr(signals: HoverTriggerSignals, name: string): string | undefined { return signals.attrs?.[name.toLowerCase()]; @@ -59,6 +60,14 @@ export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean { return HOVER_POPUP_SIGNAL_RE.test(haystack); } +export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean { + return ( + signals.cssHoverMatch === true || + attr(signals, "aria-haspopup") !== undefined || + (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false" + ); +} + export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean { const tag = signals.tag.toLowerCase(); const role = (signals.role ?? "").toLowerCase(); @@ -104,7 +113,12 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger const directInteractive = hasDirectHoverInteractiveSignal(signals); const popupSignal = hasHoverPopupSignal(signals); const isTextNavigationItem = - (tag === "a" || role === "link" || role.startsWith("menuitem")) && + (tag === "a" || + tag === "li" || + role === "link" || + role === "option" || + role.startsWith("menuitem") || + HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) && !graphic && !!label && label.toLowerCase() !== "image"; @@ -118,9 +132,7 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger const surfaceTriggerEvidence = (popupSignal && !isTextNavigationItem) || compactTopbarIcon || - signals.cssHoverMatch === true || - attr(signals, "aria-haspopup") !== undefined || - (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"; + hasStrongHoverExpansionSignal(signals); if (!directInteractive || !surfaceTriggerEvidence) { return { eligible: false, score: 0, reasons: [] }; }