feat(extension): hover candidate pool

This commit is contained in:
Ljy-0827
2026-08-07 12:02:28 +08:00
parent 6c610faed9
commit eeab97bcdd
5 changed files with 391 additions and 56 deletions
@@ -8,6 +8,44 @@ vi.stubGlobal("chrome", {
},
});
function mockRect(
el: Element,
rect: { left: number; top: number; width: number; height: number },
): void {
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
x: rect.left,
y: rect.top,
top: rect.top,
left: rect.left,
right: rect.left + rect.width,
bottom: rect.top + rect.height,
width: rect.width,
height: rect.height,
toJSON: () => ({}),
});
}
function mockHoverStyle(pointerElements: Element[]): void {
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
const style = {
cursor: pointerElements.includes(el) ? "pointer" : "",
display: "block",
pointerEvents: "auto",
position: "static",
visibility: "visible",
} as CSSStyleDeclaration;
return style;
});
}
function mouseOver(el: Element): void {
el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
}
function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
}
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
@@ -381,6 +419,205 @@ describe("record-capture semantic", () => {
});
});
it("keeps the menu opener hover when moving over an accepted menu item", () => {
document.body.innerHTML = `
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
<ul class="create-menu dropdown-menu">
<li class="dropdown-item" tabindex="0">文档C+D</li>
</ul>
`;
const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const item = document.querySelector("li")!;
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
mockHoverStyle([item]);
mouseOver(trigger);
mouseOver(item);
click(item);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "新建" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { tag: "li", name: "文档C+D" },
});
});
it("does not latch a plain div action item inside a hover surface", () => {
document.body.innerHTML = `
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
<div class="create-menu dropdown-menu">
<div class="tg-menu-item" tabindex="0">文档C+D</div>
</div>
`;
const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const item = document.querySelector(".tg-menu-item")!;
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
mockHoverStyle([item]);
mouseOver(trigger);
mouseOver(item);
click(item);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "新建" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { tag: "div", name: "文档C+D" },
});
});
it("keeps the opener hover when a surface action is clicked after the short latch window", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-07T03:43:36.000Z"));
try {
document.body.innerHTML = `
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
<div class="create-menu dropdown-menu">
<div class="tg-menu-item" tabindex="0">文档C+D</div>
</div>
`;
const capture = startRecordCapture("rec-hover-menu-div-action-slow", (step) =>
steps.push(step),
);
const trigger = document.querySelector("button")!;
const item = document.querySelector(".tg-menu-item")!;
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
mockHoverStyle([item]);
mouseOver(trigger);
vi.setSystemTime(new Date("2026-08-07T03:43:47.000Z"));
mouseOver(item);
click(item);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "新建" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { tag: "div", name: "文档C+D" },
});
} finally {
vi.useRealTimers();
}
});
it("infers a non-policy opener hover from the later surface action", () => {
document.body.innerHTML = `
<button type="button">新建</button>
<div class="create-menu dropdown-menu">
<div class="tg-menu-item" tabindex="0">文档C+D</div>
</div>
`;
const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const item = document.querySelector(".tg-menu-item")!;
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
mockHoverStyle([trigger, item]);
mouseOver(trigger);
mouseOver(item);
click(item);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "新建" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { tag: "div", name: "文档C+D" },
});
});
it("does not record a strong-looking surface item when clicking that item itself", () => {
document.body.innerHTML = `
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
<div class="create-menu dropdown-menu">
<li tabindex="0" aria-expanded="false"><div>文档C+D</div></li>
</div>
`;
const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const item = document.querySelector("li")!;
const itemInner = document.querySelector("li div")!;
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
mockHoverStyle([item, itemInner]);
mouseOver(trigger);
mouseOver(item);
click(itemInner);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "新建" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { tag: "div", name: "文档C+D" },
});
});
it("records cascaded hover triggers before clicking inside the final surface", () => {
document.body.innerHTML = `
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
<ul class="create-menu dropdown-menu">
<li class="dropdown-item" tabindex="0" aria-haspopup="menu">更多模板</li>
</ul>
<div role="menu" class="template-submenu">
<button type="button">Blank doc</button>
</div>
`;
const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step));
const trigger = document.querySelector("button[aria-label]")!;
const nestedTrigger = document.querySelector("li")!;
const finalAction = document.querySelector(".template-submenu button")!;
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 });
mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 });
mockHoverStyle([nestedTrigger, finalAction]);
mouseOver(trigger);
mouseOver(nestedTrigger);
click(finalAction);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "新建" },
});
expect(steps[1]).toMatchObject({
op: "hover",
target: { tag: "li", name: "更多模板" },
});
expect(steps[2]).toMatchObject({
op: "click",
target: { role: "button", name: "Blank doc" },
});
});
it("records avatar div hover with a semantic image target", () => {
document.body.innerHTML = `
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
+122 -49
View File
@@ -9,6 +9,8 @@ import {
evaluateHoverTrigger,
type HoverTriggerDecision,
type HoverTriggerRect,
hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy";
import {
isRecordCancelMessage,
@@ -26,7 +28,7 @@ import {
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import { decideHoverSurfaceRelation } from "./record-hover-surface";
import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
@@ -61,18 +63,21 @@ interface FillSession {
lastValue: string;
}
interface PendingHover {
interface HoverCandidate {
element: Element;
target: TargetDescriptor;
recordedAt: number;
score: number;
reasons: string[];
hasExpansionSignal: boolean;
eligible: boolean;
}
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
const HOVER_REPLACE_SCORE_MARGIN = 50;
const HOVER_CANDIDATE_LIMIT = 24;
function eventTarget(event: Event): EventTarget | null {
return event.composedPath()[0] ?? event.target;
@@ -213,13 +218,10 @@ function normalizeHoverTarget(
return desc;
}
function evaluateHoverTriggerElement(
el: Element,
desc: TargetDescriptor,
): HoverTriggerDecision | null {
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return evaluateHoverTrigger({
return {
tag: el.tagName.toLowerCase(),
role: desc.role,
label: desc.name,
@@ -228,29 +230,35 @@ function evaluateHoverTriggerElement(
cursor: style.cursor,
pointerEvents: style.pointerEvents,
hasGraphicDescendant: hasHoverGraphicDescendant(el),
});
};
}
function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
if (!(target instanceof Element)) return null;
const element = resolveHoverElement(target);
if (!element) return null;
const desc = describeTarget(element);
const decision = evaluateHoverTriggerElement(element, desc);
if (!decision?.eligible) return null;
const signals = hoverTriggerSignals(element, desc);
if (!signals) return null;
const decision = evaluateHoverTrigger(signals);
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
if (!decision.eligible && !desc.name && !desc.role) return null;
const normalizedTarget = normalizeHoverTarget(element, desc, decision);
return {
element,
target: normalizedTarget,
recordedAt: Date.now(),
score: decision.score,
reasons: decision.reasons,
hasExpansionSignal,
eligible: decision.eligible,
};
}
function shouldReplacePendingHover(
current: PendingHover,
next: PendingHover,
function shouldReplaceHoverCandidate(
current: HoverCandidate,
next: HoverCandidate,
now: number,
): boolean {
if (next.element === current.element) return false;
@@ -310,7 +318,9 @@ export function startRecordCapture(
let composing = false;
let lastUrl = location.href;
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
let pendingHover: PendingHover | null = null;
let pendingHover: HoverCandidate | null = null;
const recentHoverCandidates: HoverCandidate[] = [];
const emittedHoverElements = new WeakSet<Element>();
let generatedControlClick: Element | null = null;
let navigationActionPending = false;
let navigationActionVersion = 0;
@@ -376,6 +386,68 @@ export function startRecordCapture(
});
};
const rememberHoverCandidate = (hover: HoverCandidate) => {
const duplicate = recentHoverCandidates.findIndex(
(candidate) => candidate.element === hover.element,
);
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
recentHoverCandidates.push(hover);
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
}
};
const actionIsOnHoverTriggerItself = (
hover: HoverCandidate,
actionElement: Element | null,
relationTarget: Element,
): boolean => {
const target = actionElement ?? relationTarget;
if (target === hover.element) return true;
if (!hover.element.contains(target)) return false;
const surface = closestRecognizedHoverSurface(target);
return !surface || surface === hover.element || !hover.element.contains(surface);
};
const hoverRelatedToAction = (
hover: HoverCandidate,
relationTarget: Element,
now: number,
): boolean => {
if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false;
return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related;
};
const emitHoverStep = (hover: HoverCandidate) => {
if (emittedHoverElements.has(hover.element)) return;
emitStep({
op: "hover",
target: hover.target,
});
emittedHoverElements.add(hover.element);
};
const findHoverForAction = (
actionElement: Element | null,
relationTarget: Element,
now: number,
): HoverCandidate | null => {
const candidates = pendingHover
? [
...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element),
pendingHover,
]
: recentHoverCandidates;
for (let index = candidates.length - 1; index >= 0; index -= 1) {
const hover = candidates[index];
if (!hover) continue;
if (emittedHoverElements.has(hover.element)) continue;
if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue;
if (hoverRelatedToAction(hover, relationTarget, now)) return hover;
}
return null;
};
const emitClick = (event: MouseEvent) => {
// Only record clicks an LLM can re-identify (named interactive controls).
const target = describeEventTarget(eventTarget(event));
@@ -388,51 +460,52 @@ export function startRecordCapture(
});
};
const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
if (!pendingHover || !(actionTarget instanceof Element)) return;
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
pendingHover = null;
return;
}
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
if (!(actionTarget instanceof Element)) return;
const actionElement = resolveClickableElement(actionTarget);
if (actionElement === pendingHover.element) {
return;
}
const relationTarget = actionElement ?? actionTarget;
if (!actionElement && pendingHover.element.contains(relationTarget)) {
return;
}
const relation = decideHoverSurfaceRelation(
{
triggerElement: pendingHover.element,
},
relationTarget,
);
if (!relation.related) {
const hover = findHoverForAction(actionElement, relationTarget, Date.now());
if (!hover) {
pendingHover = null;
return;
}
emitStep({
op: "hover",
target: pendingHover.target,
});
emitHoverStep(hover);
pendingHover = null;
};
const onMouseOver = (event: MouseEvent) => {
const target = eventTarget(event);
if (isOverlayTarget(target)) return;
const now = Date.now();
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
pendingHover = null;
}
const hover = hoverTargetFromEvent(target);
const candidate = hoverCandidateFromEvent(target);
if (candidate) rememberHoverCandidate(candidate);
const hover = candidate?.eligible ? candidate : null;
if (pendingHover && target instanceof Element) {
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
if (!hover || !hover.hasExpansionSignal) return;
if (
hover.element === pendingHover.element ||
pendingHover.element.contains(hover.element)
) {
return;
}
emitHoverStep(pendingHover);
pendingHover = hover;
rememberHoverCandidate(hover);
return;
}
}
if (!hover) return;
const now = Date.now();
if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) {
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
return;
}
pendingHover = hover;
rememberHoverCandidate(hover);
};
const onClick = (event: MouseEvent) => {
@@ -464,7 +537,7 @@ export function startRecordCapture(
const fillable = fillableFromTarget(target);
if (fillable) {
emitPendingHoverBeforeAction(fillable);
emitHoverCandidateBeforeAction(fillable);
ensureFillSession(fillable);
return;
}
@@ -472,7 +545,7 @@ export function startRecordCapture(
if (target instanceof Element) {
const nearbyFillable = nearbyFillableFromSearchChrome(target);
if (nearbyFillable) {
emitPendingHoverBeforeAction(nearbyFillable);
emitHoverCandidateBeforeAction(nearbyFillable);
ensureFillSession(nearbyFillable);
return;
}
@@ -492,7 +565,7 @@ export function startRecordCapture(
commitFillSession();
if (target instanceof Element && target.closest("select")) return;
emitPendingHoverBeforeAction(target);
emitHoverCandidateBeforeAction(target);
emitClick(event);
};
@@ -534,7 +607,7 @@ export function startRecordCapture(
commitFillSession();
const target = eventTarget(event);
if (target instanceof HTMLSelectElement) {
emitPendingHoverBeforeAction(target);
emitHoverCandidateBeforeAction(target);
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
const labels = Array.from(target.selectedOptions).map((opt) =>
(opt.label || opt.textContent || opt.value).trim(),
@@ -588,7 +661,7 @@ export function startRecordCapture(
}
const desc = describeEventTarget(target);
if (!desc && !event.key) return;
emitPendingHoverBeforeAction(target);
emitHoverCandidateBeforeAction(target);
emitStep({
op: "press",
key: event.key,
@@ -50,8 +50,8 @@ function isHoverSurfaceElement(el: Element): boolean {
return false;
}
function closestHoverSurface(clickElement: Element): Element | null {
let node: Element | null = clickElement;
export function closestRecognizedHoverSurface(element: Element): Element | null {
let node: Element | null = element;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
if (isHoverSurfaceElement(node)) return node;
@@ -115,7 +115,7 @@ export function decideHoverSurfaceRelation(
context: HoverSurfaceContext,
clickElement: Element,
): HoverSurfaceDecision {
const clickSurface = closestHoverSurface(clickElement);
const clickSurface = closestRecognizedHoverSurface(clickElement);
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
}
@@ -77,6 +77,19 @@ describe("evaluateHoverTrigger", () => {
expect(decision.eligible).toBe(false);
});
it("rejects labelled dropdown list items as hover triggers", () => {
const decision = evaluateHoverTrigger({
tag: "li",
label: "文档C+D",
attrs: { class: "dropdown-item", tabindex: "0" },
rect: { x: 820, y: 48, w: 160, h: 32 },
cursor: "pointer",
pointerEvents: "auto",
});
expect(decision.eligible).toBe(false);
});
it("rejects ordinary controls without hover popup signals", () => {
const decision = evaluateHoverTrigger({
tag: "button",
+16 -4
View File
@@ -26,6 +26,7 @@ export interface HoverTriggerDecision {
const HOVER_TRIGGER_MIN_SCORE = 45;
const HOVER_POPUP_SIGNAL_RE =
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
return signals.attrs?.[name.toLowerCase()];
@@ -59,6 +60,14 @@ export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
return HOVER_POPUP_SIGNAL_RE.test(haystack);
}
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
return (
signals.cssHoverMatch === true ||
attr(signals, "aria-haspopup") !== undefined ||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
);
}
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
const tag = signals.tag.toLowerCase();
const role = (signals.role ?? "").toLowerCase();
@@ -104,7 +113,12 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger
const directInteractive = hasDirectHoverInteractiveSignal(signals);
const popupSignal = hasHoverPopupSignal(signals);
const isTextNavigationItem =
(tag === "a" || role === "link" || role.startsWith("menuitem")) &&
(tag === "a" ||
tag === "li" ||
role === "link" ||
role === "option" ||
role.startsWith("menuitem") ||
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
!graphic &&
!!label &&
label.toLowerCase() !== "image";
@@ -118,9 +132,7 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger
const surfaceTriggerEvidence =
(popupSignal && !isTextNavigationItem) ||
compactTopbarIcon ||
signals.cssHoverMatch === true ||
attr(signals, "aria-haspopup") !== undefined ||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false";
hasStrongHoverExpansionSignal(signals);
if (!directInteractive || !surfaceTriggerEvidence) {
return { eligible: false, score: 0, reasons: [] };
}