feat(extension): recording hover

This commit is contained in:
Ljy-0827
2026-08-06 17:33:46 +08:00
parent 9d4ac90592
commit 4ba6fb45e1
7 changed files with 978 additions and 124 deletions
@@ -156,4 +156,353 @@ describe("record-capture semantic", () => {
target: { name: "下一步", role: "button" },
});
});
it("records a hover trigger before clicking its revealed menu item", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button>
<ul class="user-menu dropdown-menu">
<li><a href="/u/me">My profile</a></li>
</ul>
`;
const capture = startRecordCapture("rec-hover-menu", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const item = document.querySelector("a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { role: "link", name: "My profile" },
});
});
it("records hover for compact topbar image buttons before menu item clicks", () => {
document.body.innerHTML = `
<button type="button" aria-label="image" style="position: absolute; top: 8px; left: 900px; width: 32px; height: 32px;">
<img alt="image" />
</button>
<ul class="user-menu dropdown-menu">
<li><a href="/u/me">My profile</a></li>
</ul>
`;
const capture = startRecordCapture("rec-hover-topbar", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const item = document.querySelector("a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "image" },
});
});
it("records hover when the revealed menu item is inside the hover root", () => {
document.body.innerHTML = `
<div class="user dropdown" role="button" aria-label="image">
<img alt="image" />
<ul>
<li><a href="/u/me">My profile</a></li>
</ul>
</div>
`;
const capture = startRecordCapture("rec-hover-contained-menu", (step) => steps.push(step));
const trigger = document.querySelector('[role="button"]')!;
const item = document.querySelector("a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "image" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { role: "link", name: "My profile" },
});
});
it("does not let a revealed topbar menu link replace the pending hover trigger", () => {
document.body.innerHTML = `
<a href="/u/me" aria-label="image" style="position: absolute; top: 8px; left: 900px; width: 32px; height: 32px;">
<img alt="image" />
</a>
<ul class="user-menu dropdown-menu">
<li><a class="dropdown-item profile-link" href="/u/me">My profile</a></li>
</ul>
`;
const capture = startRecordCapture("rec-hover-topbar-link", (step) => steps.push(step));
const trigger = document.querySelector('a[aria-label="image"]')!;
const item = document.querySelector("ul a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
vi.spyOn(item, "getBoundingClientRect").mockReturnValue({
x: 820,
y: 72,
top: 72,
left: 820,
right: 916,
bottom: 104,
width: 96,
height: 32,
toJSON: () => ({}),
});
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
const style = {
cursor: el === item ? "pointer" : "",
pointerEvents: "auto",
} as CSSStyleDeclaration;
return style;
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "link", name: "image" },
});
});
it("keeps the first hover trigger over a later lower-score accepted hover", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu" aria-haspopup="menu" aria-expanded="false">
<img alt="image" />
</button>
<a class="account-menu" role="button" href="/u/me">My profile</a>
`;
const capture = startRecordCapture("rec-hover-latch", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const laterHover = document.querySelector("a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
vi.spyOn(laterHover, "getBoundingClientRect").mockReturnValue({
x: 820,
y: 72,
top: 72,
left: 820,
right: 916,
bottom: 104,
width: 96,
height: 32,
toJSON: () => ({}),
});
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
const style = {
cursor: el === laterHover ? "pointer" : "",
pointerEvents: "auto",
} as CSSStyleDeclaration;
return style;
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
laterHover.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
laterHover.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { role: "button", name: "My profile" },
});
});
it("records avatar div hover with a semantic image target", () => {
document.body.innerHTML = `
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
<img class="tg-avatar__image" />
</div>
<ul class="user-menu dropdown-menu">
<li><a href="/u/me">My profile</a></li>
</ul>
`;
const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step));
const trigger = document.querySelector(".tg-avatar")!;
const image = document.querySelector("img")!;
const item = document.querySelector("a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
vi.spyOn(image, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
vi.spyOn(window, "getComputedStyle").mockReturnValue({
cursor: "pointer",
pointerEvents: "auto",
} as CSSStyleDeclaration);
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
image.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { tag: "img", role: "img", name: "image" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { role: "link", name: "My profile" },
});
});
it("does not record hover before an unrelated page click", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button>
<main>
<a href="/projects">Projects</a>
</main>
`;
const capture = startRecordCapture("rec-hover-unrelated-click", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
const link = document.querySelector("main a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["click"]);
expect(steps[0]).toMatchObject({
op: "click",
target: { role: "link", name: "Projects" },
});
});
it("does not record hover before clicking the same trigger", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button>
`;
const capture = startRecordCapture("rec-hover-same-trigger", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
y: 8,
top: 8,
left: 900,
right: 932,
bottom: 40,
width: 32,
height: 32,
toJSON: () => ({}),
});
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
trigger.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["click"]);
});
it("does not record hover for ordinary controls without popup signals", () => {
const capture = startRecordCapture("rec-hover-noise", (step) => steps.push(step));
const button = document.querySelector("button")!;
button.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["click"]);
});
});
+225 -5
View File
@@ -1,9 +1,15 @@
import {
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
type HoverTriggerDecision,
type HoverTriggerRect,
} from "@/lib/hover-trigger-policy";
import {
isRecordCancelMessage,
isRecordStartMessage,
@@ -20,6 +26,7 @@ import {
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import { decideHoverSurfaceRelation } from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
@@ -58,11 +65,15 @@ interface PendingHover {
element: Element;
target: TargetDescriptor;
recordedAt: number;
score: number;
reasons: string[];
}
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
const HOVER_REPLACE_SCORE_MARGIN = 50;
const RECORD_DEBUG_STORAGE_KEY = "bsk_record_debug";
function eventTarget(event: Event): EventTarget | null {
return event.composedPath()[0] ?? event.target;
@@ -135,12 +146,160 @@ function fillableValue(el: FillableElement): string {
return el.textContent ?? "";
}
function recordDebug(message: string, data?: unknown): void {
try {
if (localStorage.getItem(RECORD_DEBUG_STORAGE_KEY) !== "1") return;
console.info("[bsk record]", message, data === undefined ? "" : JSON.stringify(data));
} catch {
// Recording must not depend on page storage or console availability.
}
}
function elementDebugSummary(el: Element): Record<string, string | undefined> {
return {
tag: el.tagName.toLowerCase(),
id: el.id || undefined,
class: typeof el.className === "string" ? el.className || undefined : undefined,
role: el.getAttribute("role") ?? undefined,
ariaLabel: el.getAttribute("aria-label") ?? undefined,
text: el.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) || undefined,
};
}
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
return {
id: el.id || undefined,
class: typeof el.className === "string" ? el.className || undefined : undefined,
"data-testid": el.getAttribute("data-testid") ?? undefined,
"data-test": el.getAttribute("data-test") ?? undefined,
"data-cy": el.getAttribute("data-cy") ?? undefined,
"aria-label": el.getAttribute("aria-label") ?? undefined,
"aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
"aria-controls": el.getAttribute("aria-controls") ?? undefined,
"aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
"aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
"aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
contenteditable: el.getAttribute("contenteditable") ?? undefined,
disabled: el.hasAttribute("disabled") ? "" : undefined,
hidden: el.hasAttribute("hidden") ? "" : undefined,
inert: el.hasAttribute("inert") ? "" : undefined,
onclick: el.getAttribute("onclick") ?? undefined,
onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
onmouseover: el.getAttribute("onmouseover") ?? undefined,
role: el.getAttribute("role") ?? undefined,
tabindex: el.getAttribute("tabindex") ?? undefined,
title: el.getAttribute("title") ?? undefined,
};
}
function hoverTriggerRect(el: Element): HoverTriggerRect {
const rect = el.getBoundingClientRect();
return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
}
function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
if (!(el instanceof HTMLElement)) return {};
const style = getComputedStyle(el);
return { cursor: style.cursor, pointerEvents: style.pointerEvents };
}
function hasHoverGraphicDescendant(el: Element): boolean {
return el.querySelector("img,svg,use,path,i") !== null;
}
function isWeakHoverTarget(target: TargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
function looksLikeAvatarElement(el: Element): boolean {
const className = typeof el.className === "string" ? el.className : "";
return /\b(avatar|user-avatar)\b/i.test(className);
}
function normalizeHoverTarget(
el: Element,
desc: TargetDescriptor,
decision: HoverTriggerDecision,
): TargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
if (
isWeakHoverTarget(desc) &&
(looksLikeAvatarElement(el) ||
(hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
) {
return { tag: "img", role: "img", name: "image" };
}
return desc;
}
function evaluateHoverTriggerElement(
el: Element,
desc: TargetDescriptor,
): HoverTriggerDecision | null {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return evaluateHoverTrigger({
tag: el.tagName.toLowerCase(),
role: desc.role,
label: desc.name,
attrs: hoverTriggerAttrs(el),
rect: hoverTriggerRect(el),
cursor: style.cursor,
pointerEvents: style.pointerEvents,
hasGraphicDescendant: hasHoverGraphicDescendant(el),
});
}
function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
if (!(target instanceof Element)) return null;
if (!(target instanceof Element)) {
recordDebug("hover ignored: event target is not an Element", { target });
return null;
}
const element = resolveHoverElement(target);
if (!element) return null;
if (!element) {
recordDebug("hover ignored: no resolved hover element", { target });
return null;
}
const desc = describeTarget(element);
return { element, target: desc, recordedAt: Date.now() };
const decision = evaluateHoverTriggerElement(element, desc);
if (!decision?.eligible) {
recordDebug("hover ignored: rejected by trigger policy", {
desc,
score: decision?.score ?? 0,
reasons: decision?.reasons ?? [],
element: elementDebugSummary(element),
});
return null;
}
const normalizedTarget = normalizeHoverTarget(element, desc, decision);
recordDebug("hover accepted", {
desc: normalizedTarget,
raw_desc: desc,
score: decision.score,
reasons: decision.reasons,
element: elementDebugSummary(element),
});
return {
element,
target: normalizedTarget,
recordedAt: Date.now(),
score: decision.score,
reasons: decision.reasons,
};
}
function shouldReplacePendingHover(
current: PendingHover,
next: PendingHover,
now: number,
): boolean {
if (next.element === current.element) return false;
if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
if (current.element.contains(next.element)) return false;
if (next.element.contains(current.element)) return true;
return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
}
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
@@ -186,6 +345,7 @@ export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
): RecordCaptureController {
recordDebug("record capture started", { page_url: location.href });
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
};
@@ -277,7 +437,36 @@ export function startRecordCapture(
pendingHover = null;
return;
}
if (pendingHover.element.contains(clickTarget)) return;
const clickElement = resolveClickableElement(clickTarget);
if (clickElement === pendingHover.element) {
recordDebug("pending hover skipped: click resolved to same trigger", {
target: pendingHover.target,
});
return;
}
const relationTarget = clickElement ?? clickTarget;
if (!clickElement && pendingHover.element.contains(relationTarget)) {
recordDebug("pending hover skipped: anonymous click inside trigger", {
target: pendingHover.target,
});
return;
}
const relation = decideHoverSurfaceRelation(
{
triggerElement: pendingHover.element,
triggerTarget: pendingHover.target,
},
relationTarget,
);
if (!relation.related) {
recordDebug("pending hover skipped: click outside hover surface", {
target: pendingHover.target,
reason: relation.reason,
});
pendingHover = null;
return;
}
recordDebug("pending hover emitted before click", { target: pendingHover.target });
emitStep({
op: "hover",
target: pendingHover.target,
@@ -289,7 +478,37 @@ export function startRecordCapture(
const target = eventTarget(event);
if (isOverlayTarget(target)) return;
const hover = hoverTargetFromEvent(target);
if (!hover || hover.element === pendingHover?.element) return;
if (!hover) return;
const now = Date.now();
if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) {
recordDebug("pending hover kept over later hover", {
current: {
target: pendingHover.target,
score: pendingHover.score,
reasons: pendingHover.reasons,
},
next: {
target: hover.target,
score: hover.score,
reasons: hover.reasons,
},
});
return;
}
if (pendingHover) {
recordDebug("pending hover replaced", {
current: {
target: pendingHover.target,
score: pendingHover.score,
reasons: pendingHover.reasons,
},
next: {
target: hover.target,
score: hover.score,
reasons: hover.reasons,
},
});
}
pendingHover = hover;
};
@@ -482,6 +701,7 @@ export function startRecordCapture(
return {
dispose() {
recordDebug("record capture disposed", { page_url: location.href });
commitFillSession();
document.removeEventListener("click", onClick, true);
document.removeEventListener("mouseover", onMouseOver, true);
@@ -0,0 +1,102 @@
import type { TargetDescriptor } from "@/lib/describe-target";
export interface HoverSurfaceDecision {
related: boolean;
reason?: string;
surface?: Element;
}
export interface HoverSurfaceContext {
triggerElement: Element;
triggerTarget: TargetDescriptor;
}
const HOVER_SURFACE_SELECTOR = [
'[role="menu"]',
'[role="menubar"]',
'[role="listbox"]',
'[role="dialog"]',
'[role="tooltip"]',
'[aria-modal="true"]',
"[data-popper-placement]",
"[data-floating-ui-placement]",
"[data-headlessui-state]",
"[data-radix-popper-content-wrapper]",
].join(",");
const HOVER_SURFACE_WORD_RE =
/\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i;
function elementTokenText(el: Element): string {
return [
el.id,
typeof el.className === "string" ? el.className : "",
el.getAttribute("data-testid"),
el.getAttribute("data-test"),
el.getAttribute("data-cy"),
el.getAttribute("aria-label"),
el.getAttribute("role"),
]
.filter((value): value is string => typeof value === "string" && value.length > 0)
.join(" ");
}
function isHoverSurfaceElement(el: Element): boolean {
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
if (HOVER_SURFACE_WORD_RE.test(elementTokenText(el))) return true;
const tag = el.tagName.toLowerCase();
if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) {
return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el));
}
return false;
}
function closestHoverSurface(clickElement: Element): Element | null {
let node: Element | null = clickElement;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
if (isHoverSurfaceElement(node)) return node;
node = node.parentElement;
depth += 1;
}
return null;
}
function rectFor(el: Element): DOMRect | null {
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return null;
return rect;
}
function isNearTrigger(surface: Element, trigger: Element): boolean {
const surfaceRect = rectFor(surface);
const triggerRect = rectFor(trigger);
if (!surfaceRect || !triggerRect) return true;
const horizontalGap = Math.max(
0,
triggerRect.left - surfaceRect.right,
surfaceRect.left - triggerRect.right,
);
const verticalGap = Math.max(
0,
triggerRect.top - surfaceRect.bottom,
surfaceRect.top - triggerRect.bottom,
);
return horizontalGap <= 320 && verticalGap <= 320;
}
export function decideHoverSurfaceRelation(
context: HoverSurfaceContext,
clickElement: Element,
): HoverSurfaceDecision {
const clickSurface = closestHoverSurface(clickElement);
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
}
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
return { related: true, reason: "click-inside-trigger-contained-surface" };
}
return { related: false, reason: "click-outside-hover-surface" };
}