From 73724617f94a13a562efa6aa4590e5a256429847 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 17:41:33 +0800 Subject: [PATCH] feat(extension): recording hover with custom div --- .../content/__tests__/record-capture.test.ts | 144 ++++++++++++++++++ apps/extension/src/content/record-capture.ts | 88 +---------- .../src/content/record-hover-surface.ts | 43 +++++- 3 files changed, 186 insertions(+), 89 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 816f81d..e59cda0 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -470,6 +470,150 @@ describe("record-capture semantic", () => { }); }); + it("records hover before clicking an unlabelled positioned floating surface", () => { + document.body.innerHTML = ` + +
+ My profile +
+ `; + const capture = startRecordCapture("rec-hover-plain-floating-surface", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const surface = document.querySelector("div")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 48, + top: 48, + left: 820, + right: 980, + bottom: 128, + width: 160, + height: 80, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === surface ? "absolute" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + }); + + it("does not treat ordinary document-flow divs as hover surfaces", () => { + document.body.innerHTML = ` + +
+ My profile +
+ `; + const capture = startRecordCapture("rec-hover-plain-flow-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockReturnValue({ + cursor: "", + display: "block", + pointerEvents: "auto", + position: "static", + visibility: "visible", + } as CSSStyleDeclaration); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not treat unlabelled sticky containers as hover surfaces", () => { + document.body.innerHTML = ` + +
+ My profile +
+ `; + const capture = startRecordCapture("rec-hover-sticky-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const surface = document.querySelector("div")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 48, + top: 48, + left: 820, + right: 980, + bottom: 128, + width: 160, + height: 80, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === surface ? "sticky" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + it("does not record hover before clicking the same trigger", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index a632154..c879916 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -73,7 +73,6 @@ type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; const HOVER_REPLACE_SCORE_MARGIN = 50; -const RECORD_DEBUG_STORAGE_KEY = "bsk_record_debug"; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -146,26 +145,6 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } -function recordDebug(message: string, data?: unknown): void { - try { - if (localStorage.getItem(RECORD_DEBUG_STORAGE_KEY) !== "1") return; - console.info("[bsk record]", message, data === undefined ? "" : JSON.stringify(data)); - } catch { - // Recording must not depend on page storage or console availability. - } -} - -function elementDebugSummary(el: Element): Record { - return { - tag: el.tagName.toLowerCase(), - id: el.id || undefined, - class: typeof el.className === "string" ? el.className || undefined : undefined, - role: el.getAttribute("role") ?? undefined, - ariaLabel: el.getAttribute("aria-label") ?? undefined, - text: el.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) || undefined, - }; -} - function hoverTriggerAttrs(el: Element): Record { return { id: el.id || undefined, @@ -253,34 +232,13 @@ function evaluateHoverTriggerElement( } function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { - if (!(target instanceof Element)) { - recordDebug("hover ignored: event target is not an Element", { target }); - return null; - } + if (!(target instanceof Element)) return null; const element = resolveHoverElement(target); - if (!element) { - recordDebug("hover ignored: no resolved hover element", { target }); - return null; - } + if (!element) return null; const desc = describeTarget(element); const decision = evaluateHoverTriggerElement(element, desc); - if (!decision?.eligible) { - recordDebug("hover ignored: rejected by trigger policy", { - desc, - score: decision?.score ?? 0, - reasons: decision?.reasons ?? [], - element: elementDebugSummary(element), - }); - return null; - } + if (!decision?.eligible) return null; const normalizedTarget = normalizeHoverTarget(element, desc, decision); - recordDebug("hover accepted", { - desc: normalizedTarget, - raw_desc: desc, - score: decision.score, - reasons: decision.reasons, - element: elementDebugSummary(element), - }); return { element, target: normalizedTarget, @@ -345,7 +303,6 @@ export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, ): RecordCaptureController { - recordDebug("record capture started", { page_url: location.href }); const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; @@ -439,34 +396,22 @@ export function startRecordCapture( } const clickElement = resolveClickableElement(clickTarget); if (clickElement === pendingHover.element) { - recordDebug("pending hover skipped: click resolved to same trigger", { - target: pendingHover.target, - }); return; } const relationTarget = clickElement ?? clickTarget; if (!clickElement && pendingHover.element.contains(relationTarget)) { - recordDebug("pending hover skipped: anonymous click inside trigger", { - target: pendingHover.target, - }); return; } const relation = decideHoverSurfaceRelation( { triggerElement: pendingHover.element, - triggerTarget: pendingHover.target, }, relationTarget, ); if (!relation.related) { - recordDebug("pending hover skipped: click outside hover surface", { - target: pendingHover.target, - reason: relation.reason, - }); pendingHover = null; return; } - recordDebug("pending hover emitted before click", { target: pendingHover.target }); emitStep({ op: "hover", target: pendingHover.target, @@ -481,34 +426,8 @@ export function startRecordCapture( if (!hover) return; const now = Date.now(); if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) { - recordDebug("pending hover kept over later hover", { - current: { - target: pendingHover.target, - score: pendingHover.score, - reasons: pendingHover.reasons, - }, - next: { - target: hover.target, - score: hover.score, - reasons: hover.reasons, - }, - }); return; } - if (pendingHover) { - recordDebug("pending hover replaced", { - current: { - target: pendingHover.target, - score: pendingHover.score, - reasons: pendingHover.reasons, - }, - next: { - target: hover.target, - score: hover.score, - reasons: hover.reasons, - }, - }); - } pendingHover = hover; }; @@ -701,7 +620,6 @@ export function startRecordCapture( return { dispose() { - recordDebug("record capture disposed", { page_url: location.href }); commitFillSession(); document.removeEventListener("click", onClick, true); document.removeEventListener("mouseover", onMouseOver, true); diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index 0f1a7d9..c00d6fc 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -1,5 +1,3 @@ -import type { TargetDescriptor } from "@/lib/describe-target"; - export interface HoverSurfaceDecision { related: boolean; reason?: string; @@ -8,7 +6,6 @@ export interface HoverSurfaceDecision { export interface HoverSurfaceContext { triggerElement: Element; - triggerTarget: TargetDescriptor; } const HOVER_SURFACE_SELECTOR = [ @@ -26,6 +23,8 @@ const HOVER_SURFACE_SELECTOR = [ const HOVER_SURFACE_WORD_RE = /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i; +const HOVER_SURFACE_MAX_AXIS_GAP = 96; +const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; function elementTokenText(el: Element): string { return [ @@ -62,6 +61,33 @@ function closestHoverSurface(clickElement: Element): Element | null { return null; } +function isPositionedFloatingElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + const style = getComputedStyle(el); + if (!["absolute", "fixed"].includes(style.position)) return false; + if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { + return false; + } + const rect = rectFor(el); + if (!rect) return false; + const viewportArea = window.innerWidth * window.innerHeight; + const area = rect.width * rect.height; + return ( + area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO) + ); +} + +function closestFloatingSurface(clickElement: Element): Element | null { + let node: Element | null = clickElement; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isPositionedFloatingElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + function rectFor(el: Element): DOMRect | null { const rect = el.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return null; @@ -82,7 +108,7 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { triggerRect.top - surfaceRect.bottom, surfaceRect.top - triggerRect.bottom, ); - return horizontalGap <= 320 && verticalGap <= 320; + return horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP; } export function decideHoverSurfaceRelation( @@ -94,6 +120,15 @@ export function decideHoverSurfaceRelation( return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; } + const floatingSurface = closestFloatingSurface(clickElement); + if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) { + return { + related: true, + reason: "click-inside-near-floating-surface", + surface: floatingSurface, + }; + } + if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) { return { related: true, reason: "click-inside-trigger-contained-surface" }; }