diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 38ac5d7..816f81d 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -156,4 +156,353 @@ describe("record-capture semantic", () => { target: { name: "下一步", role: "button" }, }); }); + + it("records a hover trigger before clicking its revealed menu item", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("records hover for compact topbar image buttons before menu item clicks", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-topbar", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "image" }, + }); + }); + + it("records hover when the revealed menu item is inside the hover root", () => { + document.body.innerHTML = ` +
+ image + +
+ `; + const capture = startRecordCapture("rec-hover-contained-menu", (step) => steps.push(step)); + const trigger = document.querySelector('[role="button"]')!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("does not let a revealed topbar menu link replace the pending hover trigger", () => { + document.body.innerHTML = ` + + image + + + `; + const capture = startRecordCapture("rec-hover-topbar-link", (step) => steps.push(step)); + const trigger = document.querySelector('a[aria-label="image"]')!; + const item = document.querySelector("ul a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(item, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 72, + top: 72, + left: 820, + right: 916, + bottom: 104, + width: 96, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: el === item ? "pointer" : "", + pointerEvents: "auto", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "link", name: "image" }, + }); + }); + + it("keeps the first hover trigger over a later lower-score accepted hover", () => { + document.body.innerHTML = ` + + My profile + `; + const capture = startRecordCapture("rec-hover-latch", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const laterHover = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(laterHover, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 72, + top: 72, + left: 820, + right: 916, + bottom: 104, + width: 96, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: el === laterHover ? "pointer" : "", + pointerEvents: "auto", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + laterHover.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + laterHover.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "button", name: "My profile" }, + }); + }); + + it("records avatar div hover with a semantic image target", () => { + document.body.innerHTML = ` +
+ +
+ + `; + const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step)); + const trigger = document.querySelector(".tg-avatar")!; + const image = document.querySelector("img")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(image, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockReturnValue({ + cursor: "pointer", + pointerEvents: "auto", + } as CSSStyleDeclaration); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + image.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { tag: "img", role: "img", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("does not record hover before an unrelated page click", () => { + document.body.innerHTML = ` + +
+ Projects +
+ `; + const capture = startRecordCapture("rec-hover-unrelated-click", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const link = document.querySelector("main a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + expect(steps[0]).toMatchObject({ + op: "click", + target: { role: "link", name: "Projects" }, + }); + }); + + it("does not record hover before clicking the same trigger", () => { + document.body.innerHTML = ` + + `; + const capture = startRecordCapture("rec-hover-same-trigger", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + trigger.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not record hover for ordinary controls without popup signals", () => { + const capture = startRecordCapture("rec-hover-noise", (step) => steps.push(step)); + const button = document.querySelector("button")!; + + button.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); }); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 8c9d5d0..a632154 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,9 +1,15 @@ import { describeEventTarget, describeTarget, + resolveClickableElement, resolveHoverElement, type TargetDescriptor, } from "@/lib/describe-target"; +import { + evaluateHoverTrigger, + type HoverTriggerDecision, + type HoverTriggerRect, +} from "@/lib/hover-trigger-policy"; import { isRecordCancelMessage, isRecordStartMessage, @@ -20,6 +26,7 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; +import { decideHoverSurfaceRelation } from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -58,11 +65,15 @@ interface PendingHover { element: Element; target: TargetDescriptor; recordedAt: number; + score: number; + reasons: string[]; } type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; +const HOVER_REPLACE_SCORE_MARGIN = 50; +const RECORD_DEBUG_STORAGE_KEY = "bsk_record_debug"; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -135,12 +146,160 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } +function recordDebug(message: string, data?: unknown): void { + try { + if (localStorage.getItem(RECORD_DEBUG_STORAGE_KEY) !== "1") return; + console.info("[bsk record]", message, data === undefined ? "" : JSON.stringify(data)); + } catch { + // Recording must not depend on page storage or console availability. + } +} + +function elementDebugSummary(el: Element): Record { + return { + tag: el.tagName.toLowerCase(), + id: el.id || undefined, + class: typeof el.className === "string" ? el.className || undefined : undefined, + role: el.getAttribute("role") ?? undefined, + ariaLabel: el.getAttribute("aria-label") ?? undefined, + text: el.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) || undefined, + }; +} + +function hoverTriggerAttrs(el: Element): Record { + return { + id: el.id || undefined, + class: typeof el.className === "string" ? el.className || undefined : undefined, + "data-testid": el.getAttribute("data-testid") ?? undefined, + "data-test": el.getAttribute("data-test") ?? undefined, + "data-cy": el.getAttribute("data-cy") ?? undefined, + "aria-label": el.getAttribute("aria-label") ?? undefined, + "aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined, + "aria-controls": el.getAttribute("aria-controls") ?? undefined, + "aria-expanded": el.getAttribute("aria-expanded") ?? undefined, + "aria-hidden": el.getAttribute("aria-hidden") ?? undefined, + "aria-disabled": el.getAttribute("aria-disabled") ?? undefined, + contenteditable: el.getAttribute("contenteditable") ?? undefined, + disabled: el.hasAttribute("disabled") ? "" : undefined, + hidden: el.hasAttribute("hidden") ? "" : undefined, + inert: el.hasAttribute("inert") ? "" : undefined, + onclick: el.getAttribute("onclick") ?? undefined, + onmouseenter: el.getAttribute("onmouseenter") ?? undefined, + onmouseover: el.getAttribute("onmouseover") ?? undefined, + role: el.getAttribute("role") ?? undefined, + tabindex: el.getAttribute("tabindex") ?? undefined, + title: el.getAttribute("title") ?? undefined, + }; +} + +function hoverTriggerRect(el: Element): HoverTriggerRect { + const rect = el.getBoundingClientRect(); + return { x: rect.left, y: rect.top, w: rect.width, h: rect.height }; +} + +function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } { + if (!(el instanceof HTMLElement)) return {}; + const style = getComputedStyle(el); + return { cursor: style.cursor, pointerEvents: style.pointerEvents }; +} + +function hasHoverGraphicDescendant(el: Element): boolean { + return el.querySelector("img,svg,use,path,i") !== null; +} + +function isWeakHoverTarget(target: TargetDescriptor): boolean { + return !target.role && !target.name && target.tag === "div"; +} + +function looksLikeAvatarElement(el: Element): boolean { + const className = typeof el.className === "string" ? el.className : ""; + return /\b(avatar|user-avatar)\b/i.test(className); +} + +function normalizeHoverTarget( + el: Element, + desc: TargetDescriptor, + decision: HoverTriggerDecision, +): TargetDescriptor { + if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) { + return { ...desc, name: "image" }; + } + if ( + isWeakHoverTarget(desc) && + (looksLikeAvatarElement(el) || + (hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only"))) + ) { + return { tag: "img", role: "img", name: "image" }; + } + return desc; +} + +function evaluateHoverTriggerElement( + el: Element, + desc: TargetDescriptor, +): HoverTriggerDecision | null { + if (!(el instanceof HTMLElement)) return null; + const style = hoverTriggerStyle(el); + return evaluateHoverTrigger({ + tag: el.tagName.toLowerCase(), + role: desc.role, + label: desc.name, + attrs: hoverTriggerAttrs(el), + rect: hoverTriggerRect(el), + cursor: style.cursor, + pointerEvents: style.pointerEvents, + hasGraphicDescendant: hasHoverGraphicDescendant(el), + }); +} + function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { - if (!(target instanceof Element)) return null; + if (!(target instanceof Element)) { + recordDebug("hover ignored: event target is not an Element", { target }); + return null; + } const element = resolveHoverElement(target); - if (!element) return null; + if (!element) { + recordDebug("hover ignored: no resolved hover element", { target }); + return null; + } const desc = describeTarget(element); - return { element, target: desc, recordedAt: Date.now() }; + const decision = evaluateHoverTriggerElement(element, desc); + if (!decision?.eligible) { + recordDebug("hover ignored: rejected by trigger policy", { + desc, + score: decision?.score ?? 0, + reasons: decision?.reasons ?? [], + element: elementDebugSummary(element), + }); + return null; + } + const normalizedTarget = normalizeHoverTarget(element, desc, decision); + recordDebug("hover accepted", { + desc: normalizedTarget, + raw_desc: desc, + score: decision.score, + reasons: decision.reasons, + element: elementDebugSummary(element), + }); + return { + element, + target: normalizedTarget, + recordedAt: Date.now(), + score: decision.score, + reasons: decision.reasons, + }; +} + +function shouldReplacePendingHover( + current: PendingHover, + next: PendingHover, + now: number, +): boolean { + if (next.element === current.element) return false; + if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true; + if (current.element.contains(next.element)) return false; + if (next.element.contains(current.element)) return true; + return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN; } /** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */ @@ -186,6 +345,7 @@ export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, ): RecordCaptureController { + recordDebug("record capture started", { page_url: location.href }); const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; @@ -277,7 +437,36 @@ export function startRecordCapture( pendingHover = null; return; } - if (pendingHover.element.contains(clickTarget)) return; + const clickElement = resolveClickableElement(clickTarget); + if (clickElement === pendingHover.element) { + recordDebug("pending hover skipped: click resolved to same trigger", { + target: pendingHover.target, + }); + return; + } + const relationTarget = clickElement ?? clickTarget; + if (!clickElement && pendingHover.element.contains(relationTarget)) { + recordDebug("pending hover skipped: anonymous click inside trigger", { + target: pendingHover.target, + }); + return; + } + const relation = decideHoverSurfaceRelation( + { + triggerElement: pendingHover.element, + triggerTarget: pendingHover.target, + }, + relationTarget, + ); + if (!relation.related) { + recordDebug("pending hover skipped: click outside hover surface", { + target: pendingHover.target, + reason: relation.reason, + }); + pendingHover = null; + return; + } + recordDebug("pending hover emitted before click", { target: pendingHover.target }); emitStep({ op: "hover", target: pendingHover.target, @@ -289,7 +478,37 @@ export function startRecordCapture( const target = eventTarget(event); if (isOverlayTarget(target)) return; const hover = hoverTargetFromEvent(target); - if (!hover || hover.element === pendingHover?.element) return; + if (!hover) return; + const now = Date.now(); + if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) { + recordDebug("pending hover kept over later hover", { + current: { + target: pendingHover.target, + score: pendingHover.score, + reasons: pendingHover.reasons, + }, + next: { + target: hover.target, + score: hover.score, + reasons: hover.reasons, + }, + }); + return; + } + if (pendingHover) { + recordDebug("pending hover replaced", { + current: { + target: pendingHover.target, + score: pendingHover.score, + reasons: pendingHover.reasons, + }, + next: { + target: hover.target, + score: hover.score, + reasons: hover.reasons, + }, + }); + } pendingHover = hover; }; @@ -482,6 +701,7 @@ export function startRecordCapture( return { dispose() { + recordDebug("record capture disposed", { page_url: location.href }); commitFillSession(); document.removeEventListener("click", onClick, true); document.removeEventListener("mouseover", onMouseOver, true); diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts new file mode 100644 index 0000000..0f1a7d9 --- /dev/null +++ b/apps/extension/src/content/record-hover-surface.ts @@ -0,0 +1,102 @@ +import type { TargetDescriptor } from "@/lib/describe-target"; + +export interface HoverSurfaceDecision { + related: boolean; + reason?: string; + surface?: Element; +} + +export interface HoverSurfaceContext { + triggerElement: Element; + triggerTarget: TargetDescriptor; +} + +const HOVER_SURFACE_SELECTOR = [ + '[role="menu"]', + '[role="menubar"]', + '[role="listbox"]', + '[role="dialog"]', + '[role="tooltip"]', + '[aria-modal="true"]', + "[data-popper-placement]", + "[data-floating-ui-placement]", + "[data-headlessui-state]", + "[data-radix-popper-content-wrapper]", +].join(","); + +const HOVER_SURFACE_WORD_RE = + /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i; + +function elementTokenText(el: Element): string { + return [ + el.id, + typeof el.className === "string" ? el.className : "", + el.getAttribute("data-testid"), + el.getAttribute("data-test"), + el.getAttribute("data-cy"), + el.getAttribute("aria-label"), + el.getAttribute("role"), + ] + .filter((value): value is string => typeof value === "string" && value.length > 0) + .join(" "); +} + +function isHoverSurfaceElement(el: Element): boolean { + if (el.matches(HOVER_SURFACE_SELECTOR)) return true; + if (HOVER_SURFACE_WORD_RE.test(elementTokenText(el))) return true; + const tag = el.tagName.toLowerCase(); + if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) { + return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el)); + } + return false; +} + +function closestHoverSurface(clickElement: Element): Element | null { + let node: Element | null = clickElement; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isHoverSurfaceElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + +function rectFor(el: Element): DOMRect | null { + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return null; + return rect; +} + +function isNearTrigger(surface: Element, trigger: Element): boolean { + const surfaceRect = rectFor(surface); + const triggerRect = rectFor(trigger); + if (!surfaceRect || !triggerRect) return true; + const horizontalGap = Math.max( + 0, + triggerRect.left - surfaceRect.right, + surfaceRect.left - triggerRect.right, + ); + const verticalGap = Math.max( + 0, + triggerRect.top - surfaceRect.bottom, + surfaceRect.top - triggerRect.bottom, + ); + return horizontalGap <= 320 && verticalGap <= 320; +} + +export function decideHoverSurfaceRelation( + context: HoverSurfaceContext, + clickElement: Element, +): HoverSurfaceDecision { + const clickSurface = closestHoverSurface(clickElement); + if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) { + return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; + } + + if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) { + return { related: true, reason: "click-inside-trigger-contained-surface" }; + } + + return { related: false, reason: "click-outside-hover-surface" }; +} diff --git a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts new file mode 100644 index 0000000..d3b3dce --- /dev/null +++ b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { evaluateHoverTrigger } from "../hover-trigger-policy"; + +describe("evaluateHoverTrigger", () => { + it("accepts explicit popup triggers", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "Open user navigation menu", + attrs: { "aria-haspopup": "menu", "aria-expanded": "false" }, + rect: { x: 900, y: 8, w: 32, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual( + expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]), + ); + }); + + it("accepts compact topbar image buttons without popup attributes", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "image", + attrs: {}, + rect: { x: 900, y: 8, w: 32, h: 32 }, + pointerEvents: "auto", + hasGraphicDescendant: true, + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"])); + }); + + it("accepts compact topbar image links without popup attributes", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "image", + attrs: {}, + rect: { x: 900, y: 8, w: 32, h: 32 }, + pointerEvents: "auto", + hasGraphicDescendant: true, + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"])); + }); + + it("rejects topbar menu links without surface trigger evidence", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "My profile", + attrs: {}, + rect: { x: 820, y: 72, w: 96, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects text navigation items even when their attributes contain popup words", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "My profile", + attrs: { class: "dropdown-item profile-link" }, + rect: { x: 820, y: 72, w: 96, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects ordinary controls without hover popup signals", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "Search", + attrs: {}, + rect: { x: 100, y: 240, w: 120, h: 36 }, + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects unsafe or non-visible triggers", () => { + expect( + evaluateHoverTrigger({ + tag: "input", + attrs: {}, + rect: { x: 10, y: 10, w: 80, h: 24 }, + pointerEvents: "auto", + }).eligible, + ).toBe(false); + expect( + evaluateHoverTrigger({ + tag: "button", + role: "button", + attrs: { hidden: "" }, + rect: { x: 10, y: 10, w: 80, h: 24 }, + pointerEvents: "auto", + }).eligible, + ).toBe(false); + }); +}); diff --git a/apps/extension/src/lib/hover-trigger-policy.ts b/apps/extension/src/lib/hover-trigger-policy.ts new file mode 100644 index 0000000..8701c89 --- /dev/null +++ b/apps/extension/src/lib/hover-trigger-policy.ts @@ -0,0 +1,166 @@ +export interface HoverTriggerRect { + x: number; + y: number; + w: number; + h: number; +} + +export interface HoverTriggerSignals { + tag: string; + role?: string; + label?: string; + attrs?: Record; + rect?: HoverTriggerRect | null; + cursor?: string; + pointerEvents?: string; + hasGraphicDescendant?: boolean; + cssHoverMatch?: boolean; +} + +export interface HoverTriggerDecision { + eligible: boolean; + score: number; + reasons: string[]; +} + +const HOVER_TRIGGER_MIN_SCORE = 45; +const HOVER_POPUP_SIGNAL_RE = + /(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/; + +function attr(signals: HoverTriggerSignals, name: string): string | undefined { + return signals.attrs?.[name.toLowerCase()]; +} + +export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean { + const tag = signals.tag.toLowerCase(); + if (["input", "textarea", "select", "option"].includes(tag)) return true; + if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true; + if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true; + return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true"; +} + +export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean { + const attrs = signals.attrs ?? {}; + const haystack = [ + attrs.id, + attrs.class, + attrs["data-testid"], + attrs["data-test"], + attrs["data-cy"], + attrs["aria-label"], + attrs["aria-haspopup"], + attrs["aria-controls"], + attrs.title, + signals.role === "button" ? signals.label : undefined, + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return HOVER_POPUP_SIGNAL_RE.test(haystack); +} + +export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean { + const tag = signals.tag.toLowerCase(); + const role = (signals.role ?? "").toLowerCase(); + return ( + ["button", "a", "summary"].includes(tag) || + ["button", "link", "menuitem", "tab", "combobox"].includes(role) || + attr(signals, "tabindex") !== undefined || + signals.cursor === "pointer" || + attr(signals, "onclick") !== undefined || + attr(signals, "onmouseenter") !== undefined || + attr(signals, "onmouseover") !== undefined || + attr(signals, "aria-haspopup") !== undefined || + attr(signals, "aria-expanded") === "false" + ); +} + +export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision { + const rect = signals.rect; + if ( + !rect || + rect.w <= 0 || + rect.h <= 0 || + signals.pointerEvents === "none" || + attr(signals, "hidden") !== undefined || + attr(signals, "inert") !== undefined || + (attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" || + isUnsafeHoverTrigger(signals) + ) { + return { eligible: false, score: 0, reasons: [] }; + } + + const area = rect.w * rect.h; + if (area <= 0 || area > 160_000) { + return { eligible: false, score: 0, reasons: [] }; + } + + const reasons: string[] = []; + let score = 0; + const tag = signals.tag.toLowerCase(); + const role = (signals.role ?? "").toLowerCase(); + const label = signals.label?.trim(); + const graphic = signals.hasGraphicDescendant === true; + const directInteractive = hasDirectHoverInteractiveSignal(signals); + const popupSignal = hasHoverPopupSignal(signals); + const isTextNavigationItem = + (tag === "a" || role === "link" || role.startsWith("menuitem")) && + !graphic && + !!label && + label.toLowerCase() !== "image"; + const compactTopbarIcon = + (role === "button" || role === "link" || tag === "button" || tag === "a") && + graphic && + rect.y < 120 && + rect.w <= 96 && + rect.h <= 96 && + (!label || label.toLowerCase() === "image"); + const surfaceTriggerEvidence = + (popupSignal && !isTextNavigationItem) || + compactTopbarIcon || + signals.cssHoverMatch === true || + attr(signals, "aria-haspopup") !== undefined || + (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"; + if (!directInteractive || !surfaceTriggerEvidence) { + return { eligible: false, score: 0, reasons: [] }; + } + + if (attr(signals, "aria-haspopup") !== undefined) { + score += 80; + reasons.push("aria-haspopup"); + } + if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") { + score += 55; + reasons.push("collapsed"); + } + if (popupSignal) { + score += 45; + reasons.push("popup-signal"); + } + if (tag === "button" || role === "button") { + score += 30; + reasons.push("button"); + } + if (signals.cursor === "pointer") { + score += 25; + reasons.push("pointer"); + } + if (graphic && (!label || label.toLowerCase() === "image")) { + score += 40; + reasons.push("icon-only"); + } + if (rect.y < 120) { + score += 25; + reasons.push("topbar"); + } + if (rect.w <= 80 && rect.h <= 80) { + score += 20; + reasons.push("compact"); + } + if (signals.cssHoverMatch) { + score += 40; + reasons.push("css-hover"); + } + + return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons }; +} diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 086e216..bc4cd2a 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -101,7 +101,7 @@ function twoHoverTriggerSnapshotReply() { "button", "class", "user-avatar", - "secondary-trigger", + "secondary-dropdown-trigger", "position", "static", "pointer-events", @@ -123,7 +123,7 @@ function twoHoverTriggerSnapshotReply() { [], [], [i("class"), i("user-avatar")], - [i("class"), i("secondary-trigger")], + [i("class"), i("secondary-dropdown-trigger")], ], }, layout: { diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 44aba56..0f56577 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -5,6 +5,7 @@ // `styles` columns are [position, pointer-events, cursor] in that order. import type { Rect, Viewport } from "@browser-skill/vom"; +import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import type { CdpRunner } from "../shared"; @@ -442,27 +443,6 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise { ); } -function isVisibleProbeNode(node: CapturedNode): boolean { - const rect = node.rect; - return ( - !!rect && - rect.w > 0 && - rect.h > 0 && - node.pointerEvents !== "none" && - node.attrs.hidden === undefined && - node.attrs.inert === undefined && - (node.attrs["aria-hidden"] ?? "").toLowerCase() !== "true" - ); -} - -function isUnsafeHoverTrigger(node: CapturedNode): boolean { - const tag = node.tag.toLowerCase(); - if (["input", "textarea", "select", "option"].includes(tag)) return true; - if ((node.attrs.contenteditable ?? "").toLowerCase() === "true") return true; - if (node.attrs.disabled !== undefined || node.attrs.inert !== undefined) return true; - return (node.attrs["aria-disabled"] ?? "").toLowerCase() === "true"; -} - function capturedText(node: CapturedNode): string | undefined { const value = node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; @@ -484,120 +464,45 @@ function hasGraphicDescendant( return false; } -function hasHoverPopupSignal(node: CapturedNode): boolean { - const attrs = node.attrs; - const haystack = [ - attrs.id, - attrs.class, - attrs["data-testid"], - attrs["data-test"], - attrs["data-cy"], - attrs["aria-haspopup"], - attrs["aria-controls"], - ] - .filter(Boolean) - .join(" ") - .toLowerCase(); - return /(menu|dropdown|popover|popup|avatar|profile|account|user)/.test(haystack); -} - function roleOf(node: CapturedNode): string { return (node.attrs.role ?? "").toLowerCase(); } -function hasDirectInteractiveSignal(node: CapturedNode): boolean { - const tag = node.tag.toLowerCase(); - const role = roleOf(node); - return ( - ["button", "a", "summary"].includes(tag) || - ["button", "link", "menuitem", "tab", "combobox"].includes(role) || - node.attrs.tabindex !== undefined || - node.cursor === "pointer" || - node.attrs.onclick !== undefined || - node.attrs.onmouseenter !== undefined || - node.attrs.onmouseover !== undefined || - node.attrs["aria-haspopup"] !== undefined || - node.attrs["aria-expanded"] === "false" - ); -} - function scoreHoverCandidate( node: CapturedNode, childrenByParentId: Map, cssHoverPoints: Array<{ x: number; y: number }>, ): HoverCandidate | null { - if (!isVisibleProbeNode(node) || isUnsafeHoverTrigger(node)) { - return null; - } const rect = node.rect; if (!rect) return null; - const area = rect.w * rect.h; - if (area <= 0 || area > 160_000) return null; - - const reasons: string[] = []; - let score = 0; - const tag = node.tag.toLowerCase(); - const role = roleOf(node); const label = capturedText(node); - const graphic = hasGraphicDescendant(node, childrenByParentId); - const directInteractive = hasDirectInteractiveSignal(node); - const popupSignal = hasHoverPopupSignal(node); - const compactTopbarIcon = !label && graphic && rect.y < 120 && rect.w <= 96 && rect.h <= 96; - if (!directInteractive && !popupSignal && !compactTopbarIcon) return null; + const cssHoverMatch = cssHoverPoints.some( + (point) => + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h, + ); + const decision = evaluateHoverTrigger({ + tag: node.tag, + role: roleOf(node), + label, + attrs: node.attrs, + rect, + cursor: node.cursor, + pointerEvents: node.pointerEvents, + hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId), + cssHoverMatch, + }); - if (node.attrs["aria-haspopup"] !== undefined) { - score += 80; - reasons.push("aria-haspopup"); - } - if ((node.attrs["aria-expanded"] ?? "").toLowerCase() === "false") { - score += 55; - reasons.push("collapsed"); - } - if (popupSignal) { - score += 45; - reasons.push("popup-signal"); - } - if (tag === "button" || role === "button") { - score += 30; - reasons.push("button"); - } - if (node.cursor === "pointer") { - score += 25; - reasons.push("pointer"); - } - if (!label && graphic) { - score += 40; - reasons.push("icon-only"); - } - if (rect.y < 120) { - score += 25; - reasons.push("topbar"); - } - if (rect.w <= 80 && rect.h <= 80) { - score += 20; - reasons.push("compact"); - } - if ( - cssHoverPoints.some( - (point) => - point.x >= rect.x && - point.x <= rect.x + rect.w && - point.y >= rect.y && - point.y <= rect.y + rect.h, - ) - ) { - score += 40; - reasons.push("css-hover"); - } - - if (score < 45) return null; + if (!decision.eligible) return null; return { backendNodeId: node.backendNodeId, label, x: rect.x + rect.w / 2, y: rect.y + rect.h / 2, - score, - reasons, + score: decision.score, + reasons: decision.reasons, }; }