From 566b6c17fd8bac24d56d757063a5fee09a8bea87 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 7 Aug 2026 17:37:28 +0800 Subject: [PATCH] feat(extension): hover chain node fix --- .../content/__tests__/record-capture.test.ts | 101 ++++++++++++ .../__tests__/record-hover-surface.test.ts | 109 +++++++++++++ apps/extension/src/content/record-capture.ts | 149 ++++++++++++------ .../src/content/record-hover-surface.ts | 55 +++++-- 4 files changed, 359 insertions(+), 55 deletions(-) create mode 100644 apps/extension/src/content/__tests__/record-hover-surface.test.ts diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 051b743..90536f7 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -1150,6 +1150,107 @@ describe("record-capture semantic", () => { }); }); + it("does not let unrelated topbar hovers or menu pass-through items own an avatar menu", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T08:08:45.000Z")); + try { + document.body.innerHTML = ` + +
+ +
+ + `; + const star = document.querySelector("button")!; + const avatar = document.querySelector(".tg-avatar")!; + const image = document.querySelector("img")!; + const menu = document.querySelector(".user-menu")!; + const profile = document.querySelector('a[href="/u/me"]')!; + const groups = document.querySelector('a[href="/dashboard/groups"]')!; + let menuRect = { left: 0, top: 0, width: 0, height: 0 }; + let profileRect = { left: 0, top: 0, width: 0, height: 0 }; + let groupsRect = { left: 0, top: 0, width: 0, height: 0 }; + mockRect(star, { left: 760, top: 8, width: 60, height: 32 }); + mockRect(avatar, { left: 900, top: 8, width: 32, height: 32 }); + mockRect(image, { left: 900, top: 8, width: 32, height: 32 }); + vi.spyOn(menu, "getBoundingClientRect").mockImplementation( + () => + ({ + x: menuRect.left, + y: menuRect.top, + top: menuRect.top, + left: menuRect.left, + right: menuRect.left + menuRect.width, + bottom: menuRect.top + menuRect.height, + width: menuRect.width, + height: menuRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + vi.spyOn(profile, "getBoundingClientRect").mockImplementation( + () => + ({ + x: profileRect.left, + y: profileRect.top, + top: profileRect.top, + left: profileRect.left, + right: profileRect.left + profileRect.width, + bottom: profileRect.top + profileRect.height, + width: profileRect.width, + height: profileRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + vi.spyOn(groups, "getBoundingClientRect").mockImplementation( + () => + ({ + x: groupsRect.left, + y: groupsRect.top, + top: groupsRect.top, + left: groupsRect.left, + right: groupsRect.left + groupsRect.width, + bottom: groupsRect.top + groupsRect.height, + width: groupsRect.width, + height: groupsRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + mockHoverStyle([star, avatar, image, profile, groups]); + const capture = startRecordCapture("rec-hover-avatar-menu-pass-through", (step) => + steps.push(step), + ); + + mouseOver(star); + mouseOver(avatar); + mouseOver(image); + menuRect = { left: 820, top: 48, width: 160, height: 96 }; + profileRect = { left: 830, top: 56, width: 120, height: 28 }; + groupsRect = { left: 830, top: 88, width: 120, height: 28 }; + vi.runOnlyPendingTimers(); + mouseOver(profile); + mouseOver(groups); + click(groups); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { tag: "img", role: "img", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My groups" }, + }); + expect(JSON.stringify(steps)).not.toContain("Star"); + expect(JSON.stringify(steps)).not.toContain("My profile"); + } finally { + vi.useRealTimers(); + } + }); + it("does not record hover before an unrelated page click", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/__tests__/record-hover-surface.test.ts b/apps/extension/src/content/__tests__/record-hover-surface.test.ts new file mode 100644 index 0000000..08b8c96 --- /dev/null +++ b/apps/extension/src/content/__tests__/record-hover-surface.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from "vitest"; +import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface"; + +function mockRect( + el: Element, + rect: { left: number; top: number; width: number; height: number }, +): void { + vi.spyOn(el, "getBoundingClientRect").mockReturnValue({ + x: rect.left, + y: rect.top, + top: rect.top, + left: rect.left, + right: rect.left + rect.width, + bottom: rect.top + rect.height, + width: rect.width, + height: rect.height, + toJSON: () => ({}), + }); +} + +describe("record-hover-surface", () => { + it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => { + document.body.innerHTML = ` + +
+
+
+
  • + My profile +
    • +
  • + My groups +
    • +
    +
    +
    + `; + + expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe( + false, + ); + expect( + isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!), + ).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe( + true, + ); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe( + true, + ); + expect( + isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!), + ).toBe(true); + }); + + it("does not treat elements covered by an open surface as the surface owner", () => { + document.body.innerHTML = ` + + Fork + +
    + `; + const avatar = document.querySelector(".avatar")!; + const fork = document.querySelector(".fork")!; + const clone = document.querySelector(".clone")!; + const surface = document.querySelector(".tg-popover__content-body")!; + mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 }); + mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 }); + mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 }); + mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 }); + + expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true); + expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false); + expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false); + }); + + it("does not treat fixed navigation containers as plain floating surfaces", () => { + document.body.innerHTML = ` +
    + Fork + +
    +
    + `; + const navbar = document.querySelector(".navbar-app")!; + const floating = document.querySelector(".plain-floating")!; + mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 }); + mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === navbar ? "fixed" : "absolute", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + expect(isHoverSurfaceCandidateElement(navbar)).toBe(false); + expect(isHoverSurfaceCandidateElement(floating)).toBe(true); + }); +}); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index d693415..53488d7 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -34,6 +34,7 @@ import { decideHoverSurfaceRelation, type HoverSurfaceState, isHoverSurfaceCandidateElement, + isLikelyHoverSurfaceOwner, } from "./record-hover-surface"; const pendingStepSends = new Map>>(); @@ -376,6 +377,8 @@ export function startRecordCapture( const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; + const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map => + new Map(states.map((state) => [state.element, state.signature])); let fillSession: FillSession | null = null; let composing = false; let lastUrl = location.href; @@ -383,7 +386,7 @@ export function startRecordCapture( let pendingHover: HoverCandidate | null = null; const recentHoverCandidates: HoverCandidate[] = []; const emittedHoverElements = new WeakSet(); - let hoverSurfaceStates = new Map(); + let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates()); const hoverSurfaceNodes = new Map(); let generatedControlClick: Element | null = null; let navigationActionPending = false; @@ -490,13 +493,58 @@ export function startRecordCapture( states: HoverSurfaceState[], ): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element); + const inferOwnerForSurface = ( + surface: Element, + now: number, + before?: HoverCandidate, + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate) continue; + if (before && candidate.element === before.element) continue; + if (before && candidate.recordedAt > before.recordedAt) continue; + if (surface.contains(candidate.element)) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate; + } + return undefined; + }; + + const nodeForUnownedSurface = ( + surfaceState: HoverSurfaceState, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const owner = inferOwnerForSurface(surfaceState.element, now); + if (!owner) return undefined; + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + const node: HoverSurfaceNode = { + element: surfaceState.element, + signature: surfaceState.signature, + owner, + ...(parent ? { parent } : {}), + }; + hoverSurfaceNodes.set(surfaceState.element, node); + return node; + }; + const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => { const closestSurface = closestHoverSurfaceCandidate(target); if (closestSurface) { const owned = hoverSurfaceNodes.get(closestSurface); if (owned) return owned; } - return ownedSurfaceContaining(target); + const ownedContaining = ownedSurfaceContaining(target); + if (ownedContaining) return ownedContaining; + const now = Date.now(); + const currentStates = collectHoverSurfaceStates(); + pruneGoneHoverSurfaces(currentStates); + const surfaceState = closestSurface + ? currentStates.find((state) => state.element === closestSurface) + : surfaceStateContaining(target, currentStates); + hoverSurfaceStates = hoverSurfaceStateMap(currentStates); + if (!surfaceState) return undefined; + return nodeForUnownedSurface(surfaceState, currentStates, now); }; const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => { @@ -506,23 +554,6 @@ export function startRecordCapture( } }; - const inferOwnerForParentSurface = ( - surface: Element, - childOwner: HoverCandidate, - now: number, - ): HoverCandidate | undefined => { - for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { - const candidate = recentHoverCandidates[index]; - if (!candidate || candidate.element === childOwner.element) continue; - if (surface.contains(candidate.element)) continue; - if (candidate.recordedAt > childOwner.recordedAt) continue; - if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; - const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface); - if (relation.related) return candidate; - } - return undefined; - }; - const parentSurfaceNodeForOwner = ( owner: HoverCandidate, currentStates: HoverSurfaceState[], @@ -532,7 +563,7 @@ export function startRecordCapture( if (ownedParent) return ownedParent; const parentState = surfaceStateContaining(owner.element, currentStates); if (!parentState) return undefined; - const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now); + const parentOwner = inferOwnerForSurface(parentState.element, now, owner); if (!parentOwner) return undefined; const parentNode: HoverSurfaceNode = { element: parentState.element, @@ -543,44 +574,49 @@ export function startRecordCapture( return parentNode; }; - const bindOwnedHoverSurfaces = ( + const createSurfaceNode = ( + state: HoverSurfaceState, owner: HoverCandidate, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + if (parent?.element === state.element) return undefined; + if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined; + if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined; + const node: HoverSurfaceNode = { + element: state.element, + signature: state.signature, + owner, + ...(parent ? { parent } : {}), + }; + hoverSurfaceNodes.set(state.element, node); + return node; + }; + + const bindChangedHoverSurfaces = ( previousStates: Map, currentStates: HoverSurfaceState[], now: number, ) => { - const parent = parentSurfaceNodeForOwner(owner, currentStates, now); for (const state of currentStates) { if (previousStates.get(state.element) === state.signature) continue; - if (state.element.contains(owner.element)) continue; - if (parent?.element === state.element) continue; - const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element); - if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; - hoverSurfaceNodes.set(state.element, { - element: state.element, - signature: state.signature, - owner, - ...(parent ? { parent } : {}), - }); + const owner = inferOwnerForSurface(state.element, now); + if (!owner) continue; + createSurfaceNode(state, owner, currentStates, now); } }; - const refreshCausedHoverSurfaces = ( - owner: HoverCandidate, - previousStates = hoverSurfaceStates, - ) => { + const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => { const now = Date.now(); const currentStates = collectHoverSurfaceStates(); pruneGoneHoverSurfaces(currentStates); - bindOwnedHoverSurfaces(owner, previousStates, currentStates, now); - hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature])); + bindChangedHoverSurfaces(previousStates, currentStates, now); + hoverSurfaceStates = hoverSurfaceStateMap(currentStates); }; - const scheduleCausedHoverSurfaceRefresh = ( - hover: HoverCandidate, - previousStates: Map, - ) => { - setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0); + const scheduleHoverSurfaceRefresh = (previousStates: Map) => { + setTimeout(() => refreshHoverSurfaces(previousStates), 0); }; const emitHoverStep = (hover: HoverCandidate) => { @@ -619,6 +655,21 @@ export function startRecordCapture( return chain; }; + const containedHoverOwnerForAction = ( + actionElement: Element, + now: number, + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate) continue; + if (candidate.element === actionElement) continue; + if (!candidate.element.contains(actionElement)) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + return candidate; + } + return undefined; + }; + const emitClick = (event: MouseEvent) => { // Only record clicks an LLM can re-identify (named interactive controls). const target = describeEventTarget(eventTarget(event)); @@ -635,7 +686,13 @@ export function startRecordCapture( if (!(actionTarget instanceof Element)) return; const actionElement = resolveClickableElement(actionTarget) ?? actionTarget; const surface = ownedSurfaceForAction(actionElement); - const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : []; + const now = Date.now(); + const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now); + const hoverChain = surface + ? surfaceOwnerChain(surface, actionElement, now) + : containedOwner + ? [containedOwner] + : []; if (hoverChain.length === 0) { pendingHover = null; return; @@ -656,8 +713,8 @@ export function startRecordCapture( if (candidate) { const previousSurfaceStates = new Map(hoverSurfaceStates); rememberHoverCandidate(candidate); - refreshCausedHoverSurfaces(candidate, previousSurfaceStates); - scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates); + refreshHoverSurfaces(previousSurfaceStates); + scheduleHoverSurfaceRefresh(previousSurfaceStates); } const hover = candidate?.eligible ? candidate : null; if (pendingHover && target instanceof Element) { diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index e737cd3..a47b426 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -27,9 +27,9 @@ const HOVER_SURFACE_SELECTOR = [ ].join(","); const HOVER_SURFACE_WORD_RE = - /(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; + /(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; const HOVER_SURFACE_ITEM_RE = - /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; + /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; const HOVER_SURFACE_MAX_AXIS_GAP = 96; const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16; const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; @@ -54,10 +54,6 @@ export function isRecognizedHoverSurfaceElement(el: Element): boolean { if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) { return true; } - const tag = el.tagName.toLowerCase(); - if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) { - return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el)); - } return false; } @@ -84,7 +80,7 @@ export function closestRecognizedHoverSurface(element: Element): Element | null export function isPositionedFloatingElement(el: Element): boolean { if (!(el instanceof HTMLElement)) return false; const style = getComputedStyle(el); - if (!["absolute", "fixed"].includes(style.position)) return false; + if (style.position !== "absolute") return false; if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { return false; } @@ -149,10 +145,10 @@ function rectFor(el: Element): DOMRect | null { return rect; } -function isNearTrigger(surface: Element, trigger: Element): boolean { +function surfaceRelationMetrics(surface: Element, trigger: Element) { const surfaceRect = rectFor(surface); const triggerRect = rectFor(trigger); - if (!surfaceRect || !triggerRect) return true; + if (!surfaceRect || !triggerRect) return null; const horizontalOverlap = Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left); const verticalOverlap = @@ -167,6 +163,20 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { triggerRect.top - surfaceRect.bottom, surfaceRect.top - triggerRect.bottom, ); + return { + surfaceRect, + triggerRect, + horizontalOverlap, + verticalOverlap, + horizontalGap, + verticalGap, + }; +} + +function isNearTrigger(surface: Element, trigger: Element): boolean { + const metrics = surfaceRelationMetrics(surface, trigger); + if (!metrics) return true; + const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics; if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; return ( @@ -174,6 +184,33 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { ); } +export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean { + const metrics = surfaceRelationMetrics(surface, trigger); + if (!metrics) return true; + const { + surfaceRect, + triggerRect, + horizontalOverlap, + verticalOverlap, + horizontalGap, + verticalGap, + } = metrics; + const opensVertically = + horizontalOverlap > 0 && + verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP && + (triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8); + const opensSideways = + verticalOverlap > 0 && + horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && + (triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8); + const opensFromCorner = + horizontalGap > 0 && + verticalGap > 0 && + horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && + verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP; + return opensVertically || opensSideways || opensFromCorner; +} + export function decideHoverSurfaceRelation( context: HoverSurfaceContext, clickElement: Element,