feat(extension): hover chain node fix

This commit is contained in:
Ljy-0827
2026-08-07 17:37:28 +08:00
parent 2ccc0aafb9
commit 566b6c17fd
4 changed files with 359 additions and 55 deletions
@@ -1150,6 +1150,107 @@ describe("record-capture semantic", () => {
});
});
it("does not let unrelated topbar hovers or menu pass-through items own an avatar menu", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-07T08:08:45.000Z"));
try {
document.body.innerHTML = `
<button type="button" aria-label="Star">Star</button>
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
<img class="tg-avatar__image" />
</div>
<ul class="user-menu dropdown-menu">
<li><a href="/u/me">My profile</a></li>
<li><a href="/dashboard/groups">My groups</a></li>
</ul>
`;
const star = document.querySelector("button")!;
const avatar = document.querySelector(".tg-avatar")!;
const image = document.querySelector("img")!;
const menu = document.querySelector(".user-menu")!;
const profile = document.querySelector('a[href="/u/me"]')!;
const groups = document.querySelector('a[href="/dashboard/groups"]')!;
let menuRect = { left: 0, top: 0, width: 0, height: 0 };
let profileRect = { left: 0, top: 0, width: 0, height: 0 };
let groupsRect = { left: 0, top: 0, width: 0, height: 0 };
mockRect(star, { left: 760, top: 8, width: 60, height: 32 });
mockRect(avatar, { left: 900, top: 8, width: 32, height: 32 });
mockRect(image, { left: 900, top: 8, width: 32, height: 32 });
vi.spyOn(menu, "getBoundingClientRect").mockImplementation(
() =>
({
x: menuRect.left,
y: menuRect.top,
top: menuRect.top,
left: menuRect.left,
right: menuRect.left + menuRect.width,
bottom: menuRect.top + menuRect.height,
width: menuRect.width,
height: menuRect.height,
toJSON: () => ({}),
}) as DOMRect,
);
vi.spyOn(profile, "getBoundingClientRect").mockImplementation(
() =>
({
x: profileRect.left,
y: profileRect.top,
top: profileRect.top,
left: profileRect.left,
right: profileRect.left + profileRect.width,
bottom: profileRect.top + profileRect.height,
width: profileRect.width,
height: profileRect.height,
toJSON: () => ({}),
}) as DOMRect,
);
vi.spyOn(groups, "getBoundingClientRect").mockImplementation(
() =>
({
x: groupsRect.left,
y: groupsRect.top,
top: groupsRect.top,
left: groupsRect.left,
right: groupsRect.left + groupsRect.width,
bottom: groupsRect.top + groupsRect.height,
width: groupsRect.width,
height: groupsRect.height,
toJSON: () => ({}),
}) as DOMRect,
);
mockHoverStyle([star, avatar, image, profile, groups]);
const capture = startRecordCapture("rec-hover-avatar-menu-pass-through", (step) =>
steps.push(step),
);
mouseOver(star);
mouseOver(avatar);
mouseOver(image);
menuRect = { left: 820, top: 48, width: 160, height: 96 };
profileRect = { left: 830, top: 56, width: 120, height: 28 };
groupsRect = { left: 830, top: 88, width: 120, height: 28 };
vi.runOnlyPendingTimers();
mouseOver(profile);
mouseOver(groups);
click(groups);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
target: { tag: "img", role: "img", name: "image" },
});
expect(steps[1]).toMatchObject({
op: "click",
target: { role: "link", name: "My groups" },
});
expect(JSON.stringify(steps)).not.toContain("Star");
expect(JSON.stringify(steps)).not.toContain("My profile");
} finally {
vi.useRealTimers();
}
});
it("does not record hover before an unrelated page click", () => {
document.body.innerHTML = `
<button type="button" aria-label="Open user navigation menu">avatar</button>
@@ -0,0 +1,109 @@
import { describe, expect, it, vi } from "vitest";
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
function mockRect(
el: Element,
rect: { left: number; top: number; width: number; height: number },
): void {
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
x: rect.left,
y: rect.top,
top: rect.top,
left: rect.left,
right: rect.left + rect.width,
bottom: rect.top + rect.height,
width: rect.width,
height: rect.height,
toJSON: () => ({}),
});
}
describe("record-hover-surface", () => {
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
document.body.innerHTML = `
<ul class="header-menu tw-flex navbar-app__authentication-menu">
<li class="tg-popover">
<div class="tg-avatar tg-avatar--img"></div>
</li>
</ul>
<div class="tg-popover__popper">
<div class="tg-popover__content">
<div class="tg-popover__content-body">
<li class="header-menu__item">
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
</li>
<li class="header-menu__item">
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
</li>
</div>
</div>
</div>
`;
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
false,
);
expect(
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
).toBe(false);
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
true,
);
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
true,
);
expect(
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
).toBe(true);
});
it("does not treat elements covered by an open surface as the surface owner", () => {
document.body.innerHTML = `
<button class="avatar">image</button>
<a class="fork" href="/fork">Fork</a>
<button class="clone">Clone & Download</button>
<div class="tg-popover__content-body"></div>
`;
const avatar = document.querySelector(".avatar")!;
const fork = document.querySelector(".fork")!;
const clone = document.querySelector(".clone")!;
const surface = document.querySelector(".tg-popover__content-body")!;
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
});
it("does not treat fixed navigation containers as plain floating surfaces", () => {
document.body.innerHTML = `
<header class="navbar-app">
<a href="/fork">Fork</a>
<button>image</button>
</header>
<div class="plain-floating"></div>
`;
const navbar = document.querySelector(".navbar-app")!;
const floating = document.querySelector(".plain-floating")!;
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
const style = {
cursor: "",
display: "block",
pointerEvents: "auto",
position: el === navbar ? "fixed" : "absolute",
visibility: "visible",
} as CSSStyleDeclaration;
return style;
});
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
});
});
+103 -46
View File
@@ -34,6 +34,7 @@ import {
decideHoverSurfaceRelation,
type HoverSurfaceState,
isHoverSurfaceCandidateElement,
isLikelyHoverSurfaceOwner,
} from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
@@ -376,6 +377,8 @@ export function startRecordCapture(
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
};
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
new Map(states.map((state) => [state.element, state.signature]));
let fillSession: FillSession | null = null;
let composing = false;
let lastUrl = location.href;
@@ -383,7 +386,7 @@ export function startRecordCapture(
let pendingHover: HoverCandidate | null = null;
const recentHoverCandidates: HoverCandidate[] = [];
const emittedHoverElements = new WeakSet<Element>();
let hoverSurfaceStates = new Map<Element, string>();
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
let generatedControlClick: Element | null = null;
let navigationActionPending = false;
@@ -490,13 +493,58 @@ export function startRecordCapture(
states: HoverSurfaceState[],
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
const inferOwnerForSurface = (
surface: Element,
now: number,
before?: HoverCandidate,
): HoverCandidate | undefined => {
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
const candidate = recentHoverCandidates[index];
if (!candidate) continue;
if (before && candidate.element === before.element) continue;
if (before && candidate.recordedAt > before.recordedAt) continue;
if (surface.contains(candidate.element)) continue;
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
}
return undefined;
};
const nodeForUnownedSurface = (
surfaceState: HoverSurfaceState,
currentStates: HoverSurfaceState[],
now: number,
): HoverSurfaceNode | undefined => {
const owner = inferOwnerForSurface(surfaceState.element, now);
if (!owner) return undefined;
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
const node: HoverSurfaceNode = {
element: surfaceState.element,
signature: surfaceState.signature,
owner,
...(parent ? { parent } : {}),
};
hoverSurfaceNodes.set(surfaceState.element, node);
return node;
};
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
const closestSurface = closestHoverSurfaceCandidate(target);
if (closestSurface) {
const owned = hoverSurfaceNodes.get(closestSurface);
if (owned) return owned;
}
return ownedSurfaceContaining(target);
const ownedContaining = ownedSurfaceContaining(target);
if (ownedContaining) return ownedContaining;
const now = Date.now();
const currentStates = collectHoverSurfaceStates();
pruneGoneHoverSurfaces(currentStates);
const surfaceState = closestSurface
? currentStates.find((state) => state.element === closestSurface)
: surfaceStateContaining(target, currentStates);
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
if (!surfaceState) return undefined;
return nodeForUnownedSurface(surfaceState, currentStates, now);
};
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
@@ -506,23 +554,6 @@ export function startRecordCapture(
}
};
const inferOwnerForParentSurface = (
surface: Element,
childOwner: HoverCandidate,
now: number,
): HoverCandidate | undefined => {
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
const candidate = recentHoverCandidates[index];
if (!candidate || candidate.element === childOwner.element) continue;
if (surface.contains(candidate.element)) continue;
if (candidate.recordedAt > childOwner.recordedAt) continue;
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface);
if (relation.related) return candidate;
}
return undefined;
};
const parentSurfaceNodeForOwner = (
owner: HoverCandidate,
currentStates: HoverSurfaceState[],
@@ -532,7 +563,7 @@ export function startRecordCapture(
if (ownedParent) return ownedParent;
const parentState = surfaceStateContaining(owner.element, currentStates);
if (!parentState) return undefined;
const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now);
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
if (!parentOwner) return undefined;
const parentNode: HoverSurfaceNode = {
element: parentState.element,
@@ -543,44 +574,49 @@ export function startRecordCapture(
return parentNode;
};
const bindOwnedHoverSurfaces = (
const createSurfaceNode = (
state: HoverSurfaceState,
owner: HoverCandidate,
currentStates: HoverSurfaceState[],
now: number,
): HoverSurfaceNode | undefined => {
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
if (parent?.element === state.element) return undefined;
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
const node: HoverSurfaceNode = {
element: state.element,
signature: state.signature,
owner,
...(parent ? { parent } : {}),
};
hoverSurfaceNodes.set(state.element, node);
return node;
};
const bindChangedHoverSurfaces = (
previousStates: Map<Element, string>,
currentStates: HoverSurfaceState[],
now: number,
) => {
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
for (const state of currentStates) {
if (previousStates.get(state.element) === state.signature) continue;
if (state.element.contains(owner.element)) continue;
if (parent?.element === state.element) continue;
const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element);
if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
hoverSurfaceNodes.set(state.element, {
element: state.element,
signature: state.signature,
owner,
...(parent ? { parent } : {}),
});
const owner = inferOwnerForSurface(state.element, now);
if (!owner) continue;
createSurfaceNode(state, owner, currentStates, now);
}
};
const refreshCausedHoverSurfaces = (
owner: HoverCandidate,
previousStates = hoverSurfaceStates,
) => {
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
const now = Date.now();
const currentStates = collectHoverSurfaceStates();
pruneGoneHoverSurfaces(currentStates);
bindOwnedHoverSurfaces(owner, previousStates, currentStates, now);
hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature]));
bindChangedHoverSurfaces(previousStates, currentStates, now);
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
};
const scheduleCausedHoverSurfaceRefresh = (
hover: HoverCandidate,
previousStates: Map<Element, string>,
) => {
setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0);
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
};
const emitHoverStep = (hover: HoverCandidate) => {
@@ -619,6 +655,21 @@ export function startRecordCapture(
return chain;
};
const containedHoverOwnerForAction = (
actionElement: Element,
now: number,
): HoverCandidate | undefined => {
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
const candidate = recentHoverCandidates[index];
if (!candidate) continue;
if (candidate.element === actionElement) continue;
if (!candidate.element.contains(actionElement)) continue;
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
return candidate;
}
return undefined;
};
const emitClick = (event: MouseEvent) => {
// Only record clicks an LLM can re-identify (named interactive controls).
const target = describeEventTarget(eventTarget(event));
@@ -635,7 +686,13 @@ export function startRecordCapture(
if (!(actionTarget instanceof Element)) return;
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
const surface = ownedSurfaceForAction(actionElement);
const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : [];
const now = Date.now();
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
const hoverChain = surface
? surfaceOwnerChain(surface, actionElement, now)
: containedOwner
? [containedOwner]
: [];
if (hoverChain.length === 0) {
pendingHover = null;
return;
@@ -656,8 +713,8 @@ export function startRecordCapture(
if (candidate) {
const previousSurfaceStates = new Map(hoverSurfaceStates);
rememberHoverCandidate(candidate);
refreshCausedHoverSurfaces(candidate, previousSurfaceStates);
scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates);
refreshHoverSurfaces(previousSurfaceStates);
scheduleHoverSurfaceRefresh(previousSurfaceStates);
}
const hover = candidate?.eligible ? candidate : null;
if (pendingHover && target instanceof Element) {
@@ -27,9 +27,9 @@ const HOVER_SURFACE_SELECTOR = [
].join(",");
const HOVER_SURFACE_WORD_RE =
/(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
const HOVER_SURFACE_ITEM_RE =
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
@@ -54,10 +54,6 @@ export function isRecognizedHoverSurfaceElement(el: Element): boolean {
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
return true;
}
const tag = el.tagName.toLowerCase();
if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) {
return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el));
}
return false;
}
@@ -84,7 +80,7 @@ export function closestRecognizedHoverSurface(element: Element): Element | null
export function isPositionedFloatingElement(el: Element): boolean {
if (!(el instanceof HTMLElement)) return false;
const style = getComputedStyle(el);
if (!["absolute", "fixed"].includes(style.position)) return false;
if (style.position !== "absolute") return false;
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
return false;
}
@@ -149,10 +145,10 @@ function rectFor(el: Element): DOMRect | null {
return rect;
}
function isNearTrigger(surface: Element, trigger: Element): boolean {
function surfaceRelationMetrics(surface: Element, trigger: Element) {
const surfaceRect = rectFor(surface);
const triggerRect = rectFor(trigger);
if (!surfaceRect || !triggerRect) return true;
if (!surfaceRect || !triggerRect) return null;
const horizontalOverlap =
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
const verticalOverlap =
@@ -167,6 +163,20 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
triggerRect.top - surfaceRect.bottom,
surfaceRect.top - triggerRect.bottom,
);
return {
surfaceRect,
triggerRect,
horizontalOverlap,
verticalOverlap,
horizontalGap,
verticalGap,
};
}
function isNearTrigger(surface: Element, trigger: Element): boolean {
const metrics = surfaceRelationMetrics(surface, trigger);
if (!metrics) return true;
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
return (
@@ -174,6 +184,33 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
);
}
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
const metrics = surfaceRelationMetrics(surface, trigger);
if (!metrics) return true;
const {
surfaceRect,
triggerRect,
horizontalOverlap,
verticalOverlap,
horizontalGap,
verticalGap,
} = metrics;
const opensVertically =
horizontalOverlap > 0 &&
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
const opensSideways =
verticalOverlap > 0 &&
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
const opensFromCorner =
horizontalGap > 0 &&
verticalGap > 0 &&
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
return opensVertically || opensSideways || opensFromCorner;
}
export function decideHoverSurfaceRelation(
context: HoverSurfaceContext,
clickElement: Element,