@@ -630,6 +1104,7 @@ describe("record-capture semantic", () => {
const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step));
const trigger = document.querySelector(".tg-avatar")!;
const image = document.querySelector("img")!;
+ const menu = document.querySelector(".user-menu")!;
const item = document.querySelector("a")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
@@ -653,6 +1128,7 @@ describe("record-capture semantic", () => {
height: 32,
toJSON: () => ({}),
});
+ mockRect(menu, { left: 820, top: 48, width: 160, height: 80 });
vi.spyOn(window, "getComputedStyle").mockReturnValue({
cursor: "pointer",
pointerEvents: "auto",
@@ -717,6 +1193,7 @@ describe("record-capture semantic", () => {
`;
const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
+ const menu = document.querySelector(".user-menu")!;
const input = document.querySelector("input")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
@@ -729,6 +1206,7 @@ describe("record-capture semantic", () => {
height: 32,
toJSON: () => ({}),
});
+ mockRect(menu, { left: 820, top: 48, width: 180, height: 96 });
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
@@ -762,6 +1240,7 @@ describe("record-capture semantic", () => {
`;
const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step));
const trigger = document.querySelector("button")!;
+ const menu = document.querySelector(".user-menu")!;
const select = document.querySelector("select")!;
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
x: 900,
@@ -774,6 +1253,7 @@ describe("record-capture semantic", () => {
height: 32,
toJSON: () => ({}),
});
+ mockRect(menu, { left: 820, top: 48, width: 180, height: 96 });
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
select.value = "away";
diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts
index 530085b..d693415 100644
--- a/apps/extension/src/content/record-capture.ts
+++ b/apps/extension/src/content/record-capture.ts
@@ -28,7 +28,13 @@ import {
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
-import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface";
+import {
+ closestHoverSurfaceCandidate,
+ collectHoverSurfaceStates,
+ decideHoverSurfaceRelation,
+ type HoverSurfaceState,
+ isHoverSurfaceCandidateElement,
+} from "./record-hover-surface";
const pendingStepSends = new Map
>>();
const failedStepDeliveries = new Set();
@@ -68,16 +74,23 @@ interface HoverCandidate {
target: TargetDescriptor;
recordedAt: number;
score: number;
- hasExpansionSignal: boolean;
eligible: boolean;
}
+interface HoverSurfaceNode {
+ element: Element;
+ signature: string;
+ owner: HoverCandidate;
+ parent?: HoverSurfaceNode;
+}
+
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
const HOVER_REPLACE_SCORE_MARGIN = 50;
const HOVER_CANDIDATE_LIMIT = 24;
+const HOVER_TRIGGER_LABEL_MAX = 48;
function eventTarget(event: Event): EventTarget | null {
return event.composedPath()[0] ?? event.target;
@@ -191,6 +204,51 @@ function hasHoverGraphicDescendant(el: Element): boolean {
return el.querySelector("img,svg,use,path,i") !== null;
}
+function normalizeLabelText(value: string): string {
+ return value.replace(/\s+/g, " ").trim();
+}
+
+function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string {
+ const normalized = normalizeLabelText(value);
+ if (normalized.length <= max) return normalized;
+ return `${normalized.slice(0, max - 1)}…`;
+}
+
+function collectHoverTriggerLabelText(root: Element): string {
+ let text = "";
+ const visit = (node: Node) => {
+ if (node.nodeType === Node.TEXT_NODE) {
+ text += ` ${node.textContent ?? ""}`;
+ return;
+ }
+ if (!(node instanceof Element)) return;
+ if (node !== root && isHoverSurfaceCandidateElement(node)) {
+ return;
+ }
+ for (const child of node.childNodes) visit(child);
+ };
+ for (const child of root.childNodes) visit(child);
+ return normalizeLabelText(text);
+}
+
+function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
+ if (!desc.name) return desc;
+ const fullText = normalizeLabelText(el.textContent ?? "");
+ const compactName = collectHoverTriggerLabelText(el);
+ if (!compactName || compactName === desc.name) return desc;
+ const compactComparable = compactName.replace(/\s+/g, "");
+ const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, "");
+ const fullTextComparable = fullText.replace(/\s+/g, "");
+ if (
+ descComparable.startsWith(compactComparable) &&
+ fullTextComparable.startsWith(descComparable) &&
+ compactName.length < desc.name.length
+ ) {
+ return { ...desc, name: truncateLabel(compactName) };
+ }
+ return desc;
+}
+
function isWeakHoverTarget(target: TargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
@@ -245,13 +303,17 @@ function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | n
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
if (!decision.eligible && !desc.name && !desc.role) return null;
- const normalizedTarget = normalizeHoverTarget(element, desc, decision);
+ const normalizedTarget = normalizeHoverTarget(
+ element,
+ compactHoverTargetName(element, desc),
+ decision,
+ );
+ const now = Date.now();
return {
element,
target: normalizedTarget,
- recordedAt: Date.now(),
+ recordedAt: now,
score: decision.score,
- hasExpansionSignal,
eligible: decision.eligible,
};
}
@@ -321,6 +383,8 @@ export function startRecordCapture(
let pendingHover: HoverCandidate | null = null;
const recentHoverCandidates: HoverCandidate[] = [];
const emittedHoverElements = new WeakSet();
+ let hoverSurfaceStates = new Map();
+ const hoverSurfaceNodes = new Map();
let generatedControlClick: Element | null = null;
let navigationActionPending = false;
let navigationActionVersion = 0;
@@ -397,25 +461,126 @@ export function startRecordCapture(
}
};
- const actionIsOnHoverTriggerItself = (
- hover: HoverCandidate,
- actionElement: Element | null,
- relationTarget: Element,
- ): boolean => {
- const target = actionElement ?? relationTarget;
- if (target === hover.element) return true;
- if (!hover.element.contains(target)) return false;
- const surface = closestRecognizedHoverSurface(target);
- return !surface || surface === hover.element || !hover.element.contains(surface);
+ const surfaceArea = (el: Element): number => {
+ const rect = el.getBoundingClientRect();
+ return rect.width * rect.height;
};
- const hoverRelatedToAction = (
- hover: HoverCandidate,
- relationTarget: Element,
+ const smallestContaining = (
+ target: Element,
+ items: Iterable,
+ elementFor: (item: T) => Element,
+ ): T | undefined => {
+ let best: T | undefined;
+ for (const item of items) {
+ const element = elementFor(item);
+ if (!element.contains(target)) continue;
+ if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) {
+ best = item;
+ }
+ }
+ return best;
+ };
+
+ const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined =>
+ smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element);
+
+ const surfaceStateContaining = (
+ target: Element,
+ states: HoverSurfaceState[],
+ ): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
+
+ const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
+ const closestSurface = closestHoverSurfaceCandidate(target);
+ if (closestSurface) {
+ const owned = hoverSurfaceNodes.get(closestSurface);
+ if (owned) return owned;
+ }
+ return ownedSurfaceContaining(target);
+ };
+
+ const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
+ const current = new Set(currentStates.map((state) => state.element));
+ for (const element of hoverSurfaceNodes.keys()) {
+ if (!current.has(element)) hoverSurfaceNodes.delete(element);
+ }
+ };
+
+ const inferOwnerForParentSurface = (
+ surface: Element,
+ childOwner: HoverCandidate,
now: number,
- ): boolean => {
- if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false;
- return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related;
+ ): HoverCandidate | undefined => {
+ for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
+ const candidate = recentHoverCandidates[index];
+ if (!candidate || candidate.element === childOwner.element) continue;
+ if (surface.contains(candidate.element)) continue;
+ if (candidate.recordedAt > childOwner.recordedAt) continue;
+ if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
+ const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface);
+ if (relation.related) return candidate;
+ }
+ return undefined;
+ };
+
+ const parentSurfaceNodeForOwner = (
+ owner: HoverCandidate,
+ currentStates: HoverSurfaceState[],
+ now: number,
+ ): HoverSurfaceNode | undefined => {
+ const ownedParent = ownedSurfaceContaining(owner.element);
+ if (ownedParent) return ownedParent;
+ const parentState = surfaceStateContaining(owner.element, currentStates);
+ if (!parentState) return undefined;
+ const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now);
+ if (!parentOwner) return undefined;
+ const parentNode: HoverSurfaceNode = {
+ element: parentState.element,
+ signature: parentState.signature,
+ owner: parentOwner,
+ };
+ hoverSurfaceNodes.set(parentState.element, parentNode);
+ return parentNode;
+ };
+
+ const bindOwnedHoverSurfaces = (
+ owner: HoverCandidate,
+ previousStates: Map,
+ currentStates: HoverSurfaceState[],
+ now: number,
+ ) => {
+ const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
+ for (const state of currentStates) {
+ if (previousStates.get(state.element) === state.signature) continue;
+ if (state.element.contains(owner.element)) continue;
+ if (parent?.element === state.element) continue;
+ const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element);
+ if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
+ hoverSurfaceNodes.set(state.element, {
+ element: state.element,
+ signature: state.signature,
+ owner,
+ ...(parent ? { parent } : {}),
+ });
+ }
+ };
+
+ const refreshCausedHoverSurfaces = (
+ owner: HoverCandidate,
+ previousStates = hoverSurfaceStates,
+ ) => {
+ const now = Date.now();
+ const currentStates = collectHoverSurfaceStates();
+ pruneGoneHoverSurfaces(currentStates);
+ bindOwnedHoverSurfaces(owner, previousStates, currentStates, now);
+ hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature]));
+ };
+
+ const scheduleCausedHoverSurfaceRefresh = (
+ hover: HoverCandidate,
+ previousStates: Map,
+ ) => {
+ setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0);
};
const emitHoverStep = (hover: HoverCandidate) => {
@@ -427,25 +592,31 @@ export function startRecordCapture(
emittedHoverElements.add(hover.element);
};
- const findHoverForAction = (
- actionElement: Element | null,
- relationTarget: Element,
+ const surfaceOwnerChain = (
+ surface: HoverSurfaceNode,
+ actionElement: Element,
now: number,
- ): HoverCandidate | null => {
- const candidates = pendingHover
- ? [
- ...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element),
- pendingHover,
- ]
- : recentHoverCandidates;
- for (let index = candidates.length - 1; index >= 0; index -= 1) {
- const hover = candidates[index];
- if (!hover) continue;
- if (emittedHoverElements.has(hover.element)) continue;
- if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue;
- if (hoverRelatedToAction(hover, relationTarget, now)) return hover;
+ ): HoverCandidate[] => {
+ const chain: HoverCandidate[] = [];
+ const selected = new WeakSet();
+ const surfaces: HoverSurfaceNode[] = [];
+ let node: HoverSurfaceNode | undefined = surface;
+ while (node && surfaces.length < 4) {
+ surfaces.unshift(node);
+ node = node.parent;
}
- return null;
+ for (const owned of surfaces) {
+ const owner = owned.owner;
+ if (owner.element === actionElement || actionElement.contains(owner.element)) {
+ continue;
+ }
+ if (selected.has(owner.element)) continue;
+ if (emittedHoverElements.has(owner.element)) continue;
+ if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
+ selected.add(owner.element);
+ chain.push(owner);
+ }
+ return chain;
};
const emitClick = (event: MouseEvent) => {
@@ -462,14 +633,14 @@ export function startRecordCapture(
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
if (!(actionTarget instanceof Element)) return;
- const actionElement = resolveClickableElement(actionTarget);
- const relationTarget = actionElement ?? actionTarget;
- const hover = findHoverForAction(actionElement, relationTarget, Date.now());
- if (!hover) {
+ const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
+ const surface = ownedSurfaceForAction(actionElement);
+ const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : [];
+ if (hoverChain.length === 0) {
pendingHover = null;
return;
}
- emitHoverStep(hover);
+ for (const hover of hoverChain) emitHoverStep(hover);
pendingHover = null;
};
@@ -482,21 +653,16 @@ export function startRecordCapture(
pendingHover = null;
}
const candidate = hoverCandidateFromEvent(target);
- if (candidate) rememberHoverCandidate(candidate);
+ if (candidate) {
+ const previousSurfaceStates = new Map(hoverSurfaceStates);
+ rememberHoverCandidate(candidate);
+ refreshCausedHoverSurfaces(candidate, previousSurfaceStates);
+ scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates);
+ }
const hover = candidate?.eligible ? candidate : null;
if (pendingHover && target instanceof Element) {
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
- if (!hover || !hover.hasExpansionSignal) return;
- if (
- hover.element === pendingHover.element ||
- pendingHover.element.contains(hover.element)
- ) {
- return;
- }
- emitHoverStep(pendingHover);
- pendingHover = hover;
- rememberHoverCandidate(hover);
return;
}
}
@@ -505,7 +671,6 @@ export function startRecordCapture(
return;
}
pendingHover = hover;
- rememberHoverCandidate(hover);
};
const onClick = (event: MouseEvent) => {
diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts
index c6fbd43..e737cd3 100644
--- a/apps/extension/src/content/record-hover-surface.ts
+++ b/apps/extension/src/content/record-hover-surface.ts
@@ -8,6 +8,11 @@ export interface HoverSurfaceContext {
triggerElement: Element;
}
+export interface HoverSurfaceState {
+ element: Element;
+ signature: string;
+}
+
const HOVER_SURFACE_SELECTOR = [
'[role="menu"]',
'[role="menubar"]',
@@ -22,8 +27,11 @@ const HOVER_SURFACE_SELECTOR = [
].join(",");
const HOVER_SURFACE_WORD_RE =
- /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i;
+ /(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
+const HOVER_SURFACE_ITEM_RE =
+ /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
+const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
function elementTokenText(el: Element): string {
@@ -40,9 +48,12 @@ function elementTokenText(el: Element): string {
.join(" ");
}
-function isHoverSurfaceElement(el: Element): boolean {
+export function isRecognizedHoverSurfaceElement(el: Element): boolean {
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
- if (HOVER_SURFACE_WORD_RE.test(elementTokenText(el))) return true;
+ const tokenText = elementTokenText(el);
+ if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
+ return true;
+ }
const tag = el.tagName.toLowerCase();
if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) {
return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el));
@@ -50,18 +61,27 @@ function isHoverSurfaceElement(el: Element): boolean {
return false;
}
+function isVisibleSurfaceElement(el: Element): boolean {
+ if (!(el instanceof HTMLElement)) return false;
+ const style = getComputedStyle(el);
+ if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
+ return false;
+ }
+ return rectFor(el) !== null;
+}
+
export function closestRecognizedHoverSurface(element: Element): Element | null {
let node: Element | null = element;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
- if (isHoverSurfaceElement(node)) return node;
+ if (isRecognizedHoverSurfaceElement(node)) return node;
node = node.parentElement;
depth += 1;
}
return null;
}
-function isPositionedFloatingElement(el: Element): boolean {
+export function isPositionedFloatingElement(el: Element): boolean {
if (!(el instanceof HTMLElement)) return false;
const style = getComputedStyle(el);
if (!["absolute", "fixed"].includes(style.position)) return false;
@@ -77,6 +97,21 @@ function isPositionedFloatingElement(el: Element): boolean {
);
}
+export function isHoverSurfaceCandidateElement(el: Element): boolean {
+ return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el);
+}
+
+export function closestHoverSurfaceCandidate(element: Element): Element | null {
+ let node: Element | null = element;
+ let depth = 0;
+ while (node && node !== document.body && node !== document.documentElement && depth < 10) {
+ if (isHoverSurfaceCandidateElement(node)) return node;
+ node = node.parentElement;
+ depth += 1;
+ }
+ return null;
+}
+
function closestFloatingSurface(clickElement: Element): Element | null {
let node: Element | null = clickElement;
let depth = 0;
@@ -88,6 +123,26 @@ function closestFloatingSurface(clickElement: Element): Element | null {
return null;
}
+function hoverSurfaceSignature(el: Element): string {
+ const rect = rectFor(el);
+ return [
+ rect
+ ? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}`
+ : "",
+ (el.textContent ?? "").trim().slice(0, 512),
+ ].join("|");
+}
+
+export function collectHoverSurfaceStates(): HoverSurfaceState[] {
+ const states: HoverSurfaceState[] = [];
+ for (const el of document.querySelectorAll("*")) {
+ if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) {
+ states.push({ element: el, signature: hoverSurfaceSignature(el) });
+ }
+ }
+ return states;
+}
+
function rectFor(el: Element): DOMRect | null {
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return null;
@@ -98,6 +153,10 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
const surfaceRect = rectFor(surface);
const triggerRect = rectFor(trigger);
if (!surfaceRect || !triggerRect) return true;
+ const horizontalOverlap =
+ Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
+ const verticalOverlap =
+ Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top);
const horizontalGap = Math.max(
0,
triggerRect.left - surfaceRect.right,
@@ -108,7 +167,11 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
triggerRect.top - surfaceRect.bottom,
surfaceRect.top - triggerRect.bottom,
);
- return horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP;
+ if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
+ if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
+ return (
+ horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP
+ );
}
export function decideHoverSurfaceRelation(