mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): hover chain node fix
This commit is contained in:
@@ -1150,6 +1150,107 @@ describe("record-capture semantic", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not let unrelated topbar hovers or menu pass-through items own an avatar menu", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date("2026-08-07T08:08:45.000Z"));
|
||||||
|
try {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="Star">Star</button>
|
||||||
|
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
|
||||||
|
<img class="tg-avatar__image" />
|
||||||
|
</div>
|
||||||
|
<ul class="user-menu dropdown-menu">
|
||||||
|
<li><a href="/u/me">My profile</a></li>
|
||||||
|
<li><a href="/dashboard/groups">My groups</a></li>
|
||||||
|
</ul>
|
||||||
|
`;
|
||||||
|
const star = document.querySelector("button")!;
|
||||||
|
const avatar = document.querySelector(".tg-avatar")!;
|
||||||
|
const image = document.querySelector("img")!;
|
||||||
|
const menu = document.querySelector(".user-menu")!;
|
||||||
|
const profile = document.querySelector('a[href="/u/me"]')!;
|
||||||
|
const groups = document.querySelector('a[href="/dashboard/groups"]')!;
|
||||||
|
let menuRect = { left: 0, top: 0, width: 0, height: 0 };
|
||||||
|
let profileRect = { left: 0, top: 0, width: 0, height: 0 };
|
||||||
|
let groupsRect = { left: 0, top: 0, width: 0, height: 0 };
|
||||||
|
mockRect(star, { left: 760, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(avatar, { left: 900, top: 8, width: 32, height: 32 });
|
||||||
|
mockRect(image, { left: 900, top: 8, width: 32, height: 32 });
|
||||||
|
vi.spyOn(menu, "getBoundingClientRect").mockImplementation(
|
||||||
|
() =>
|
||||||
|
({
|
||||||
|
x: menuRect.left,
|
||||||
|
y: menuRect.top,
|
||||||
|
top: menuRect.top,
|
||||||
|
left: menuRect.left,
|
||||||
|
right: menuRect.left + menuRect.width,
|
||||||
|
bottom: menuRect.top + menuRect.height,
|
||||||
|
width: menuRect.width,
|
||||||
|
height: menuRect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
}) as DOMRect,
|
||||||
|
);
|
||||||
|
vi.spyOn(profile, "getBoundingClientRect").mockImplementation(
|
||||||
|
() =>
|
||||||
|
({
|
||||||
|
x: profileRect.left,
|
||||||
|
y: profileRect.top,
|
||||||
|
top: profileRect.top,
|
||||||
|
left: profileRect.left,
|
||||||
|
right: profileRect.left + profileRect.width,
|
||||||
|
bottom: profileRect.top + profileRect.height,
|
||||||
|
width: profileRect.width,
|
||||||
|
height: profileRect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
}) as DOMRect,
|
||||||
|
);
|
||||||
|
vi.spyOn(groups, "getBoundingClientRect").mockImplementation(
|
||||||
|
() =>
|
||||||
|
({
|
||||||
|
x: groupsRect.left,
|
||||||
|
y: groupsRect.top,
|
||||||
|
top: groupsRect.top,
|
||||||
|
left: groupsRect.left,
|
||||||
|
right: groupsRect.left + groupsRect.width,
|
||||||
|
bottom: groupsRect.top + groupsRect.height,
|
||||||
|
width: groupsRect.width,
|
||||||
|
height: groupsRect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
}) as DOMRect,
|
||||||
|
);
|
||||||
|
mockHoverStyle([star, avatar, image, profile, groups]);
|
||||||
|
const capture = startRecordCapture("rec-hover-avatar-menu-pass-through", (step) =>
|
||||||
|
steps.push(step),
|
||||||
|
);
|
||||||
|
|
||||||
|
mouseOver(star);
|
||||||
|
mouseOver(avatar);
|
||||||
|
mouseOver(image);
|
||||||
|
menuRect = { left: 820, top: 48, width: 160, height: 96 };
|
||||||
|
profileRect = { left: 830, top: 56, width: 120, height: 28 };
|
||||||
|
groupsRect = { left: 830, top: 88, width: 120, height: 28 };
|
||||||
|
vi.runOnlyPendingTimers();
|
||||||
|
mouseOver(profile);
|
||||||
|
mouseOver(groups);
|
||||||
|
click(groups);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "img", role: "img", name: "image" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { role: "link", name: "My groups" },
|
||||||
|
});
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("Star");
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("My profile");
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("does not record hover before an unrelated page click", () => {
|
it("does not record hover before an unrelated page click", () => {
|
||||||
document.body.innerHTML = `
|
document.body.innerHTML = `
|
||||||
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
|
||||||
|
|
||||||
|
function mockRect(
|
||||||
|
el: Element,
|
||||||
|
rect: { left: number; top: number; width: number; height: number },
|
||||||
|
): void {
|
||||||
|
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
top: rect.top,
|
||||||
|
left: rect.left,
|
||||||
|
right: rect.left + rect.width,
|
||||||
|
bottom: rect.top + rect.height,
|
||||||
|
width: rect.width,
|
||||||
|
height: rect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record-hover-surface", () => {
|
||||||
|
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<ul class="header-menu tw-flex navbar-app__authentication-menu">
|
||||||
|
<li class="tg-popover">
|
||||||
|
<div class="tg-avatar tg-avatar--img"></div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div class="tg-popover__popper">
|
||||||
|
<div class="tg-popover__content">
|
||||||
|
<div class="tg-popover__content-body">
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
|
||||||
|
</li>
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
|
||||||
|
</li>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
|
||||||
|
).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat elements covered by an open surface as the surface owner", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button class="avatar">image</button>
|
||||||
|
<a class="fork" href="/fork">Fork</a>
|
||||||
|
<button class="clone">Clone & Download</button>
|
||||||
|
<div class="tg-popover__content-body"></div>
|
||||||
|
`;
|
||||||
|
const avatar = document.querySelector(".avatar")!;
|
||||||
|
const fork = document.querySelector(".fork")!;
|
||||||
|
const clone = document.querySelector(".clone")!;
|
||||||
|
const surface = document.querySelector(".tg-popover__content-body")!;
|
||||||
|
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
|
||||||
|
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
|
||||||
|
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
|
||||||
|
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
|
||||||
|
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat fixed navigation containers as plain floating surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<header class="navbar-app">
|
||||||
|
<a href="/fork">Fork</a>
|
||||||
|
<button>image</button>
|
||||||
|
</header>
|
||||||
|
<div class="plain-floating"></div>
|
||||||
|
`;
|
||||||
|
const navbar = document.querySelector(".navbar-app")!;
|
||||||
|
const floating = document.querySelector(".plain-floating")!;
|
||||||
|
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
|
||||||
|
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||||
|
const style = {
|
||||||
|
cursor: "",
|
||||||
|
display: "block",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
position: el === navbar ? "fixed" : "absolute",
|
||||||
|
visibility: "visible",
|
||||||
|
} as CSSStyleDeclaration;
|
||||||
|
return style;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -34,6 +34,7 @@ import {
|
|||||||
decideHoverSurfaceRelation,
|
decideHoverSurfaceRelation,
|
||||||
type HoverSurfaceState,
|
type HoverSurfaceState,
|
||||||
isHoverSurfaceCandidateElement,
|
isHoverSurfaceCandidateElement,
|
||||||
|
isLikelyHoverSurfaceOwner,
|
||||||
} from "./record-hover-surface";
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
@@ -376,6 +377,8 @@ export function startRecordCapture(
|
|||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
};
|
};
|
||||||
|
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||||
|
new Map(states.map((state) => [state.element, state.signature]));
|
||||||
let fillSession: FillSession | null = null;
|
let fillSession: FillSession | null = null;
|
||||||
let composing = false;
|
let composing = false;
|
||||||
let lastUrl = location.href;
|
let lastUrl = location.href;
|
||||||
@@ -383,7 +386,7 @@ export function startRecordCapture(
|
|||||||
let pendingHover: HoverCandidate | null = null;
|
let pendingHover: HoverCandidate | null = null;
|
||||||
const recentHoverCandidates: HoverCandidate[] = [];
|
const recentHoverCandidates: HoverCandidate[] = [];
|
||||||
const emittedHoverElements = new WeakSet<Element>();
|
const emittedHoverElements = new WeakSet<Element>();
|
||||||
let hoverSurfaceStates = new Map<Element, string>();
|
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
|
||||||
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
|
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
|
||||||
let generatedControlClick: Element | null = null;
|
let generatedControlClick: Element | null = null;
|
||||||
let navigationActionPending = false;
|
let navigationActionPending = false;
|
||||||
@@ -490,13 +493,58 @@ export function startRecordCapture(
|
|||||||
states: HoverSurfaceState[],
|
states: HoverSurfaceState[],
|
||||||
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
|
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
|
||||||
|
|
||||||
|
const inferOwnerForSurface = (
|
||||||
|
surface: Element,
|
||||||
|
now: number,
|
||||||
|
before?: HoverCandidate,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (before && candidate.element === before.element) continue;
|
||||||
|
if (before && candidate.recordedAt > before.recordedAt) continue;
|
||||||
|
if (surface.contains(candidate.element)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
const nodeForUnownedSurface = (
|
||||||
|
surfaceState: HoverSurfaceState,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const owner = inferOwnerForSurface(surfaceState.element, now);
|
||||||
|
if (!owner) return undefined;
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: surfaceState.element,
|
||||||
|
signature: surfaceState.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(surfaceState.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
|
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
|
||||||
const closestSurface = closestHoverSurfaceCandidate(target);
|
const closestSurface = closestHoverSurfaceCandidate(target);
|
||||||
if (closestSurface) {
|
if (closestSurface) {
|
||||||
const owned = hoverSurfaceNodes.get(closestSurface);
|
const owned = hoverSurfaceNodes.get(closestSurface);
|
||||||
if (owned) return owned;
|
if (owned) return owned;
|
||||||
}
|
}
|
||||||
return ownedSurfaceContaining(target);
|
const ownedContaining = ownedSurfaceContaining(target);
|
||||||
|
if (ownedContaining) return ownedContaining;
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
const surfaceState = closestSurface
|
||||||
|
? currentStates.find((state) => state.element === closestSurface)
|
||||||
|
: surfaceStateContaining(target, currentStates);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
if (!surfaceState) return undefined;
|
||||||
|
return nodeForUnownedSurface(surfaceState, currentStates, now);
|
||||||
};
|
};
|
||||||
|
|
||||||
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
|
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
|
||||||
@@ -506,23 +554,6 @@ export function startRecordCapture(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const inferOwnerForParentSurface = (
|
|
||||||
surface: Element,
|
|
||||||
childOwner: HoverCandidate,
|
|
||||||
now: number,
|
|
||||||
): HoverCandidate | undefined => {
|
|
||||||
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
|
||||||
const candidate = recentHoverCandidates[index];
|
|
||||||
if (!candidate || candidate.element === childOwner.element) continue;
|
|
||||||
if (surface.contains(candidate.element)) continue;
|
|
||||||
if (candidate.recordedAt > childOwner.recordedAt) continue;
|
|
||||||
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
|
||||||
const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface);
|
|
||||||
if (relation.related) return candidate;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
};
|
|
||||||
|
|
||||||
const parentSurfaceNodeForOwner = (
|
const parentSurfaceNodeForOwner = (
|
||||||
owner: HoverCandidate,
|
owner: HoverCandidate,
|
||||||
currentStates: HoverSurfaceState[],
|
currentStates: HoverSurfaceState[],
|
||||||
@@ -532,7 +563,7 @@ export function startRecordCapture(
|
|||||||
if (ownedParent) return ownedParent;
|
if (ownedParent) return ownedParent;
|
||||||
const parentState = surfaceStateContaining(owner.element, currentStates);
|
const parentState = surfaceStateContaining(owner.element, currentStates);
|
||||||
if (!parentState) return undefined;
|
if (!parentState) return undefined;
|
||||||
const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now);
|
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
|
||||||
if (!parentOwner) return undefined;
|
if (!parentOwner) return undefined;
|
||||||
const parentNode: HoverSurfaceNode = {
|
const parentNode: HoverSurfaceNode = {
|
||||||
element: parentState.element,
|
element: parentState.element,
|
||||||
@@ -543,44 +574,49 @@ export function startRecordCapture(
|
|||||||
return parentNode;
|
return parentNode;
|
||||||
};
|
};
|
||||||
|
|
||||||
const bindOwnedHoverSurfaces = (
|
const createSurfaceNode = (
|
||||||
|
state: HoverSurfaceState,
|
||||||
owner: HoverCandidate,
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
if (parent?.element === state.element) return undefined;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
|
||||||
|
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: state.element,
|
||||||
|
signature: state.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(state.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const bindChangedHoverSurfaces = (
|
||||||
previousStates: Map<Element, string>,
|
previousStates: Map<Element, string>,
|
||||||
currentStates: HoverSurfaceState[],
|
currentStates: HoverSurfaceState[],
|
||||||
now: number,
|
now: number,
|
||||||
) => {
|
) => {
|
||||||
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
|
||||||
for (const state of currentStates) {
|
for (const state of currentStates) {
|
||||||
if (previousStates.get(state.element) === state.signature) continue;
|
if (previousStates.get(state.element) === state.signature) continue;
|
||||||
if (state.element.contains(owner.element)) continue;
|
const owner = inferOwnerForSurface(state.element, now);
|
||||||
if (parent?.element === state.element) continue;
|
if (!owner) continue;
|
||||||
const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element);
|
createSurfaceNode(state, owner, currentStates, now);
|
||||||
if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
|
||||||
hoverSurfaceNodes.set(state.element, {
|
|
||||||
element: state.element,
|
|
||||||
signature: state.signature,
|
|
||||||
owner,
|
|
||||||
...(parent ? { parent } : {}),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const refreshCausedHoverSurfaces = (
|
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
|
||||||
owner: HoverCandidate,
|
|
||||||
previousStates = hoverSurfaceStates,
|
|
||||||
) => {
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const currentStates = collectHoverSurfaceStates();
|
const currentStates = collectHoverSurfaceStates();
|
||||||
pruneGoneHoverSurfaces(currentStates);
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
bindOwnedHoverSurfaces(owner, previousStates, currentStates, now);
|
bindChangedHoverSurfaces(previousStates, currentStates, now);
|
||||||
hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature]));
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
};
|
};
|
||||||
|
|
||||||
const scheduleCausedHoverSurfaceRefresh = (
|
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
|
||||||
hover: HoverCandidate,
|
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
|
||||||
previousStates: Map<Element, string>,
|
|
||||||
) => {
|
|
||||||
setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const emitHoverStep = (hover: HoverCandidate) => {
|
const emitHoverStep = (hover: HoverCandidate) => {
|
||||||
@@ -619,6 +655,21 @@ export function startRecordCapture(
|
|||||||
return chain;
|
return chain;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const containedHoverOwnerForAction = (
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (candidate.element === actionElement) continue;
|
||||||
|
if (!candidate.element.contains(actionElement)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
const emitClick = (event: MouseEvent) => {
|
const emitClick = (event: MouseEvent) => {
|
||||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||||
const target = describeEventTarget(eventTarget(event));
|
const target = describeEventTarget(eventTarget(event));
|
||||||
@@ -635,7 +686,13 @@ export function startRecordCapture(
|
|||||||
if (!(actionTarget instanceof Element)) return;
|
if (!(actionTarget instanceof Element)) return;
|
||||||
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
|
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
|
||||||
const surface = ownedSurfaceForAction(actionElement);
|
const surface = ownedSurfaceForAction(actionElement);
|
||||||
const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : [];
|
const now = Date.now();
|
||||||
|
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
|
||||||
|
const hoverChain = surface
|
||||||
|
? surfaceOwnerChain(surface, actionElement, now)
|
||||||
|
: containedOwner
|
||||||
|
? [containedOwner]
|
||||||
|
: [];
|
||||||
if (hoverChain.length === 0) {
|
if (hoverChain.length === 0) {
|
||||||
pendingHover = null;
|
pendingHover = null;
|
||||||
return;
|
return;
|
||||||
@@ -656,8 +713,8 @@ export function startRecordCapture(
|
|||||||
if (candidate) {
|
if (candidate) {
|
||||||
const previousSurfaceStates = new Map(hoverSurfaceStates);
|
const previousSurfaceStates = new Map(hoverSurfaceStates);
|
||||||
rememberHoverCandidate(candidate);
|
rememberHoverCandidate(candidate);
|
||||||
refreshCausedHoverSurfaces(candidate, previousSurfaceStates);
|
refreshHoverSurfaces(previousSurfaceStates);
|
||||||
scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates);
|
scheduleHoverSurfaceRefresh(previousSurfaceStates);
|
||||||
}
|
}
|
||||||
const hover = candidate?.eligible ? candidate : null;
|
const hover = candidate?.eligible ? candidate : null;
|
||||||
if (pendingHover && target instanceof Element) {
|
if (pendingHover && target instanceof Element) {
|
||||||
|
|||||||
@@ -27,9 +27,9 @@ const HOVER_SURFACE_SELECTOR = [
|
|||||||
].join(",");
|
].join(",");
|
||||||
|
|
||||||
const HOVER_SURFACE_WORD_RE =
|
const HOVER_SURFACE_WORD_RE =
|
||||||
/(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
|
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
|
||||||
const HOVER_SURFACE_ITEM_RE =
|
const HOVER_SURFACE_ITEM_RE =
|
||||||
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
|
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
|
||||||
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
||||||
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
|
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
|
||||||
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
||||||
@@ -54,10 +54,6 @@ export function isRecognizedHoverSurfaceElement(el: Element): boolean {
|
|||||||
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
|
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
const tag = el.tagName.toLowerCase();
|
|
||||||
if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) {
|
|
||||||
return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el));
|
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,7 +80,7 @@ export function closestRecognizedHoverSurface(element: Element): Element | null
|
|||||||
export function isPositionedFloatingElement(el: Element): boolean {
|
export function isPositionedFloatingElement(el: Element): boolean {
|
||||||
if (!(el instanceof HTMLElement)) return false;
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
const style = getComputedStyle(el);
|
const style = getComputedStyle(el);
|
||||||
if (!["absolute", "fixed"].includes(style.position)) return false;
|
if (style.position !== "absolute") return false;
|
||||||
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -149,10 +145,10 @@ function rectFor(el: Element): DOMRect | null {
|
|||||||
return rect;
|
return rect;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isNearTrigger(surface: Element, trigger: Element): boolean {
|
function surfaceRelationMetrics(surface: Element, trigger: Element) {
|
||||||
const surfaceRect = rectFor(surface);
|
const surfaceRect = rectFor(surface);
|
||||||
const triggerRect = rectFor(trigger);
|
const triggerRect = rectFor(trigger);
|
||||||
if (!surfaceRect || !triggerRect) return true;
|
if (!surfaceRect || !triggerRect) return null;
|
||||||
const horizontalOverlap =
|
const horizontalOverlap =
|
||||||
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
|
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
|
||||||
const verticalOverlap =
|
const verticalOverlap =
|
||||||
@@ -167,6 +163,20 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
|
|||||||
triggerRect.top - surfaceRect.bottom,
|
triggerRect.top - surfaceRect.bottom,
|
||||||
surfaceRect.top - triggerRect.bottom,
|
surfaceRect.top - triggerRect.bottom,
|
||||||
);
|
);
|
||||||
|
return {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isNearTrigger(surface: Element, trigger: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
|
||||||
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
return (
|
return (
|
||||||
@@ -174,6 +184,33 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
} = metrics;
|
||||||
|
const opensVertically =
|
||||||
|
horizontalOverlap > 0 &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
|
||||||
|
const opensSideways =
|
||||||
|
verticalOverlap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
|
||||||
|
const opensFromCorner =
|
||||||
|
horizontalGap > 0 &&
|
||||||
|
verticalGap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
|
||||||
|
return opensVertically || opensSideways || opensFromCorner;
|
||||||
|
}
|
||||||
|
|
||||||
export function decideHoverSurfaceRelation(
|
export function decideHoverSurfaceRelation(
|
||||||
context: HoverSurfaceContext,
|
context: HoverSurfaceContext,
|
||||||
clickElement: Element,
|
clickElement: Element,
|
||||||
|
|||||||
Reference in New Issue
Block a user