mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): hover candidate pool
This commit is contained in:
@@ -8,6 +8,44 @@ vi.stubGlobal("chrome", {
|
||||
},
|
||||
});
|
||||
|
||||
function mockRect(
|
||||
el: Element,
|
||||
rect: { left: number; top: number; width: number; height: number },
|
||||
): void {
|
||||
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||
x: rect.left,
|
||||
y: rect.top,
|
||||
top: rect.top,
|
||||
left: rect.left,
|
||||
right: rect.left + rect.width,
|
||||
bottom: rect.top + rect.height,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
}
|
||||
|
||||
function mockHoverStyle(pointerElements: Element[]): void {
|
||||
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||
const style = {
|
||||
cursor: pointerElements.includes(el) ? "pointer" : "",
|
||||
display: "block",
|
||||
pointerEvents: "auto",
|
||||
position: "static",
|
||||
visibility: "visible",
|
||||
} as CSSStyleDeclaration;
|
||||
return style;
|
||||
});
|
||||
}
|
||||
|
||||
function mouseOver(el: Element): void {
|
||||
el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||
}
|
||||
|
||||
function click(el: Element): void {
|
||||
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
}
|
||||
|
||||
describe("handleRecordContentMessage stop/cancel", () => {
|
||||
it("ignores STOP when no recording is active", () => {
|
||||
const dispose = vi.fn();
|
||||
@@ -381,6 +419,205 @@ describe("record-capture semantic", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the menu opener hover when moving over an accepted menu item", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||
<ul class="create-menu dropdown-menu">
|
||||
<li class="dropdown-item" tabindex="0">文档C+D</li>
|
||||
</ul>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button")!;
|
||||
const item = document.querySelector("li")!;
|
||||
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||
mockHoverStyle([item]);
|
||||
|
||||
mouseOver(trigger);
|
||||
mouseOver(item);
|
||||
click(item);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "新建" },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
target: { tag: "li", name: "文档C+D" },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not latch a plain div action item inside a hover surface", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||
<div class="create-menu dropdown-menu">
|
||||
<div class="tg-menu-item" tabindex="0">文档C+D</div>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button")!;
|
||||
const item = document.querySelector(".tg-menu-item")!;
|
||||
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||
mockHoverStyle([item]);
|
||||
|
||||
mouseOver(trigger);
|
||||
mouseOver(item);
|
||||
click(item);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "新建" },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
target: { tag: "div", name: "文档C+D" },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the opener hover when a surface action is clicked after the short latch window", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-07T03:43:36.000Z"));
|
||||
try {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||
<div class="create-menu dropdown-menu">
|
||||
<div class="tg-menu-item" tabindex="0">文档C+D</div>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-menu-div-action-slow", (step) =>
|
||||
steps.push(step),
|
||||
);
|
||||
const trigger = document.querySelector("button")!;
|
||||
const item = document.querySelector(".tg-menu-item")!;
|
||||
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||
mockHoverStyle([item]);
|
||||
|
||||
mouseOver(trigger);
|
||||
vi.setSystemTime(new Date("2026-08-07T03:43:47.000Z"));
|
||||
mouseOver(item);
|
||||
click(item);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "新建" },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
target: { tag: "div", name: "文档C+D" },
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("infers a non-policy opener hover from the later surface action", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button">新建</button>
|
||||
<div class="create-menu dropdown-menu">
|
||||
<div class="tg-menu-item" tabindex="0">文档C+D</div>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button")!;
|
||||
const item = document.querySelector(".tg-menu-item")!;
|
||||
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||
mockHoverStyle([trigger, item]);
|
||||
|
||||
mouseOver(trigger);
|
||||
mouseOver(item);
|
||||
click(item);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "新建" },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
target: { tag: "div", name: "文档C+D" },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not record a strong-looking surface item when clicking that item itself", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||
<div class="create-menu dropdown-menu">
|
||||
<li tabindex="0" aria-expanded="false"><div>文档C+D</div></li>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button")!;
|
||||
const item = document.querySelector("li")!;
|
||||
const itemInner = document.querySelector("li div")!;
|
||||
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||
mockHoverStyle([item, itemInner]);
|
||||
|
||||
mouseOver(trigger);
|
||||
mouseOver(item);
|
||||
click(itemInner);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "新建" },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
target: { tag: "div", name: "文档C+D" },
|
||||
});
|
||||
});
|
||||
|
||||
it("records cascaded hover triggers before clicking inside the final surface", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||
<ul class="create-menu dropdown-menu">
|
||||
<li class="dropdown-item" tabindex="0" aria-haspopup="menu">更多模板</li>
|
||||
</ul>
|
||||
<div role="menu" class="template-submenu">
|
||||
<button type="button">Blank doc</button>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button[aria-label]")!;
|
||||
const nestedTrigger = document.querySelector("li")!;
|
||||
const finalAction = document.querySelector(".template-submenu button")!;
|
||||
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||
mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 });
|
||||
mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 });
|
||||
mockHoverStyle([nestedTrigger, finalAction]);
|
||||
|
||||
mouseOver(trigger);
|
||||
mouseOver(nestedTrigger);
|
||||
click(finalAction);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "新建" },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { tag: "li", name: "更多模板" },
|
||||
});
|
||||
expect(steps[2]).toMatchObject({
|
||||
op: "click",
|
||||
target: { role: "button", name: "Blank doc" },
|
||||
});
|
||||
});
|
||||
|
||||
it("records avatar div hover with a semantic image target", () => {
|
||||
document.body.innerHTML = `
|
||||
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
|
||||
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
evaluateHoverTrigger,
|
||||
type HoverTriggerDecision,
|
||||
type HoverTriggerRect,
|
||||
hasDirectHoverInteractiveSignal,
|
||||
hasStrongHoverExpansionSignal,
|
||||
} from "@/lib/hover-trigger-policy";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
@@ -26,7 +28,7 @@ import {
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
import { decideHoverSurfaceRelation } from "./record-hover-surface";
|
||||
import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface";
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
@@ -61,18 +63,21 @@ interface FillSession {
|
||||
lastValue: string;
|
||||
}
|
||||
|
||||
interface PendingHover {
|
||||
interface HoverCandidate {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
recordedAt: number;
|
||||
score: number;
|
||||
reasons: string[];
|
||||
hasExpansionSignal: boolean;
|
||||
eligible: boolean;
|
||||
}
|
||||
|
||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||
|
||||
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
|
||||
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||
const HOVER_CANDIDATE_LIMIT = 24;
|
||||
|
||||
function eventTarget(event: Event): EventTarget | null {
|
||||
return event.composedPath()[0] ?? event.target;
|
||||
@@ -213,13 +218,10 @@ function normalizeHoverTarget(
|
||||
return desc;
|
||||
}
|
||||
|
||||
function evaluateHoverTriggerElement(
|
||||
el: Element,
|
||||
desc: TargetDescriptor,
|
||||
): HoverTriggerDecision | null {
|
||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||
if (!(el instanceof HTMLElement)) return null;
|
||||
const style = hoverTriggerStyle(el);
|
||||
return evaluateHoverTrigger({
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
role: desc.role,
|
||||
label: desc.name,
|
||||
@@ -228,29 +230,35 @@ function evaluateHoverTriggerElement(
|
||||
cursor: style.cursor,
|
||||
pointerEvents: style.pointerEvents,
|
||||
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
|
||||
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const element = resolveHoverElement(target);
|
||||
if (!element) return null;
|
||||
const desc = describeTarget(element);
|
||||
const decision = evaluateHoverTriggerElement(element, desc);
|
||||
if (!decision?.eligible) return null;
|
||||
const signals = hoverTriggerSignals(element, desc);
|
||||
if (!signals) return null;
|
||||
const decision = evaluateHoverTrigger(signals);
|
||||
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
|
||||
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
|
||||
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
|
||||
if (!decision.eligible && !desc.name && !desc.role) return null;
|
||||
const normalizedTarget = normalizeHoverTarget(element, desc, decision);
|
||||
return {
|
||||
element,
|
||||
target: normalizedTarget,
|
||||
recordedAt: Date.now(),
|
||||
score: decision.score,
|
||||
reasons: decision.reasons,
|
||||
hasExpansionSignal,
|
||||
eligible: decision.eligible,
|
||||
};
|
||||
}
|
||||
|
||||
function shouldReplacePendingHover(
|
||||
current: PendingHover,
|
||||
next: PendingHover,
|
||||
function shouldReplaceHoverCandidate(
|
||||
current: HoverCandidate,
|
||||
next: HoverCandidate,
|
||||
now: number,
|
||||
): boolean {
|
||||
if (next.element === current.element) return false;
|
||||
@@ -310,7 +318,9 @@ export function startRecordCapture(
|
||||
let composing = false;
|
||||
let lastUrl = location.href;
|
||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||
let pendingHover: PendingHover | null = null;
|
||||
let pendingHover: HoverCandidate | null = null;
|
||||
const recentHoverCandidates: HoverCandidate[] = [];
|
||||
const emittedHoverElements = new WeakSet<Element>();
|
||||
let generatedControlClick: Element | null = null;
|
||||
let navigationActionPending = false;
|
||||
let navigationActionVersion = 0;
|
||||
@@ -376,6 +386,68 @@ export function startRecordCapture(
|
||||
});
|
||||
};
|
||||
|
||||
const rememberHoverCandidate = (hover: HoverCandidate) => {
|
||||
const duplicate = recentHoverCandidates.findIndex(
|
||||
(candidate) => candidate.element === hover.element,
|
||||
);
|
||||
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
|
||||
recentHoverCandidates.push(hover);
|
||||
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
|
||||
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
|
||||
}
|
||||
};
|
||||
|
||||
const actionIsOnHoverTriggerItself = (
|
||||
hover: HoverCandidate,
|
||||
actionElement: Element | null,
|
||||
relationTarget: Element,
|
||||
): boolean => {
|
||||
const target = actionElement ?? relationTarget;
|
||||
if (target === hover.element) return true;
|
||||
if (!hover.element.contains(target)) return false;
|
||||
const surface = closestRecognizedHoverSurface(target);
|
||||
return !surface || surface === hover.element || !hover.element.contains(surface);
|
||||
};
|
||||
|
||||
const hoverRelatedToAction = (
|
||||
hover: HoverCandidate,
|
||||
relationTarget: Element,
|
||||
now: number,
|
||||
): boolean => {
|
||||
if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false;
|
||||
return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related;
|
||||
};
|
||||
|
||||
const emitHoverStep = (hover: HoverCandidate) => {
|
||||
if (emittedHoverElements.has(hover.element)) return;
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: hover.target,
|
||||
});
|
||||
emittedHoverElements.add(hover.element);
|
||||
};
|
||||
|
||||
const findHoverForAction = (
|
||||
actionElement: Element | null,
|
||||
relationTarget: Element,
|
||||
now: number,
|
||||
): HoverCandidate | null => {
|
||||
const candidates = pendingHover
|
||||
? [
|
||||
...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element),
|
||||
pendingHover,
|
||||
]
|
||||
: recentHoverCandidates;
|
||||
for (let index = candidates.length - 1; index >= 0; index -= 1) {
|
||||
const hover = candidates[index];
|
||||
if (!hover) continue;
|
||||
if (emittedHoverElements.has(hover.element)) continue;
|
||||
if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue;
|
||||
if (hoverRelatedToAction(hover, relationTarget, now)) return hover;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const emitClick = (event: MouseEvent) => {
|
||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||
const target = describeEventTarget(eventTarget(event));
|
||||
@@ -388,51 +460,52 @@ export function startRecordCapture(
|
||||
});
|
||||
};
|
||||
|
||||
const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
|
||||
if (!pendingHover || !(actionTarget instanceof Element)) return;
|
||||
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
|
||||
pendingHover = null;
|
||||
return;
|
||||
}
|
||||
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
|
||||
if (!(actionTarget instanceof Element)) return;
|
||||
const actionElement = resolveClickableElement(actionTarget);
|
||||
if (actionElement === pendingHover.element) {
|
||||
return;
|
||||
}
|
||||
const relationTarget = actionElement ?? actionTarget;
|
||||
if (!actionElement && pendingHover.element.contains(relationTarget)) {
|
||||
return;
|
||||
}
|
||||
const relation = decideHoverSurfaceRelation(
|
||||
{
|
||||
triggerElement: pendingHover.element,
|
||||
},
|
||||
relationTarget,
|
||||
);
|
||||
if (!relation.related) {
|
||||
const hover = findHoverForAction(actionElement, relationTarget, Date.now());
|
||||
if (!hover) {
|
||||
pendingHover = null;
|
||||
return;
|
||||
}
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: pendingHover.target,
|
||||
});
|
||||
emitHoverStep(hover);
|
||||
pendingHover = null;
|
||||
};
|
||||
|
||||
const onMouseOver = (event: MouseEvent) => {
|
||||
const target = eventTarget(event);
|
||||
if (isOverlayTarget(target)) return;
|
||||
const now = Date.now();
|
||||
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
||||
if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||
pendingHover = null;
|
||||
}
|
||||
const hover = hoverTargetFromEvent(target);
|
||||
const candidate = hoverCandidateFromEvent(target);
|
||||
if (candidate) rememberHoverCandidate(candidate);
|
||||
const hover = candidate?.eligible ? candidate : null;
|
||||
if (pendingHover && target instanceof Element) {
|
||||
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
|
||||
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
|
||||
if (!hover || !hover.hasExpansionSignal) return;
|
||||
if (
|
||||
hover.element === pendingHover.element ||
|
||||
pendingHover.element.contains(hover.element)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
emitHoverStep(pendingHover);
|
||||
pendingHover = hover;
|
||||
rememberHoverCandidate(hover);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!hover) return;
|
||||
const now = Date.now();
|
||||
if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) {
|
||||
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
|
||||
return;
|
||||
}
|
||||
pendingHover = hover;
|
||||
rememberHoverCandidate(hover);
|
||||
};
|
||||
|
||||
const onClick = (event: MouseEvent) => {
|
||||
@@ -464,7 +537,7 @@ export function startRecordCapture(
|
||||
|
||||
const fillable = fillableFromTarget(target);
|
||||
if (fillable) {
|
||||
emitPendingHoverBeforeAction(fillable);
|
||||
emitHoverCandidateBeforeAction(fillable);
|
||||
ensureFillSession(fillable);
|
||||
return;
|
||||
}
|
||||
@@ -472,7 +545,7 @@ export function startRecordCapture(
|
||||
if (target instanceof Element) {
|
||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||
if (nearbyFillable) {
|
||||
emitPendingHoverBeforeAction(nearbyFillable);
|
||||
emitHoverCandidateBeforeAction(nearbyFillable);
|
||||
ensureFillSession(nearbyFillable);
|
||||
return;
|
||||
}
|
||||
@@ -492,7 +565,7 @@ export function startRecordCapture(
|
||||
|
||||
commitFillSession();
|
||||
if (target instanceof Element && target.closest("select")) return;
|
||||
emitPendingHoverBeforeAction(target);
|
||||
emitHoverCandidateBeforeAction(target);
|
||||
emitClick(event);
|
||||
};
|
||||
|
||||
@@ -534,7 +607,7 @@ export function startRecordCapture(
|
||||
commitFillSession();
|
||||
const target = eventTarget(event);
|
||||
if (target instanceof HTMLSelectElement) {
|
||||
emitPendingHoverBeforeAction(target);
|
||||
emitHoverCandidateBeforeAction(target);
|
||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||
(opt.label || opt.textContent || opt.value).trim(),
|
||||
@@ -588,7 +661,7 @@ export function startRecordCapture(
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
emitPendingHoverBeforeAction(target);
|
||||
emitHoverCandidateBeforeAction(target);
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
|
||||
@@ -50,8 +50,8 @@ function isHoverSurfaceElement(el: Element): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
function closestHoverSurface(clickElement: Element): Element | null {
|
||||
let node: Element | null = clickElement;
|
||||
export function closestRecognizedHoverSurface(element: Element): Element | null {
|
||||
let node: Element | null = element;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||
if (isHoverSurfaceElement(node)) return node;
|
||||
@@ -115,7 +115,7 @@ export function decideHoverSurfaceRelation(
|
||||
context: HoverSurfaceContext,
|
||||
clickElement: Element,
|
||||
): HoverSurfaceDecision {
|
||||
const clickSurface = closestHoverSurface(clickElement);
|
||||
const clickSurface = closestRecognizedHoverSurface(clickElement);
|
||||
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
||||
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||
}
|
||||
|
||||
@@ -77,6 +77,19 @@ describe("evaluateHoverTrigger", () => {
|
||||
expect(decision.eligible).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects labelled dropdown list items as hover triggers", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "li",
|
||||
label: "文档C+D",
|
||||
attrs: { class: "dropdown-item", tabindex: "0" },
|
||||
rect: { x: 820, y: 48, w: 160, h: 32 },
|
||||
cursor: "pointer",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects ordinary controls without hover popup signals", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "button",
|
||||
|
||||
@@ -26,6 +26,7 @@ export interface HoverTriggerDecision {
|
||||
const HOVER_TRIGGER_MIN_SCORE = 45;
|
||||
const HOVER_POPUP_SIGNAL_RE =
|
||||
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
||||
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
|
||||
|
||||
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
||||
return signals.attrs?.[name.toLowerCase()];
|
||||
@@ -59,6 +60,14 @@ export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
|
||||
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
||||
}
|
||||
|
||||
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
|
||||
return (
|
||||
signals.cssHoverMatch === true ||
|
||||
attr(signals, "aria-haspopup") !== undefined ||
|
||||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
|
||||
);
|
||||
}
|
||||
|
||||
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
||||
const tag = signals.tag.toLowerCase();
|
||||
const role = (signals.role ?? "").toLowerCase();
|
||||
@@ -104,7 +113,12 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger
|
||||
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
||||
const popupSignal = hasHoverPopupSignal(signals);
|
||||
const isTextNavigationItem =
|
||||
(tag === "a" || role === "link" || role.startsWith("menuitem")) &&
|
||||
(tag === "a" ||
|
||||
tag === "li" ||
|
||||
role === "link" ||
|
||||
role === "option" ||
|
||||
role.startsWith("menuitem") ||
|
||||
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
|
||||
!graphic &&
|
||||
!!label &&
|
||||
label.toLowerCase() !== "image";
|
||||
@@ -118,9 +132,7 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger
|
||||
const surfaceTriggerEvidence =
|
||||
(popupSignal && !isTextNavigationItem) ||
|
||||
compactTopbarIcon ||
|
||||
signals.cssHoverMatch === true ||
|
||||
attr(signals, "aria-haspopup") !== undefined ||
|
||||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false";
|
||||
hasStrongHoverExpansionSignal(signals);
|
||||
if (!directInteractive || !surfaceTriggerEvidence) {
|
||||
return { eligible: false, score: 0, reasons: [] };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user