mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): hover candidate pool
This commit is contained in:
@@ -8,6 +8,44 @@ vi.stubGlobal("chrome", {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function mockRect(
|
||||||
|
el: Element,
|
||||||
|
rect: { left: number; top: number; width: number; height: number },
|
||||||
|
): void {
|
||||||
|
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
top: rect.top,
|
||||||
|
left: rect.left,
|
||||||
|
right: rect.left + rect.width,
|
||||||
|
bottom: rect.top + rect.height,
|
||||||
|
width: rect.width,
|
||||||
|
height: rect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function mockHoverStyle(pointerElements: Element[]): void {
|
||||||
|
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||||
|
const style = {
|
||||||
|
cursor: pointerElements.includes(el) ? "pointer" : "",
|
||||||
|
display: "block",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
position: "static",
|
||||||
|
visibility: "visible",
|
||||||
|
} as CSSStyleDeclaration;
|
||||||
|
return style;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function mouseOver(el: Element): void {
|
||||||
|
el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function click(el: Element): void {
|
||||||
|
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||||
|
}
|
||||||
|
|
||||||
describe("handleRecordContentMessage stop/cancel", () => {
|
describe("handleRecordContentMessage stop/cancel", () => {
|
||||||
it("ignores STOP when no recording is active", () => {
|
it("ignores STOP when no recording is active", () => {
|
||||||
const dispose = vi.fn();
|
const dispose = vi.fn();
|
||||||
@@ -381,6 +419,205 @@ describe("record-capture semantic", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps the menu opener hover when moving over an accepted menu item", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||||
|
<ul class="create-menu dropdown-menu">
|
||||||
|
<li class="dropdown-item" tabindex="0">文档C+D</li>
|
||||||
|
</ul>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const item = document.querySelector("li")!;
|
||||||
|
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||||
|
mockHoverStyle([item]);
|
||||||
|
|
||||||
|
mouseOver(trigger);
|
||||||
|
mouseOver(item);
|
||||||
|
click(item);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "新建" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "li", name: "文档C+D" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not latch a plain div action item inside a hover surface", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||||
|
<div class="create-menu dropdown-menu">
|
||||||
|
<div class="tg-menu-item" tabindex="0">文档C+D</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const item = document.querySelector(".tg-menu-item")!;
|
||||||
|
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||||
|
mockHoverStyle([item]);
|
||||||
|
|
||||||
|
mouseOver(trigger);
|
||||||
|
mouseOver(item);
|
||||||
|
click(item);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "新建" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "div", name: "文档C+D" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the opener hover when a surface action is clicked after the short latch window", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date("2026-08-07T03:43:36.000Z"));
|
||||||
|
try {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||||
|
<div class="create-menu dropdown-menu">
|
||||||
|
<div class="tg-menu-item" tabindex="0">文档C+D</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-menu-div-action-slow", (step) =>
|
||||||
|
steps.push(step),
|
||||||
|
);
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const item = document.querySelector(".tg-menu-item")!;
|
||||||
|
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||||
|
mockHoverStyle([item]);
|
||||||
|
|
||||||
|
mouseOver(trigger);
|
||||||
|
vi.setSystemTime(new Date("2026-08-07T03:43:47.000Z"));
|
||||||
|
mouseOver(item);
|
||||||
|
click(item);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "新建" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "div", name: "文档C+D" },
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("infers a non-policy opener hover from the later surface action", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button">新建</button>
|
||||||
|
<div class="create-menu dropdown-menu">
|
||||||
|
<div class="tg-menu-item" tabindex="0">文档C+D</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const item = document.querySelector(".tg-menu-item")!;
|
||||||
|
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||||
|
mockHoverStyle([trigger, item]);
|
||||||
|
|
||||||
|
mouseOver(trigger);
|
||||||
|
mouseOver(item);
|
||||||
|
click(item);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "新建" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "div", name: "文档C+D" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not record a strong-looking surface item when clicking that item itself", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||||
|
<div class="create-menu dropdown-menu">
|
||||||
|
<li tabindex="0" aria-expanded="false"><div>文档C+D</div></li>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const item = document.querySelector("li")!;
|
||||||
|
const itemInner = document.querySelector("li div")!;
|
||||||
|
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(item, { left: 820, top: 48, width: 160, height: 32 });
|
||||||
|
mockHoverStyle([item, itemInner]);
|
||||||
|
|
||||||
|
mouseOver(trigger);
|
||||||
|
mouseOver(item);
|
||||||
|
click(itemInner);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "新建" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "div", name: "文档C+D" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("records cascaded hover triggers before clicking inside the final surface", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="新建" aria-haspopup="menu">新建</button>
|
||||||
|
<ul class="create-menu dropdown-menu">
|
||||||
|
<li class="dropdown-item" tabindex="0" aria-haspopup="menu">更多模板</li>
|
||||||
|
</ul>
|
||||||
|
<div role="menu" class="template-submenu">
|
||||||
|
<button type="button">Blank doc</button>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button[aria-label]")!;
|
||||||
|
const nestedTrigger = document.querySelector("li")!;
|
||||||
|
const finalAction = document.querySelector(".template-submenu button")!;
|
||||||
|
mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 });
|
||||||
|
mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 });
|
||||||
|
mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 });
|
||||||
|
mockHoverStyle([nestedTrigger, finalAction]);
|
||||||
|
|
||||||
|
mouseOver(trigger);
|
||||||
|
mouseOver(nestedTrigger);
|
||||||
|
click(finalAction);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "新建" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "li", name: "更多模板" },
|
||||||
|
});
|
||||||
|
expect(steps[2]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { role: "button", name: "Blank doc" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("records avatar div hover with a semantic image target", () => {
|
it("records avatar div hover with a semantic image target", () => {
|
||||||
document.body.innerHTML = `
|
document.body.innerHTML = `
|
||||||
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
|
<div class="tg-avatar tg-avatar--img tg-avatar--shape-hexagon">
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import {
|
|||||||
evaluateHoverTrigger,
|
evaluateHoverTrigger,
|
||||||
type HoverTriggerDecision,
|
type HoverTriggerDecision,
|
||||||
type HoverTriggerRect,
|
type HoverTriggerRect,
|
||||||
|
hasDirectHoverInteractiveSignal,
|
||||||
|
hasStrongHoverExpansionSignal,
|
||||||
} from "@/lib/hover-trigger-policy";
|
} from "@/lib/hover-trigger-policy";
|
||||||
import {
|
import {
|
||||||
isRecordCancelMessage,
|
isRecordCancelMessage,
|
||||||
@@ -26,7 +28,7 @@ import {
|
|||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||||
import { decideHoverSurfaceRelation } from "./record-hover-surface";
|
import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
const failedStepDeliveries = new Set<string>();
|
const failedStepDeliveries = new Set<string>();
|
||||||
@@ -61,18 +63,21 @@ interface FillSession {
|
|||||||
lastValue: string;
|
lastValue: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface PendingHover {
|
interface HoverCandidate {
|
||||||
element: Element;
|
element: Element;
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
recordedAt: number;
|
recordedAt: number;
|
||||||
score: number;
|
score: number;
|
||||||
reasons: string[];
|
hasExpansionSignal: boolean;
|
||||||
|
eligible: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||||
|
|
||||||
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||||
|
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
|
||||||
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||||
|
const HOVER_CANDIDATE_LIMIT = 24;
|
||||||
|
|
||||||
function eventTarget(event: Event): EventTarget | null {
|
function eventTarget(event: Event): EventTarget | null {
|
||||||
return event.composedPath()[0] ?? event.target;
|
return event.composedPath()[0] ?? event.target;
|
||||||
@@ -213,13 +218,10 @@ function normalizeHoverTarget(
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function evaluateHoverTriggerElement(
|
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||||
el: Element,
|
|
||||||
desc: TargetDescriptor,
|
|
||||||
): HoverTriggerDecision | null {
|
|
||||||
if (!(el instanceof HTMLElement)) return null;
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
const style = hoverTriggerStyle(el);
|
const style = hoverTriggerStyle(el);
|
||||||
return evaluateHoverTrigger({
|
return {
|
||||||
tag: el.tagName.toLowerCase(),
|
tag: el.tagName.toLowerCase(),
|
||||||
role: desc.role,
|
role: desc.role,
|
||||||
label: desc.name,
|
label: desc.name,
|
||||||
@@ -228,29 +230,35 @@ function evaluateHoverTriggerElement(
|
|||||||
cursor: style.cursor,
|
cursor: style.cursor,
|
||||||
pointerEvents: style.pointerEvents,
|
pointerEvents: style.pointerEvents,
|
||||||
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
||||||
});
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
|
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const element = resolveHoverElement(target);
|
const element = resolveHoverElement(target);
|
||||||
if (!element) return null;
|
if (!element) return null;
|
||||||
const desc = describeTarget(element);
|
const desc = describeTarget(element);
|
||||||
const decision = evaluateHoverTriggerElement(element, desc);
|
const signals = hoverTriggerSignals(element, desc);
|
||||||
if (!decision?.eligible) return null;
|
if (!signals) return null;
|
||||||
|
const decision = evaluateHoverTrigger(signals);
|
||||||
|
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
|
||||||
|
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
|
||||||
|
if (!decision.eligible && !desc.name && !desc.role) return null;
|
||||||
const normalizedTarget = normalizeHoverTarget(element, desc, decision);
|
const normalizedTarget = normalizeHoverTarget(element, desc, decision);
|
||||||
return {
|
return {
|
||||||
element,
|
element,
|
||||||
target: normalizedTarget,
|
target: normalizedTarget,
|
||||||
recordedAt: Date.now(),
|
recordedAt: Date.now(),
|
||||||
score: decision.score,
|
score: decision.score,
|
||||||
reasons: decision.reasons,
|
hasExpansionSignal,
|
||||||
|
eligible: decision.eligible,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function shouldReplacePendingHover(
|
function shouldReplaceHoverCandidate(
|
||||||
current: PendingHover,
|
current: HoverCandidate,
|
||||||
next: PendingHover,
|
next: HoverCandidate,
|
||||||
now: number,
|
now: number,
|
||||||
): boolean {
|
): boolean {
|
||||||
if (next.element === current.element) return false;
|
if (next.element === current.element) return false;
|
||||||
@@ -310,7 +318,9 @@ export function startRecordCapture(
|
|||||||
let composing = false;
|
let composing = false;
|
||||||
let lastUrl = location.href;
|
let lastUrl = location.href;
|
||||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||||
let pendingHover: PendingHover | null = null;
|
let pendingHover: HoverCandidate | null = null;
|
||||||
|
const recentHoverCandidates: HoverCandidate[] = [];
|
||||||
|
const emittedHoverElements = new WeakSet<Element>();
|
||||||
let generatedControlClick: Element | null = null;
|
let generatedControlClick: Element | null = null;
|
||||||
let navigationActionPending = false;
|
let navigationActionPending = false;
|
||||||
let navigationActionVersion = 0;
|
let navigationActionVersion = 0;
|
||||||
@@ -376,6 +386,68 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const rememberHoverCandidate = (hover: HoverCandidate) => {
|
||||||
|
const duplicate = recentHoverCandidates.findIndex(
|
||||||
|
(candidate) => candidate.element === hover.element,
|
||||||
|
);
|
||||||
|
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
|
||||||
|
recentHoverCandidates.push(hover);
|
||||||
|
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
|
||||||
|
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const actionIsOnHoverTriggerItself = (
|
||||||
|
hover: HoverCandidate,
|
||||||
|
actionElement: Element | null,
|
||||||
|
relationTarget: Element,
|
||||||
|
): boolean => {
|
||||||
|
const target = actionElement ?? relationTarget;
|
||||||
|
if (target === hover.element) return true;
|
||||||
|
if (!hover.element.contains(target)) return false;
|
||||||
|
const surface = closestRecognizedHoverSurface(target);
|
||||||
|
return !surface || surface === hover.element || !hover.element.contains(surface);
|
||||||
|
};
|
||||||
|
|
||||||
|
const hoverRelatedToAction = (
|
||||||
|
hover: HoverCandidate,
|
||||||
|
relationTarget: Element,
|
||||||
|
now: number,
|
||||||
|
): boolean => {
|
||||||
|
if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false;
|
||||||
|
return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related;
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitHoverStep = (hover: HoverCandidate) => {
|
||||||
|
if (emittedHoverElements.has(hover.element)) return;
|
||||||
|
emitStep({
|
||||||
|
op: "hover",
|
||||||
|
target: hover.target,
|
||||||
|
});
|
||||||
|
emittedHoverElements.add(hover.element);
|
||||||
|
};
|
||||||
|
|
||||||
|
const findHoverForAction = (
|
||||||
|
actionElement: Element | null,
|
||||||
|
relationTarget: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate | null => {
|
||||||
|
const candidates = pendingHover
|
||||||
|
? [
|
||||||
|
...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element),
|
||||||
|
pendingHover,
|
||||||
|
]
|
||||||
|
: recentHoverCandidates;
|
||||||
|
for (let index = candidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const hover = candidates[index];
|
||||||
|
if (!hover) continue;
|
||||||
|
if (emittedHoverElements.has(hover.element)) continue;
|
||||||
|
if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue;
|
||||||
|
if (hoverRelatedToAction(hover, relationTarget, now)) return hover;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
const emitClick = (event: MouseEvent) => {
|
const emitClick = (event: MouseEvent) => {
|
||||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||||
const target = describeEventTarget(eventTarget(event));
|
const target = describeEventTarget(eventTarget(event));
|
||||||
@@ -388,51 +460,52 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
|
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
|
||||||
if (!pendingHover || !(actionTarget instanceof Element)) return;
|
if (!(actionTarget instanceof Element)) return;
|
||||||
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
|
|
||||||
pendingHover = null;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const actionElement = resolveClickableElement(actionTarget);
|
const actionElement = resolveClickableElement(actionTarget);
|
||||||
if (actionElement === pendingHover.element) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const relationTarget = actionElement ?? actionTarget;
|
const relationTarget = actionElement ?? actionTarget;
|
||||||
if (!actionElement && pendingHover.element.contains(relationTarget)) {
|
const hover = findHoverForAction(actionElement, relationTarget, Date.now());
|
||||||
return;
|
if (!hover) {
|
||||||
}
|
|
||||||
const relation = decideHoverSurfaceRelation(
|
|
||||||
{
|
|
||||||
triggerElement: pendingHover.element,
|
|
||||||
},
|
|
||||||
relationTarget,
|
|
||||||
);
|
|
||||||
if (!relation.related) {
|
|
||||||
pendingHover = null;
|
pendingHover = null;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
emitStep({
|
emitHoverStep(hover);
|
||||||
op: "hover",
|
|
||||||
target: pendingHover.target,
|
|
||||||
});
|
|
||||||
pendingHover = null;
|
pendingHover = null;
|
||||||
};
|
};
|
||||||
|
|
||||||
const onMouseOver = (event: MouseEvent) => {
|
const onMouseOver = (event: MouseEvent) => {
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
if (isOverlayTarget(target)) return;
|
if (isOverlayTarget(target)) return;
|
||||||
|
const now = Date.now();
|
||||||
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
||||||
if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||||
pendingHover = null;
|
pendingHover = null;
|
||||||
}
|
}
|
||||||
const hover = hoverTargetFromEvent(target);
|
const candidate = hoverCandidateFromEvent(target);
|
||||||
|
if (candidate) rememberHoverCandidate(candidate);
|
||||||
|
const hover = candidate?.eligible ? candidate : null;
|
||||||
|
if (pendingHover && target instanceof Element) {
|
||||||
|
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
|
||||||
|
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
|
||||||
|
if (!hover || !hover.hasExpansionSignal) return;
|
||||||
|
if (
|
||||||
|
hover.element === pendingHover.element ||
|
||||||
|
pendingHover.element.contains(hover.element)
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
emitHoverStep(pendingHover);
|
||||||
|
pendingHover = hover;
|
||||||
|
rememberHoverCandidate(hover);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!hover) return;
|
if (!hover) return;
|
||||||
const now = Date.now();
|
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
|
||||||
if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
pendingHover = hover;
|
pendingHover = hover;
|
||||||
|
rememberHoverCandidate(hover);
|
||||||
};
|
};
|
||||||
|
|
||||||
const onClick = (event: MouseEvent) => {
|
const onClick = (event: MouseEvent) => {
|
||||||
@@ -464,7 +537,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
const fillable = fillableFromTarget(target);
|
const fillable = fillableFromTarget(target);
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
emitPendingHoverBeforeAction(fillable);
|
emitHoverCandidateBeforeAction(fillable);
|
||||||
ensureFillSession(fillable);
|
ensureFillSession(fillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -472,7 +545,7 @@ export function startRecordCapture(
|
|||||||
if (target instanceof Element) {
|
if (target instanceof Element) {
|
||||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||||
if (nearbyFillable) {
|
if (nearbyFillable) {
|
||||||
emitPendingHoverBeforeAction(nearbyFillable);
|
emitHoverCandidateBeforeAction(nearbyFillable);
|
||||||
ensureFillSession(nearbyFillable);
|
ensureFillSession(nearbyFillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -492,7 +565,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
if (target instanceof Element && target.closest("select")) return;
|
if (target instanceof Element && target.closest("select")) return;
|
||||||
emitPendingHoverBeforeAction(target);
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitClick(event);
|
emitClick(event);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -534,7 +607,7 @@ export function startRecordCapture(
|
|||||||
commitFillSession();
|
commitFillSession();
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
if (target instanceof HTMLSelectElement) {
|
if (target instanceof HTMLSelectElement) {
|
||||||
emitPendingHoverBeforeAction(target);
|
emitHoverCandidateBeforeAction(target);
|
||||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||||
(opt.label || opt.textContent || opt.value).trim(),
|
(opt.label || opt.textContent || opt.value).trim(),
|
||||||
@@ -588,7 +661,7 @@ export function startRecordCapture(
|
|||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
emitPendingHoverBeforeAction(target);
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
|
|||||||
@@ -50,8 +50,8 @@ function isHoverSurfaceElement(el: Element): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
function closestHoverSurface(clickElement: Element): Element | null {
|
export function closestRecognizedHoverSurface(element: Element): Element | null {
|
||||||
let node: Element | null = clickElement;
|
let node: Element | null = element;
|
||||||
let depth = 0;
|
let depth = 0;
|
||||||
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
if (isHoverSurfaceElement(node)) return node;
|
if (isHoverSurfaceElement(node)) return node;
|
||||||
@@ -115,7 +115,7 @@ export function decideHoverSurfaceRelation(
|
|||||||
context: HoverSurfaceContext,
|
context: HoverSurfaceContext,
|
||||||
clickElement: Element,
|
clickElement: Element,
|
||||||
): HoverSurfaceDecision {
|
): HoverSurfaceDecision {
|
||||||
const clickSurface = closestHoverSurface(clickElement);
|
const clickSurface = closestRecognizedHoverSurface(clickElement);
|
||||||
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
||||||
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,6 +77,19 @@ describe("evaluateHoverTrigger", () => {
|
|||||||
expect(decision.eligible).toBe(false);
|
expect(decision.eligible).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects labelled dropdown list items as hover triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "li",
|
||||||
|
label: "文档C+D",
|
||||||
|
attrs: { class: "dropdown-item", tabindex: "0" },
|
||||||
|
rect: { x: 820, y: 48, w: 160, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects ordinary controls without hover popup signals", () => {
|
it("rejects ordinary controls without hover popup signals", () => {
|
||||||
const decision = evaluateHoverTrigger({
|
const decision = evaluateHoverTrigger({
|
||||||
tag: "button",
|
tag: "button",
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ export interface HoverTriggerDecision {
|
|||||||
const HOVER_TRIGGER_MIN_SCORE = 45;
|
const HOVER_TRIGGER_MIN_SCORE = 45;
|
||||||
const HOVER_POPUP_SIGNAL_RE =
|
const HOVER_POPUP_SIGNAL_RE =
|
||||||
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
||||||
|
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
|
||||||
|
|
||||||
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
||||||
return signals.attrs?.[name.toLowerCase()];
|
return signals.attrs?.[name.toLowerCase()];
|
||||||
@@ -59,6 +60,14 @@ export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
|
|||||||
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
return (
|
||||||
|
signals.cssHoverMatch === true ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
||||||
const tag = signals.tag.toLowerCase();
|
const tag = signals.tag.toLowerCase();
|
||||||
const role = (signals.role ?? "").toLowerCase();
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
@@ -104,7 +113,12 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger
|
|||||||
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
||||||
const popupSignal = hasHoverPopupSignal(signals);
|
const popupSignal = hasHoverPopupSignal(signals);
|
||||||
const isTextNavigationItem =
|
const isTextNavigationItem =
|
||||||
(tag === "a" || role === "link" || role.startsWith("menuitem")) &&
|
(tag === "a" ||
|
||||||
|
tag === "li" ||
|
||||||
|
role === "link" ||
|
||||||
|
role === "option" ||
|
||||||
|
role.startsWith("menuitem") ||
|
||||||
|
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
|
||||||
!graphic &&
|
!graphic &&
|
||||||
!!label &&
|
!!label &&
|
||||||
label.toLowerCase() !== "image";
|
label.toLowerCase() !== "image";
|
||||||
@@ -118,9 +132,7 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger
|
|||||||
const surfaceTriggerEvidence =
|
const surfaceTriggerEvidence =
|
||||||
(popupSignal && !isTextNavigationItem) ||
|
(popupSignal && !isTextNavigationItem) ||
|
||||||
compactTopbarIcon ||
|
compactTopbarIcon ||
|
||||||
signals.cssHoverMatch === true ||
|
hasStrongHoverExpansionSignal(signals);
|
||||||
attr(signals, "aria-haspopup") !== undefined ||
|
|
||||||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false";
|
|
||||||
if (!directInteractive || !surfaceTriggerEvidence) {
|
if (!directInteractive || !surfaceTriggerEvidence) {
|
||||||
return { eligible: false, score: 0, reasons: [] };
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user