mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): recording hover with custom div
This commit is contained in:
@@ -470,6 +470,150 @@ describe("record-capture semantic", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("records hover before clicking an unlabelled positioned floating surface", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||
<div style="position: absolute; top: 48px; left: 820px; width: 160px; height: 80px;">
|
||||
<a href="/u/me">My profile</a>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-plain-floating-surface", (step) =>
|
||||
steps.push(step),
|
||||
);
|
||||
const trigger = document.querySelector("button")!;
|
||||
const surface = document.querySelector("div")!;
|
||||
const link = document.querySelector("a")!;
|
||||
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
|
||||
x: 900,
|
||||
y: 8,
|
||||
top: 8,
|
||||
left: 900,
|
||||
right: 932,
|
||||
bottom: 40,
|
||||
width: 32,
|
||||
height: 32,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({
|
||||
x: 820,
|
||||
y: 48,
|
||||
top: 48,
|
||||
left: 820,
|
||||
right: 980,
|
||||
bottom: 128,
|
||||
width: 160,
|
||||
height: 80,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||
const style = {
|
||||
cursor: "",
|
||||
display: "block",
|
||||
pointerEvents: "auto",
|
||||
position: el === surface ? "absolute" : "static",
|
||||
visibility: "visible",
|
||||
} as CSSStyleDeclaration;
|
||||
return style;
|
||||
});
|
||||
|
||||
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||
link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "Open user navigation menu" },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat ordinary document-flow divs as hover surfaces", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||
<div>
|
||||
<a href="/u/me">My profile</a>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-plain-flow-surface", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button")!;
|
||||
const link = document.querySelector("a")!;
|
||||
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
|
||||
x: 900,
|
||||
y: 8,
|
||||
top: 8,
|
||||
left: 900,
|
||||
right: 932,
|
||||
bottom: 40,
|
||||
width: 32,
|
||||
height: 32,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
vi.spyOn(window, "getComputedStyle").mockReturnValue({
|
||||
cursor: "",
|
||||
display: "block",
|
||||
pointerEvents: "auto",
|
||||
position: "static",
|
||||
visibility: "visible",
|
||||
} as CSSStyleDeclaration);
|
||||
|
||||
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||
link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||
});
|
||||
|
||||
it("does not treat unlabelled sticky containers as hover surfaces", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||
<div style="position: sticky; top: 48px; left: 820px; width: 160px; height: 80px;">
|
||||
<a href="/u/me">My profile</a>
|
||||
</div>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-hover-sticky-surface", (step) => steps.push(step));
|
||||
const trigger = document.querySelector("button")!;
|
||||
const surface = document.querySelector("div")!;
|
||||
const link = document.querySelector("a")!;
|
||||
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
|
||||
x: 900,
|
||||
y: 8,
|
||||
top: 8,
|
||||
left: 900,
|
||||
right: 932,
|
||||
bottom: 40,
|
||||
width: 32,
|
||||
height: 32,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({
|
||||
x: 820,
|
||||
y: 48,
|
||||
top: 48,
|
||||
left: 820,
|
||||
right: 980,
|
||||
bottom: 128,
|
||||
width: 160,
|
||||
height: 80,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||
const style = {
|
||||
cursor: "",
|
||||
display: "block",
|
||||
pointerEvents: "auto",
|
||||
position: el === surface ? "sticky" : "static",
|
||||
visibility: "visible",
|
||||
} as CSSStyleDeclaration;
|
||||
return style;
|
||||
});
|
||||
|
||||
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||
link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||
});
|
||||
|
||||
it("does not record hover before clicking the same trigger", () => {
|
||||
document.body.innerHTML = `
|
||||
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||
|
||||
@@ -73,7 +73,6 @@ type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||
|
||||
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||
const RECORD_DEBUG_STORAGE_KEY = "bsk_record_debug";
|
||||
|
||||
function eventTarget(event: Event): EventTarget | null {
|
||||
return event.composedPath()[0] ?? event.target;
|
||||
@@ -146,26 +145,6 @@ function fillableValue(el: FillableElement): string {
|
||||
return el.textContent ?? "";
|
||||
}
|
||||
|
||||
function recordDebug(message: string, data?: unknown): void {
|
||||
try {
|
||||
if (localStorage.getItem(RECORD_DEBUG_STORAGE_KEY) !== "1") return;
|
||||
console.info("[bsk record]", message, data === undefined ? "" : JSON.stringify(data));
|
||||
} catch {
|
||||
// Recording must not depend on page storage or console availability.
|
||||
}
|
||||
}
|
||||
|
||||
function elementDebugSummary(el: Element): Record<string, string | undefined> {
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
id: el.id || undefined,
|
||||
class: typeof el.className === "string" ? el.className || undefined : undefined,
|
||||
role: el.getAttribute("role") ?? undefined,
|
||||
ariaLabel: el.getAttribute("aria-label") ?? undefined,
|
||||
text: el.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
|
||||
return {
|
||||
id: el.id || undefined,
|
||||
@@ -253,34 +232,13 @@ function evaluateHoverTriggerElement(
|
||||
}
|
||||
|
||||
function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
|
||||
if (!(target instanceof Element)) {
|
||||
recordDebug("hover ignored: event target is not an Element", { target });
|
||||
return null;
|
||||
}
|
||||
if (!(target instanceof Element)) return null;
|
||||
const element = resolveHoverElement(target);
|
||||
if (!element) {
|
||||
recordDebug("hover ignored: no resolved hover element", { target });
|
||||
return null;
|
||||
}
|
||||
if (!element) return null;
|
||||
const desc = describeTarget(element);
|
||||
const decision = evaluateHoverTriggerElement(element, desc);
|
||||
if (!decision?.eligible) {
|
||||
recordDebug("hover ignored: rejected by trigger policy", {
|
||||
desc,
|
||||
score: decision?.score ?? 0,
|
||||
reasons: decision?.reasons ?? [],
|
||||
element: elementDebugSummary(element),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
if (!decision?.eligible) return null;
|
||||
const normalizedTarget = normalizeHoverTarget(element, desc, decision);
|
||||
recordDebug("hover accepted", {
|
||||
desc: normalizedTarget,
|
||||
raw_desc: desc,
|
||||
score: decision.score,
|
||||
reasons: decision.reasons,
|
||||
element: elementDebugSummary(element),
|
||||
});
|
||||
return {
|
||||
element,
|
||||
target: normalizedTarget,
|
||||
@@ -345,7 +303,6 @@ export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
): RecordCaptureController {
|
||||
recordDebug("record capture started", { page_url: location.href });
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
};
|
||||
@@ -439,34 +396,22 @@ export function startRecordCapture(
|
||||
}
|
||||
const clickElement = resolveClickableElement(clickTarget);
|
||||
if (clickElement === pendingHover.element) {
|
||||
recordDebug("pending hover skipped: click resolved to same trigger", {
|
||||
target: pendingHover.target,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const relationTarget = clickElement ?? clickTarget;
|
||||
if (!clickElement && pendingHover.element.contains(relationTarget)) {
|
||||
recordDebug("pending hover skipped: anonymous click inside trigger", {
|
||||
target: pendingHover.target,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const relation = decideHoverSurfaceRelation(
|
||||
{
|
||||
triggerElement: pendingHover.element,
|
||||
triggerTarget: pendingHover.target,
|
||||
},
|
||||
relationTarget,
|
||||
);
|
||||
if (!relation.related) {
|
||||
recordDebug("pending hover skipped: click outside hover surface", {
|
||||
target: pendingHover.target,
|
||||
reason: relation.reason,
|
||||
});
|
||||
pendingHover = null;
|
||||
return;
|
||||
}
|
||||
recordDebug("pending hover emitted before click", { target: pendingHover.target });
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: pendingHover.target,
|
||||
@@ -481,34 +426,8 @@ export function startRecordCapture(
|
||||
if (!hover) return;
|
||||
const now = Date.now();
|
||||
if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) {
|
||||
recordDebug("pending hover kept over later hover", {
|
||||
current: {
|
||||
target: pendingHover.target,
|
||||
score: pendingHover.score,
|
||||
reasons: pendingHover.reasons,
|
||||
},
|
||||
next: {
|
||||
target: hover.target,
|
||||
score: hover.score,
|
||||
reasons: hover.reasons,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (pendingHover) {
|
||||
recordDebug("pending hover replaced", {
|
||||
current: {
|
||||
target: pendingHover.target,
|
||||
score: pendingHover.score,
|
||||
reasons: pendingHover.reasons,
|
||||
},
|
||||
next: {
|
||||
target: hover.target,
|
||||
score: hover.score,
|
||||
reasons: hover.reasons,
|
||||
},
|
||||
});
|
||||
}
|
||||
pendingHover = hover;
|
||||
};
|
||||
|
||||
@@ -701,7 +620,6 @@ export function startRecordCapture(
|
||||
|
||||
return {
|
||||
dispose() {
|
||||
recordDebug("record capture disposed", { page_url: location.href });
|
||||
commitFillSession();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("mouseover", onMouseOver, true);
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { TargetDescriptor } from "@/lib/describe-target";
|
||||
|
||||
export interface HoverSurfaceDecision {
|
||||
related: boolean;
|
||||
reason?: string;
|
||||
@@ -8,7 +6,6 @@ export interface HoverSurfaceDecision {
|
||||
|
||||
export interface HoverSurfaceContext {
|
||||
triggerElement: Element;
|
||||
triggerTarget: TargetDescriptor;
|
||||
}
|
||||
|
||||
const HOVER_SURFACE_SELECTOR = [
|
||||
@@ -26,6 +23,8 @@ const HOVER_SURFACE_SELECTOR = [
|
||||
|
||||
const HOVER_SURFACE_WORD_RE =
|
||||
/\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i;
|
||||
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
||||
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
||||
|
||||
function elementTokenText(el: Element): string {
|
||||
return [
|
||||
@@ -62,6 +61,33 @@ function closestHoverSurface(clickElement: Element): Element | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isPositionedFloatingElement(el: Element): boolean {
|
||||
if (!(el instanceof HTMLElement)) return false;
|
||||
const style = getComputedStyle(el);
|
||||
if (!["absolute", "fixed"].includes(style.position)) return false;
|
||||
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||
return false;
|
||||
}
|
||||
const rect = rectFor(el);
|
||||
if (!rect) return false;
|
||||
const viewportArea = window.innerWidth * window.innerHeight;
|
||||
const area = rect.width * rect.height;
|
||||
return (
|
||||
area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO)
|
||||
);
|
||||
}
|
||||
|
||||
function closestFloatingSurface(clickElement: Element): Element | null {
|
||||
let node: Element | null = clickElement;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||
if (isPositionedFloatingElement(node)) return node;
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function rectFor(el: Element): DOMRect | null {
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width <= 0 || rect.height <= 0) return null;
|
||||
@@ -82,7 +108,7 @@ function isNearTrigger(surface: Element, trigger: Element): boolean {
|
||||
triggerRect.top - surfaceRect.bottom,
|
||||
surfaceRect.top - triggerRect.bottom,
|
||||
);
|
||||
return horizontalGap <= 320 && verticalGap <= 320;
|
||||
return horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP;
|
||||
}
|
||||
|
||||
export function decideHoverSurfaceRelation(
|
||||
@@ -94,6 +120,15 @@ export function decideHoverSurfaceRelation(
|
||||
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||
}
|
||||
|
||||
const floatingSurface = closestFloatingSurface(clickElement);
|
||||
if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) {
|
||||
return {
|
||||
related: true,
|
||||
reason: "click-inside-near-floating-surface",
|
||||
surface: floatingSurface,
|
||||
};
|
||||
}
|
||||
|
||||
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
|
||||
return { related: true, reason: "click-inside-trigger-contained-surface" };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user