mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
566b6c17fd | ||
|
|
2ccc0aafb9 | ||
|
|
eeab97bcdd | ||
|
|
6c610faed9 | ||
|
|
d685c75d25 | ||
|
|
73724617f9 | ||
|
|
4ba6fb45e1 | ||
|
|
9d4ac90592 | ||
|
|
cde6f22ecf | ||
|
|
d24bb6aa93 | ||
|
|
dbc4845c0d | ||
|
|
91225e5fdb |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
|
||||
|
||||
function mockRect(
|
||||
el: Element,
|
||||
rect: { left: number; top: number; width: number; height: number },
|
||||
): void {
|
||||
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||
x: rect.left,
|
||||
y: rect.top,
|
||||
top: rect.top,
|
||||
left: rect.left,
|
||||
right: rect.left + rect.width,
|
||||
bottom: rect.top + rect.height,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
toJSON: () => ({}),
|
||||
});
|
||||
}
|
||||
|
||||
describe("record-hover-surface", () => {
|
||||
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
|
||||
document.body.innerHTML = `
|
||||
<ul class="header-menu tw-flex navbar-app__authentication-menu">
|
||||
<li class="tg-popover">
|
||||
<div class="tg-avatar tg-avatar--img"></div>
|
||||
</li>
|
||||
</ul>
|
||||
<div class="tg-popover__popper">
|
||||
<div class="tg-popover__content">
|
||||
<div class="tg-popover__content-body">
|
||||
<li class="header-menu__item">
|
||||
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
|
||||
</li>
|
||||
<li class="header-menu__item">
|
||||
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
|
||||
</li>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
|
||||
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
|
||||
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
|
||||
).toBe(false);
|
||||
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat elements covered by an open surface as the surface owner", () => {
|
||||
document.body.innerHTML = `
|
||||
<button class="avatar">image</button>
|
||||
<a class="fork" href="/fork">Fork</a>
|
||||
<button class="clone">Clone & Download</button>
|
||||
<div class="tg-popover__content-body"></div>
|
||||
`;
|
||||
const avatar = document.querySelector(".avatar")!;
|
||||
const fork = document.querySelector(".fork")!;
|
||||
const clone = document.querySelector(".clone")!;
|
||||
const surface = document.querySelector(".tg-popover__content-body")!;
|
||||
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
|
||||
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
|
||||
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
|
||||
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
|
||||
|
||||
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
|
||||
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
|
||||
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not treat fixed navigation containers as plain floating surfaces", () => {
|
||||
document.body.innerHTML = `
|
||||
<header class="navbar-app">
|
||||
<a href="/fork">Fork</a>
|
||||
<button>image</button>
|
||||
</header>
|
||||
<div class="plain-floating"></div>
|
||||
`;
|
||||
const navbar = document.querySelector(".navbar-app")!;
|
||||
const floating = document.querySelector(".plain-floating")!;
|
||||
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
|
||||
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
|
||||
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||
const style = {
|
||||
cursor: "",
|
||||
display: "block",
|
||||
pointerEvents: "auto",
|
||||
position: el === navbar ? "fixed" : "absolute",
|
||||
visibility: "visible",
|
||||
} as CSSStyleDeclaration;
|
||||
return style;
|
||||
});
|
||||
|
||||
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
|
||||
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,17 @@
|
||||
import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
|
||||
import {
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
resolveClickableElement,
|
||||
resolveHoverElement,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
evaluateHoverTrigger,
|
||||
type HoverTriggerDecision,
|
||||
type HoverTriggerRect,
|
||||
hasDirectHoverInteractiveSignal,
|
||||
hasStrongHoverExpansionSignal,
|
||||
} from "@/lib/hover-trigger-policy";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
@@ -15,6 +28,14 @@ import {
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
import {
|
||||
closestHoverSurfaceCandidate,
|
||||
collectHoverSurfaceStates,
|
||||
decideHoverSurfaceRelation,
|
||||
type HoverSurfaceState,
|
||||
isHoverSurfaceCandidateElement,
|
||||
isLikelyHoverSurfaceOwner,
|
||||
} from "./record-hover-surface";
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
@@ -49,8 +70,29 @@ interface FillSession {
|
||||
lastValue: string;
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
recordedAt: number;
|
||||
score: number;
|
||||
eligible: boolean;
|
||||
}
|
||||
|
||||
interface HoverSurfaceNode {
|
||||
element: Element;
|
||||
signature: string;
|
||||
owner: HoverCandidate;
|
||||
parent?: HoverSurfaceNode;
|
||||
}
|
||||
|
||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||
|
||||
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
|
||||
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||
const HOVER_CANDIDATE_LIMIT = 24;
|
||||
const HOVER_TRIGGER_LABEL_MAX = 48;
|
||||
|
||||
function eventTarget(event: Event): EventTarget | null {
|
||||
return event.composedPath()[0] ?? event.target;
|
||||
}
|
||||
@@ -122,6 +164,173 @@ function fillableValue(el: FillableElement): string {
|
||||
return el.textContent ?? "";
|
||||
}
|
||||
|
||||
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
|
||||
return {
|
||||
id: el.id || undefined,
|
||||
class: typeof el.className === "string" ? el.className || undefined : undefined,
|
||||
"data-testid": el.getAttribute("data-testid") ?? undefined,
|
||||
"data-test": el.getAttribute("data-test") ?? undefined,
|
||||
"data-cy": el.getAttribute("data-cy") ?? undefined,
|
||||
"aria-label": el.getAttribute("aria-label") ?? undefined,
|
||||
"aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
|
||||
"aria-controls": el.getAttribute("aria-controls") ?? undefined,
|
||||
"aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
|
||||
"aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
|
||||
"aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
|
||||
contenteditable: el.getAttribute("contenteditable") ?? undefined,
|
||||
disabled: el.hasAttribute("disabled") ? "" : undefined,
|
||||
hidden: el.hasAttribute("hidden") ? "" : undefined,
|
||||
inert: el.hasAttribute("inert") ? "" : undefined,
|
||||
onclick: el.getAttribute("onclick") ?? undefined,
|
||||
onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
|
||||
onmouseover: el.getAttribute("onmouseover") ?? undefined,
|
||||
role: el.getAttribute("role") ?? undefined,
|
||||
tabindex: el.getAttribute("tabindex") ?? undefined,
|
||||
title: el.getAttribute("title") ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function hoverTriggerRect(el: Element): HoverTriggerRect {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
|
||||
}
|
||||
|
||||
function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
|
||||
if (!(el instanceof HTMLElement)) return {};
|
||||
const style = getComputedStyle(el);
|
||||
return { cursor: style.cursor, pointerEvents: style.pointerEvents };
|
||||
}
|
||||
|
||||
function hasHoverGraphicDescendant(el: Element): boolean {
|
||||
return el.querySelector("img,svg,use,path,i") !== null;
|
||||
}
|
||||
|
||||
function normalizeLabelText(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string {
|
||||
const normalized = normalizeLabelText(value);
|
||||
if (normalized.length <= max) return normalized;
|
||||
return `${normalized.slice(0, max - 1)}…`;
|
||||
}
|
||||
|
||||
function collectHoverTriggerLabelText(root: Element): string {
|
||||
let text = "";
|
||||
const visit = (node: Node) => {
|
||||
if (node.nodeType === Node.TEXT_NODE) {
|
||||
text += ` ${node.textContent ?? ""}`;
|
||||
return;
|
||||
}
|
||||
if (!(node instanceof Element)) return;
|
||||
if (node !== root && isHoverSurfaceCandidateElement(node)) {
|
||||
return;
|
||||
}
|
||||
for (const child of node.childNodes) visit(child);
|
||||
};
|
||||
for (const child of root.childNodes) visit(child);
|
||||
return normalizeLabelText(text);
|
||||
}
|
||||
|
||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||
if (!desc.name) return desc;
|
||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||
const compactName = collectHoverTriggerLabelText(el);
|
||||
if (!compactName || compactName === desc.name) return desc;
|
||||
const compactComparable = compactName.replace(/\s+/g, "");
|
||||
const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, "");
|
||||
const fullTextComparable = fullText.replace(/\s+/g, "");
|
||||
if (
|
||||
descComparable.startsWith(compactComparable) &&
|
||||
fullTextComparable.startsWith(descComparable) &&
|
||||
compactName.length < desc.name.length
|
||||
) {
|
||||
return { ...desc, name: truncateLabel(compactName) };
|
||||
}
|
||||
return desc;
|
||||
}
|
||||
|
||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||
return !target.role && !target.name && target.tag === "div";
|
||||
}
|
||||
|
||||
function looksLikeAvatarElement(el: Element): boolean {
|
||||
const className = typeof el.className === "string" ? el.className : "";
|
||||
return /\b(avatar|user-avatar)\b/i.test(className);
|
||||
}
|
||||
|
||||
function normalizeHoverTarget(
|
||||
el: Element,
|
||||
desc: TargetDescriptor,
|
||||
decision: HoverTriggerDecision,
|
||||
): TargetDescriptor {
|
||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||
return { ...desc, name: "image" };
|
||||
}
|
||||
if (
|
||||
isWeakHoverTarget(desc) &&
|
||||
(looksLikeAvatarElement(el) ||
|
||||
(hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
|
||||
) {
|
||||
return { tag: "img", role: "img", name: "image" };
|
||||
}
|
||||
return desc;
|
||||
}
|
||||
|
||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||
if (!(el instanceof HTMLElement)) return null;
|
||||
const style = hoverTriggerStyle(el);
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
role: desc.role,
|
||||
label: desc.name,
|
||||
attrs: hoverTriggerAttrs(el),
|
||||
rect: hoverTriggerRect(el),
|
||||
cursor: style.cursor,
|
||||
pointerEvents: style.pointerEvents,
|
||||
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
||||
};
|
||||
}
|
||||
|
||||
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const element = resolveHoverElement(target);
|
||||
if (!element) return null;
|
||||
const desc = describeTarget(element);
|
||||
const signals = hoverTriggerSignals(element, desc);
|
||||
if (!signals) return null;
|
||||
const decision = evaluateHoverTrigger(signals);
|
||||
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
|
||||
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
|
||||
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
|
||||
if (!decision.eligible && !desc.name && !desc.role) return null;
|
||||
const normalizedTarget = normalizeHoverTarget(
|
||||
element,
|
||||
compactHoverTargetName(element, desc),
|
||||
decision,
|
||||
);
|
||||
const now = Date.now();
|
||||
return {
|
||||
element,
|
||||
target: normalizedTarget,
|
||||
recordedAt: now,
|
||||
score: decision.score,
|
||||
eligible: decision.eligible,
|
||||
};
|
||||
}
|
||||
|
||||
function shouldReplaceHoverCandidate(
|
||||
current: HoverCandidate,
|
||||
next: HoverCandidate,
|
||||
now: number,
|
||||
): boolean {
|
||||
if (next.element === current.element) return false;
|
||||
if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
|
||||
if (current.element.contains(next.element)) return false;
|
||||
if (next.element.contains(current.element)) return true;
|
||||
return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
|
||||
}
|
||||
|
||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||
if (!session || !(target instanceof Element)) return false;
|
||||
@@ -168,10 +377,17 @@ export function startRecordCapture(
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
};
|
||||
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||
new Map(states.map((state) => [state.element, state.signature]));
|
||||
let fillSession: FillSession | null = null;
|
||||
let composing = false;
|
||||
let lastUrl = location.href;
|
||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||
let pendingHover: HoverCandidate | null = null;
|
||||
const recentHoverCandidates: HoverCandidate[] = [];
|
||||
const emittedHoverElements = new WeakSet<Element>();
|
||||
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
|
||||
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
|
||||
let generatedControlClick: Element | null = null;
|
||||
let navigationActionPending = false;
|
||||
let navigationActionVersion = 0;
|
||||
@@ -237,6 +453,223 @@ export function startRecordCapture(
|
||||
});
|
||||
};
|
||||
|
||||
const rememberHoverCandidate = (hover: HoverCandidate) => {
|
||||
const duplicate = recentHoverCandidates.findIndex(
|
||||
(candidate) => candidate.element === hover.element,
|
||||
);
|
||||
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
|
||||
recentHoverCandidates.push(hover);
|
||||
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
|
||||
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
|
||||
}
|
||||
};
|
||||
|
||||
const surfaceArea = (el: Element): number => {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width * rect.height;
|
||||
};
|
||||
|
||||
const smallestContaining = <T>(
|
||||
target: Element,
|
||||
items: Iterable<T>,
|
||||
elementFor: (item: T) => Element,
|
||||
): T | undefined => {
|
||||
let best: T | undefined;
|
||||
for (const item of items) {
|
||||
const element = elementFor(item);
|
||||
if (!element.contains(target)) continue;
|
||||
if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) {
|
||||
best = item;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
};
|
||||
|
||||
const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined =>
|
||||
smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element);
|
||||
|
||||
const surfaceStateContaining = (
|
||||
target: Element,
|
||||
states: HoverSurfaceState[],
|
||||
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
|
||||
|
||||
const inferOwnerForSurface = (
|
||||
surface: Element,
|
||||
now: number,
|
||||
before?: HoverCandidate,
|
||||
): HoverCandidate | undefined => {
|
||||
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||
const candidate = recentHoverCandidates[index];
|
||||
if (!candidate) continue;
|
||||
if (before && candidate.element === before.element) continue;
|
||||
if (before && candidate.recordedAt > before.recordedAt) continue;
|
||||
if (surface.contains(candidate.element)) continue;
|
||||
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const nodeForUnownedSurface = (
|
||||
surfaceState: HoverSurfaceState,
|
||||
currentStates: HoverSurfaceState[],
|
||||
now: number,
|
||||
): HoverSurfaceNode | undefined => {
|
||||
const owner = inferOwnerForSurface(surfaceState.element, now);
|
||||
if (!owner) return undefined;
|
||||
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||
const node: HoverSurfaceNode = {
|
||||
element: surfaceState.element,
|
||||
signature: surfaceState.signature,
|
||||
owner,
|
||||
...(parent ? { parent } : {}),
|
||||
};
|
||||
hoverSurfaceNodes.set(surfaceState.element, node);
|
||||
return node;
|
||||
};
|
||||
|
||||
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
|
||||
const closestSurface = closestHoverSurfaceCandidate(target);
|
||||
if (closestSurface) {
|
||||
const owned = hoverSurfaceNodes.get(closestSurface);
|
||||
if (owned) return owned;
|
||||
}
|
||||
const ownedContaining = ownedSurfaceContaining(target);
|
||||
if (ownedContaining) return ownedContaining;
|
||||
const now = Date.now();
|
||||
const currentStates = collectHoverSurfaceStates();
|
||||
pruneGoneHoverSurfaces(currentStates);
|
||||
const surfaceState = closestSurface
|
||||
? currentStates.find((state) => state.element === closestSurface)
|
||||
: surfaceStateContaining(target, currentStates);
|
||||
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||
if (!surfaceState) return undefined;
|
||||
return nodeForUnownedSurface(surfaceState, currentStates, now);
|
||||
};
|
||||
|
||||
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
|
||||
const current = new Set(currentStates.map((state) => state.element));
|
||||
for (const element of hoverSurfaceNodes.keys()) {
|
||||
if (!current.has(element)) hoverSurfaceNodes.delete(element);
|
||||
}
|
||||
};
|
||||
|
||||
const parentSurfaceNodeForOwner = (
|
||||
owner: HoverCandidate,
|
||||
currentStates: HoverSurfaceState[],
|
||||
now: number,
|
||||
): HoverSurfaceNode | undefined => {
|
||||
const ownedParent = ownedSurfaceContaining(owner.element);
|
||||
if (ownedParent) return ownedParent;
|
||||
const parentState = surfaceStateContaining(owner.element, currentStates);
|
||||
if (!parentState) return undefined;
|
||||
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
|
||||
if (!parentOwner) return undefined;
|
||||
const parentNode: HoverSurfaceNode = {
|
||||
element: parentState.element,
|
||||
signature: parentState.signature,
|
||||
owner: parentOwner,
|
||||
};
|
||||
hoverSurfaceNodes.set(parentState.element, parentNode);
|
||||
return parentNode;
|
||||
};
|
||||
|
||||
const createSurfaceNode = (
|
||||
state: HoverSurfaceState,
|
||||
owner: HoverCandidate,
|
||||
currentStates: HoverSurfaceState[],
|
||||
now: number,
|
||||
): HoverSurfaceNode | undefined => {
|
||||
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||
if (parent?.element === state.element) return undefined;
|
||||
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
|
||||
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
|
||||
const node: HoverSurfaceNode = {
|
||||
element: state.element,
|
||||
signature: state.signature,
|
||||
owner,
|
||||
...(parent ? { parent } : {}),
|
||||
};
|
||||
hoverSurfaceNodes.set(state.element, node);
|
||||
return node;
|
||||
};
|
||||
|
||||
const bindChangedHoverSurfaces = (
|
||||
previousStates: Map<Element, string>,
|
||||
currentStates: HoverSurfaceState[],
|
||||
now: number,
|
||||
) => {
|
||||
for (const state of currentStates) {
|
||||
if (previousStates.get(state.element) === state.signature) continue;
|
||||
const owner = inferOwnerForSurface(state.element, now);
|
||||
if (!owner) continue;
|
||||
createSurfaceNode(state, owner, currentStates, now);
|
||||
}
|
||||
};
|
||||
|
||||
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
|
||||
const now = Date.now();
|
||||
const currentStates = collectHoverSurfaceStates();
|
||||
pruneGoneHoverSurfaces(currentStates);
|
||||
bindChangedHoverSurfaces(previousStates, currentStates, now);
|
||||
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||
};
|
||||
|
||||
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
|
||||
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
|
||||
};
|
||||
|
||||
const emitHoverStep = (hover: HoverCandidate) => {
|
||||
if (emittedHoverElements.has(hover.element)) return;
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: hover.target,
|
||||
});
|
||||
emittedHoverElements.add(hover.element);
|
||||
};
|
||||
|
||||
const surfaceOwnerChain = (
|
||||
surface: HoverSurfaceNode,
|
||||
actionElement: Element,
|
||||
now: number,
|
||||
): HoverCandidate[] => {
|
||||
const chain: HoverCandidate[] = [];
|
||||
const selected = new WeakSet<Element>();
|
||||
const surfaces: HoverSurfaceNode[] = [];
|
||||
let node: HoverSurfaceNode | undefined = surface;
|
||||
while (node && surfaces.length < 4) {
|
||||
surfaces.unshift(node);
|
||||
node = node.parent;
|
||||
}
|
||||
for (const owned of surfaces) {
|
||||
const owner = owned.owner;
|
||||
if (owner.element === actionElement || actionElement.contains(owner.element)) {
|
||||
continue;
|
||||
}
|
||||
if (selected.has(owner.element)) continue;
|
||||
if (emittedHoverElements.has(owner.element)) continue;
|
||||
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||
selected.add(owner.element);
|
||||
chain.push(owner);
|
||||
}
|
||||
return chain;
|
||||
};
|
||||
|
||||
const containedHoverOwnerForAction = (
|
||||
actionElement: Element,
|
||||
now: number,
|
||||
): HoverCandidate | undefined => {
|
||||
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||
const candidate = recentHoverCandidates[index];
|
||||
if (!candidate) continue;
|
||||
if (candidate.element === actionElement) continue;
|
||||
if (!candidate.element.contains(actionElement)) continue;
|
||||
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||
return candidate;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const emitClick = (event: MouseEvent) => {
|
||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||
const target = describeEventTarget(eventTarget(event));
|
||||
@@ -249,6 +682,54 @@ export function startRecordCapture(
|
||||
});
|
||||
};
|
||||
|
||||
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
|
||||
if (!(actionTarget instanceof Element)) return;
|
||||
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
|
||||
const surface = ownedSurfaceForAction(actionElement);
|
||||
const now = Date.now();
|
||||
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
|
||||
const hoverChain = surface
|
||||
? surfaceOwnerChain(surface, actionElement, now)
|
||||
: containedOwner
|
||||
? [containedOwner]
|
||||
: [];
|
||||
if (hoverChain.length === 0) {
|
||||
pendingHover = null;
|
||||
return;
|
||||
}
|
||||
for (const hover of hoverChain) emitHoverStep(hover);
|
||||
pendingHover = null;
|
||||
};
|
||||
|
||||
const onMouseOver = (event: MouseEvent) => {
|
||||
const target = eventTarget(event);
|
||||
if (isOverlayTarget(target)) return;
|
||||
const now = Date.now();
|
||||
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
||||
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||
pendingHover = null;
|
||||
}
|
||||
const candidate = hoverCandidateFromEvent(target);
|
||||
if (candidate) {
|
||||
const previousSurfaceStates = new Map(hoverSurfaceStates);
|
||||
rememberHoverCandidate(candidate);
|
||||
refreshHoverSurfaces(previousSurfaceStates);
|
||||
scheduleHoverSurfaceRefresh(previousSurfaceStates);
|
||||
}
|
||||
const hover = candidate?.eligible ? candidate : null;
|
||||
if (pendingHover && target instanceof Element) {
|
||||
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
|
||||
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!hover) return;
|
||||
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
|
||||
return;
|
||||
}
|
||||
pendingHover = hover;
|
||||
};
|
||||
|
||||
const onClick = (event: MouseEvent) => {
|
||||
if (event.button !== 0) return;
|
||||
const target = eventTarget(event);
|
||||
@@ -278,6 +759,7 @@ export function startRecordCapture(
|
||||
|
||||
const fillable = fillableFromTarget(target);
|
||||
if (fillable) {
|
||||
emitHoverCandidateBeforeAction(fillable);
|
||||
ensureFillSession(fillable);
|
||||
return;
|
||||
}
|
||||
@@ -285,6 +767,7 @@ export function startRecordCapture(
|
||||
if (target instanceof Element) {
|
||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||
if (nearbyFillable) {
|
||||
emitHoverCandidateBeforeAction(nearbyFillable);
|
||||
ensureFillSession(nearbyFillable);
|
||||
return;
|
||||
}
|
||||
@@ -304,6 +787,7 @@ export function startRecordCapture(
|
||||
|
||||
commitFillSession();
|
||||
if (target instanceof Element && target.closest("select")) return;
|
||||
emitHoverCandidateBeforeAction(target);
|
||||
emitClick(event);
|
||||
};
|
||||
|
||||
@@ -345,6 +829,7 @@ export function startRecordCapture(
|
||||
commitFillSession();
|
||||
const target = eventTarget(event);
|
||||
if (target instanceof HTMLSelectElement) {
|
||||
emitHoverCandidateBeforeAction(target);
|
||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||
(opt.label || opt.textContent || opt.value).trim(),
|
||||
@@ -398,6 +883,7 @@ export function startRecordCapture(
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
emitHoverCandidateBeforeAction(target);
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
@@ -408,6 +894,7 @@ export function startRecordCapture(
|
||||
};
|
||||
|
||||
document.addEventListener("click", onClick, true);
|
||||
document.addEventListener("mouseover", onMouseOver, true);
|
||||
document.addEventListener("focusin", onFocusIn, true);
|
||||
document.addEventListener("focusout", onFocusOut, true);
|
||||
document.addEventListener("input", onInput, true);
|
||||
@@ -438,6 +925,7 @@ export function startRecordCapture(
|
||||
dispose() {
|
||||
commitFillSession();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("mouseover", onMouseOver, true);
|
||||
document.removeEventListener("focusin", onFocusIn, true);
|
||||
document.removeEventListener("focusout", onFocusOut, true);
|
||||
document.removeEventListener("input", onInput, true);
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
export interface HoverSurfaceDecision {
|
||||
related: boolean;
|
||||
reason?: string;
|
||||
surface?: Element;
|
||||
}
|
||||
|
||||
export interface HoverSurfaceContext {
|
||||
triggerElement: Element;
|
||||
}
|
||||
|
||||
export interface HoverSurfaceState {
|
||||
element: Element;
|
||||
signature: string;
|
||||
}
|
||||
|
||||
const HOVER_SURFACE_SELECTOR = [
|
||||
'[role="menu"]',
|
||||
'[role="menubar"]',
|
||||
'[role="listbox"]',
|
||||
'[role="dialog"]',
|
||||
'[role="tooltip"]',
|
||||
'[aria-modal="true"]',
|
||||
"[data-popper-placement]",
|
||||
"[data-floating-ui-placement]",
|
||||
"[data-headlessui-state]",
|
||||
"[data-radix-popper-content-wrapper]",
|
||||
].join(",");
|
||||
|
||||
const HOVER_SURFACE_WORD_RE =
|
||||
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
|
||||
const HOVER_SURFACE_ITEM_RE =
|
||||
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
|
||||
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
||||
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
|
||||
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
||||
|
||||
function elementTokenText(el: Element): string {
|
||||
return [
|
||||
el.id,
|
||||
typeof el.className === "string" ? el.className : "",
|
||||
el.getAttribute("data-testid"),
|
||||
el.getAttribute("data-test"),
|
||||
el.getAttribute("data-cy"),
|
||||
el.getAttribute("aria-label"),
|
||||
el.getAttribute("role"),
|
||||
]
|
||||
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
export function isRecognizedHoverSurfaceElement(el: Element): boolean {
|
||||
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
|
||||
const tokenText = elementTokenText(el);
|
||||
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isVisibleSurfaceElement(el: Element): boolean {
|
||||
if (!(el instanceof HTMLElement)) return false;
|
||||
const style = getComputedStyle(el);
|
||||
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||
return false;
|
||||
}
|
||||
return rectFor(el) !== null;
|
||||
}
|
||||
|
||||
export function closestRecognizedHoverSurface(element: Element): Element | null {
|
||||
let node: Element | null = element;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||
if (isRecognizedHoverSurfaceElement(node)) return node;
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isPositionedFloatingElement(el: Element): boolean {
|
||||
if (!(el instanceof HTMLElement)) return false;
|
||||
const style = getComputedStyle(el);
|
||||
if (style.position !== "absolute") return false;
|
||||
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||
return false;
|
||||
}
|
||||
const rect = rectFor(el);
|
||||
if (!rect) return false;
|
||||
const viewportArea = window.innerWidth * window.innerHeight;
|
||||
const area = rect.width * rect.height;
|
||||
return (
|
||||
area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO)
|
||||
);
|
||||
}
|
||||
|
||||
export function isHoverSurfaceCandidateElement(el: Element): boolean {
|
||||
return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el);
|
||||
}
|
||||
|
||||
export function closestHoverSurfaceCandidate(element: Element): Element | null {
|
||||
let node: Element | null = element;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||
if (isHoverSurfaceCandidateElement(node)) return node;
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function closestFloatingSurface(clickElement: Element): Element | null {
|
||||
let node: Element | null = clickElement;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||
if (isPositionedFloatingElement(node)) return node;
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function hoverSurfaceSignature(el: Element): string {
|
||||
const rect = rectFor(el);
|
||||
return [
|
||||
rect
|
||||
? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}`
|
||||
: "",
|
||||
(el.textContent ?? "").trim().slice(0, 512),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
export function collectHoverSurfaceStates(): HoverSurfaceState[] {
|
||||
const states: HoverSurfaceState[] = [];
|
||||
for (const el of document.querySelectorAll("*")) {
|
||||
if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) {
|
||||
states.push({ element: el, signature: hoverSurfaceSignature(el) });
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function rectFor(el: Element): DOMRect | null {
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width <= 0 || rect.height <= 0) return null;
|
||||
return rect;
|
||||
}
|
||||
|
||||
function surfaceRelationMetrics(surface: Element, trigger: Element) {
|
||||
const surfaceRect = rectFor(surface);
|
||||
const triggerRect = rectFor(trigger);
|
||||
if (!surfaceRect || !triggerRect) return null;
|
||||
const horizontalOverlap =
|
||||
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
|
||||
const verticalOverlap =
|
||||
Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top);
|
||||
const horizontalGap = Math.max(
|
||||
0,
|
||||
triggerRect.left - surfaceRect.right,
|
||||
surfaceRect.left - triggerRect.right,
|
||||
);
|
||||
const verticalGap = Math.max(
|
||||
0,
|
||||
triggerRect.top - surfaceRect.bottom,
|
||||
surfaceRect.top - triggerRect.bottom,
|
||||
);
|
||||
return {
|
||||
surfaceRect,
|
||||
triggerRect,
|
||||
horizontalOverlap,
|
||||
verticalOverlap,
|
||||
horizontalGap,
|
||||
verticalGap,
|
||||
};
|
||||
}
|
||||
|
||||
function isNearTrigger(surface: Element, trigger: Element): boolean {
|
||||
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||
if (!metrics) return true;
|
||||
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
|
||||
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||
return (
|
||||
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP
|
||||
);
|
||||
}
|
||||
|
||||
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
|
||||
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||
if (!metrics) return true;
|
||||
const {
|
||||
surfaceRect,
|
||||
triggerRect,
|
||||
horizontalOverlap,
|
||||
verticalOverlap,
|
||||
horizontalGap,
|
||||
verticalGap,
|
||||
} = metrics;
|
||||
const opensVertically =
|
||||
horizontalOverlap > 0 &&
|
||||
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
|
||||
const opensSideways =
|
||||
verticalOverlap > 0 &&
|
||||
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
|
||||
const opensFromCorner =
|
||||
horizontalGap > 0 &&
|
||||
verticalGap > 0 &&
|
||||
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
|
||||
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
|
||||
return opensVertically || opensSideways || opensFromCorner;
|
||||
}
|
||||
|
||||
export function decideHoverSurfaceRelation(
|
||||
context: HoverSurfaceContext,
|
||||
clickElement: Element,
|
||||
): HoverSurfaceDecision {
|
||||
const clickSurface = closestRecognizedHoverSurface(clickElement);
|
||||
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
||||
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||
}
|
||||
|
||||
const floatingSurface = closestFloatingSurface(clickElement);
|
||||
if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) {
|
||||
return {
|
||||
related: true,
|
||||
reason: "click-inside-near-floating-surface",
|
||||
surface: floatingSurface,
|
||||
};
|
||||
}
|
||||
|
||||
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
|
||||
return { related: true, reason: "click-inside-trigger-contained-surface" };
|
||||
}
|
||||
|
||||
return { related: false, reason: "click-outside-hover-surface" };
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { evaluateHoverTrigger } from "../hover-trigger-policy";
|
||||
|
||||
describe("evaluateHoverTrigger", () => {
|
||||
it("accepts explicit popup triggers", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
label: "Open user navigation menu",
|
||||
attrs: { "aria-haspopup": "menu", "aria-expanded": "false" },
|
||||
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||
cursor: "pointer",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(true);
|
||||
expect(decision.reasons).toEqual(
|
||||
expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts compact topbar image buttons without popup attributes", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
label: "image",
|
||||
attrs: {},
|
||||
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||
pointerEvents: "auto",
|
||||
hasGraphicDescendant: true,
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(true);
|
||||
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||
});
|
||||
|
||||
it("accepts compact topbar image links without popup attributes", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "a",
|
||||
role: "link",
|
||||
label: "image",
|
||||
attrs: {},
|
||||
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||
pointerEvents: "auto",
|
||||
hasGraphicDescendant: true,
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(true);
|
||||
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||
});
|
||||
|
||||
it("rejects topbar menu links without surface trigger evidence", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "a",
|
||||
role: "link",
|
||||
label: "My profile",
|
||||
attrs: {},
|
||||
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||
cursor: "pointer",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects text navigation items even when their attributes contain popup words", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "a",
|
||||
role: "link",
|
||||
label: "My profile",
|
||||
attrs: { class: "dropdown-item profile-link" },
|
||||
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||
cursor: "pointer",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects labelled dropdown list items as hover triggers", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "li",
|
||||
label: "文档C+D",
|
||||
attrs: { class: "dropdown-item", tabindex: "0" },
|
||||
rect: { x: 820, y: 48, w: 160, h: 32 },
|
||||
cursor: "pointer",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects ordinary controls without hover popup signals", () => {
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
label: "Search",
|
||||
attrs: {},
|
||||
rect: { x: 100, y: 240, w: 120, h: 36 },
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
|
||||
expect(decision.eligible).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects unsafe or non-visible triggers", () => {
|
||||
expect(
|
||||
evaluateHoverTrigger({
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||
pointerEvents: "auto",
|
||||
}).eligible,
|
||||
).toBe(false);
|
||||
expect(
|
||||
evaluateHoverTrigger({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
attrs: { hidden: "" },
|
||||
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||
pointerEvents: "auto",
|
||||
}).eligible,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "span", role: "button", name: "Account" },
|
||||
page_url: "https://example.com/app",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "a", role: "link", name: "Profile" },
|
||||
page_url: "https://example.com/app",
|
||||
},
|
||||
],
|
||||
"https://example.com/app",
|
||||
);
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
page: "p1",
|
||||
target: { name: "Account" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
|
||||
@@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveHoverElement(target: Element): Element | null {
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (clickable) return clickable;
|
||||
|
||||
let node: Element | null = target;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
|
||||
if (looksClickable(node)) return node;
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
export interface HoverTriggerRect {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface HoverTriggerSignals {
|
||||
tag: string;
|
||||
role?: string;
|
||||
label?: string;
|
||||
attrs?: Record<string, string | undefined>;
|
||||
rect?: HoverTriggerRect | null;
|
||||
cursor?: string;
|
||||
pointerEvents?: string;
|
||||
hasGraphicDescendant?: boolean;
|
||||
cssHoverMatch?: boolean;
|
||||
}
|
||||
|
||||
export interface HoverTriggerDecision {
|
||||
eligible: boolean;
|
||||
score: number;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
const HOVER_TRIGGER_MIN_SCORE = 45;
|
||||
const HOVER_POPUP_SIGNAL_RE =
|
||||
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
||||
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
|
||||
|
||||
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
||||
return signals.attrs?.[name.toLowerCase()];
|
||||
}
|
||||
|
||||
export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean {
|
||||
const tag = signals.tag.toLowerCase();
|
||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||
if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true;
|
||||
if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true;
|
||||
return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true";
|
||||
}
|
||||
|
||||
export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
|
||||
const attrs = signals.attrs ?? {};
|
||||
const haystack = [
|
||||
attrs.id,
|
||||
attrs.class,
|
||||
attrs["data-testid"],
|
||||
attrs["data-test"],
|
||||
attrs["data-cy"],
|
||||
attrs["aria-label"],
|
||||
attrs["aria-haspopup"],
|
||||
attrs["aria-controls"],
|
||||
attrs.title,
|
||||
signals.role === "button" ? signals.label : undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
||||
}
|
||||
|
||||
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
|
||||
return (
|
||||
signals.cssHoverMatch === true ||
|
||||
attr(signals, "aria-haspopup") !== undefined ||
|
||||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
|
||||
);
|
||||
}
|
||||
|
||||
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
||||
const tag = signals.tag.toLowerCase();
|
||||
const role = (signals.role ?? "").toLowerCase();
|
||||
return (
|
||||
["button", "a", "summary"].includes(tag) ||
|
||||
["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
|
||||
attr(signals, "tabindex") !== undefined ||
|
||||
signals.cursor === "pointer" ||
|
||||
attr(signals, "onclick") !== undefined ||
|
||||
attr(signals, "onmouseenter") !== undefined ||
|
||||
attr(signals, "onmouseover") !== undefined ||
|
||||
attr(signals, "aria-haspopup") !== undefined ||
|
||||
attr(signals, "aria-expanded") === "false"
|
||||
);
|
||||
}
|
||||
|
||||
export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision {
|
||||
const rect = signals.rect;
|
||||
if (
|
||||
!rect ||
|
||||
rect.w <= 0 ||
|
||||
rect.h <= 0 ||
|
||||
signals.pointerEvents === "none" ||
|
||||
attr(signals, "hidden") !== undefined ||
|
||||
attr(signals, "inert") !== undefined ||
|
||||
(attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" ||
|
||||
isUnsafeHoverTrigger(signals)
|
||||
) {
|
||||
return { eligible: false, score: 0, reasons: [] };
|
||||
}
|
||||
|
||||
const area = rect.w * rect.h;
|
||||
if (area <= 0 || area > 160_000) {
|
||||
return { eligible: false, score: 0, reasons: [] };
|
||||
}
|
||||
|
||||
const reasons: string[] = [];
|
||||
let score = 0;
|
||||
const tag = signals.tag.toLowerCase();
|
||||
const role = (signals.role ?? "").toLowerCase();
|
||||
const label = signals.label?.trim();
|
||||
const graphic = signals.hasGraphicDescendant === true;
|
||||
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
||||
const popupSignal = hasHoverPopupSignal(signals);
|
||||
const isTextNavigationItem =
|
||||
(tag === "a" ||
|
||||
tag === "li" ||
|
||||
role === "link" ||
|
||||
role === "option" ||
|
||||
role.startsWith("menuitem") ||
|
||||
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
|
||||
!graphic &&
|
||||
!!label &&
|
||||
label.toLowerCase() !== "image";
|
||||
const compactTopbarIcon =
|
||||
(role === "button" || role === "link" || tag === "button" || tag === "a") &&
|
||||
graphic &&
|
||||
rect.y < 120 &&
|
||||
rect.w <= 96 &&
|
||||
rect.h <= 96 &&
|
||||
(!label || label.toLowerCase() === "image");
|
||||
const surfaceTriggerEvidence =
|
||||
(popupSignal && !isTextNavigationItem) ||
|
||||
compactTopbarIcon ||
|
||||
hasStrongHoverExpansionSignal(signals);
|
||||
if (!directInteractive || !surfaceTriggerEvidence) {
|
||||
return { eligible: false, score: 0, reasons: [] };
|
||||
}
|
||||
|
||||
if (attr(signals, "aria-haspopup") !== undefined) {
|
||||
score += 80;
|
||||
reasons.push("aria-haspopup");
|
||||
}
|
||||
if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") {
|
||||
score += 55;
|
||||
reasons.push("collapsed");
|
||||
}
|
||||
if (popupSignal) {
|
||||
score += 45;
|
||||
reasons.push("popup-signal");
|
||||
}
|
||||
if (tag === "button" || role === "button") {
|
||||
score += 30;
|
||||
reasons.push("button");
|
||||
}
|
||||
if (signals.cursor === "pointer") {
|
||||
score += 25;
|
||||
reasons.push("pointer");
|
||||
}
|
||||
if (graphic && (!label || label.toLowerCase() === "image")) {
|
||||
score += 40;
|
||||
reasons.push("icon-only");
|
||||
}
|
||||
if (rect.y < 120) {
|
||||
score += 25;
|
||||
reasons.push("topbar");
|
||||
}
|
||||
if (rect.w <= 80 && rect.h <= 80) {
|
||||
score += 20;
|
||||
reasons.push("compact");
|
||||
}
|
||||
if (signals.cssHoverMatch) {
|
||||
score += 40;
|
||||
reasons.push("css-hover");
|
||||
}
|
||||
|
||||
return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons };
|
||||
}
|
||||
@@ -33,7 +33,7 @@ export interface RecordStartMessage {
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "fill" | "press" | "select" | "navigate";
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
|
||||
@@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "hover":
|
||||
return payload.target
|
||||
? {
|
||||
op: "hover",
|
||||
target: payload.target,
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "fill":
|
||||
return payload.target
|
||||
? {
|
||||
|
||||
@@ -139,6 +139,13 @@ function toV2Step(
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
};
|
||||
case "fill":
|
||||
return {
|
||||
op: "fill",
|
||||
|
||||
@@ -9,6 +9,8 @@ import type {
|
||||
} from "@/transport/types";
|
||||
import { ToolDispatcher } from "../dispatcher";
|
||||
|
||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||
|
||||
function fakeTransport() {
|
||||
const handlers = new Set<FrameHandler>();
|
||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||
@@ -129,7 +131,7 @@ describe("ToolDispatcher", () => {
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
||||
@@ -172,7 +174,7 @@ describe("ToolDispatcher", () => {
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
@@ -311,6 +313,277 @@ describe("ToolDispatcher", () => {
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => {
|
||||
order.push("hover");
|
||||
return {} as T;
|
||||
}),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
|
||||
(
|
||||
dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
|
||||
).rememberHover("aa11", {
|
||||
tab_id: 7,
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options?: { releaseAfter?: boolean },
|
||||
) => Promise<T>;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
};
|
||||
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
|
||||
order.push("observe");
|
||||
return {};
|
||||
});
|
||||
expect(order).toEqual(["hover", "observe"]);
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
await helpers.withHoverReassert(
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
async () => {
|
||||
order.push("click");
|
||||
return {};
|
||||
},
|
||||
{ releaseAfter: true },
|
||||
);
|
||||
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not reassert or disable hover latches from another session", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options?: { releaseAfter?: boolean },
|
||||
) => Promise<T>;
|
||||
releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
|
||||
};
|
||||
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
|
||||
await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
|
||||
releaseAfter: true,
|
||||
});
|
||||
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.releaseHoverLatch("aa11", 7);
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
const helpers = dispatcher as unknown as {
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
};
|
||||
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
await helpers.setHoverBypass("bb22", 7, true);
|
||||
await helpers.setHoverBypass("aa11", 7, false);
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.setHoverBypass("bb22", 7, false);
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("limits default-target hover reassertion to the active tab", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
|
||||
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 9,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
|
||||
|
||||
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
|
||||
|
||||
expect(cdp.send).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 30,
|
||||
y: 40,
|
||||
});
|
||||
});
|
||||
|
||||
it("releases the current session hover latch before navigation-style work", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReleaseForRequest: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
|
||||
await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
handleClick,
|
||||
handleFill,
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
modifiersBitfield,
|
||||
@@ -435,6 +436,50 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleHover", () => {
|
||||
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
|
||||
const res = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
|
||||
{
|
||||
cdp: fake.cdp,
|
||||
tabsApi: fake.tabsApi,
|
||||
bypassOverlay,
|
||||
keepOverlayBypassAfterHover: true,
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||
expect(bypassOverlay).toHaveBeenCalledTimes(1);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
|
||||
@@ -1430,7 +1430,7 @@ describe("buildVomScene", () => {
|
||||
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
|
||||
});
|
||||
|
||||
it("maps hover surface probes to matching node labels", () => {
|
||||
it("maps hover surface probes to matching backend node ids", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1451,7 +1451,7 @@ describe("buildVomScene", () => {
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
@@ -1480,7 +1480,260 @@ describe("buildVomScene", () => {
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("maps hover surface probes to rendered descendants in the same trigger subtree", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "image" },
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
|
||||
it("deduplicates hover surface probes by original trigger backend id", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "image" },
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps hover surface probes to DOM-recovered custom controls", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "img" },
|
||||
name: { type: "computedString", value: "image" },
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain(
|
||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||
);
|
||||
});
|
||||
|
||||
it("does not attach hover probes to distant geometry matches", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "link" },
|
||||
name: { type: "computedString", value: "0" },
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
@@ -1953,7 +2206,7 @@ describe("handleSnapshot", () => {
|
||||
});
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
@@ -1974,6 +2227,7 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
|
||||
@@ -8,6 +8,8 @@ import type {
|
||||
EvaluateParams,
|
||||
FillParams,
|
||||
GetHtmlParams,
|
||||
HoverParams,
|
||||
HoverResult,
|
||||
NavigateBackParams,
|
||||
NavigateForwardParams,
|
||||
NavigateParams,
|
||||
@@ -33,7 +35,7 @@ import { handleConsole } from "./console";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
handleNavigateBack,
|
||||
@@ -56,7 +58,7 @@ import {
|
||||
type SessionStartParams,
|
||||
type SessionStopParams,
|
||||
} from "./session";
|
||||
import { chromeTabsApi } from "./shared";
|
||||
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||
import {
|
||||
type BorrowConfirmationApprover,
|
||||
handleTabBorrow,
|
||||
@@ -81,6 +83,18 @@ type DispatcherCdpRunner = CdpRunner &
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
|
||||
interface HoverLatch {
|
||||
sessionId: string;
|
||||
tabId: number;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface HoverLatchScope {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
}
|
||||
|
||||
export interface DispatcherDeps {
|
||||
transport: Transport;
|
||||
sessions: SessionManager;
|
||||
@@ -126,6 +140,8 @@ export class ToolDispatcher {
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
private readonly hoverBypassTabs = new Map<number, string>();
|
||||
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||
/**
|
||||
* Per-rpc-id `AbortController` registry. Populated inside
|
||||
* [`dispatch`] before we await the tool handler and torn down in
|
||||
@@ -271,16 +287,20 @@ export class ToolDispatcher {
|
||||
switch (req.method) {
|
||||
case "tool.session_start":
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
||||
case "tool.session_stop":
|
||||
case "tool.session_stop": {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
||||
case "tool.tab_close":
|
||||
return handleTabClose(this.sessions, req.params as TabCloseParams);
|
||||
return this.withHoverReleaseForRequest(req.params as TabCloseParams, () =>
|
||||
handleTabClose(this.sessions, req.params as TabCloseParams),
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
||||
case "tool.tab_borrow":
|
||||
@@ -319,17 +339,31 @@ export class ToolDispatcher {
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
);
|
||||
case "tool.snapshot":
|
||||
return handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
return this.withHoverReassert(req.params as SnapshotParams, () =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.observe":
|
||||
return handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
case "tool.observe": {
|
||||
const params = req.params as ObserveParams;
|
||||
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||
return this.withHoverReassert(params, () =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
params,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
case "tool.get_html":
|
||||
return handleGetHtml(
|
||||
this.sessions,
|
||||
@@ -337,68 +371,95 @@ export class ToolDispatcher {
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.navigate":
|
||||
return handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateParams, () =>
|
||||
handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.navigate_back":
|
||||
return handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () =>
|
||||
handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.navigate_forward":
|
||||
return handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () =>
|
||||
handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.reload":
|
||||
return handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as ReloadParams, () =>
|
||||
handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.click":
|
||||
return handleClick(
|
||||
this.sessions,
|
||||
return this.withHoverReassert(
|
||||
req.params as ClickParams,
|
||||
() =>
|
||||
handleClick(
|
||||
this.sessions,
|
||||
req.params as ClickParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
{ releaseAfter: true },
|
||||
);
|
||||
case "tool.hover": {
|
||||
const result = await handleHover(
|
||||
this.sessions,
|
||||
req.params as HoverParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
bypassOverlay: (tabId, enabled) =>
|
||||
this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled),
|
||||
keepOverlayBypassAfterHover: true,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||
}
|
||||
case "tool.fill":
|
||||
return handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as FillParams, () =>
|
||||
handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.press":
|
||||
return handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as PressParams, () =>
|
||||
handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.select":
|
||||
return handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as SelectParams, () =>
|
||||
handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
@@ -471,6 +532,114 @@ export class ToolDispatcher {
|
||||
} satisfies RpcError;
|
||||
}
|
||||
}
|
||||
|
||||
private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise<void> {
|
||||
const owner = this.hoverBypassTabs.get(tabId);
|
||||
if (enabled) {
|
||||
if (owner === sessionId) return;
|
||||
if (owner === undefined) await bypassOverlay(tabId, true);
|
||||
this.hoverBypassTabs.set(tabId, sessionId);
|
||||
} else {
|
||||
if (owner !== sessionId) return;
|
||||
await bypassOverlay(tabId, false);
|
||||
this.hoverBypassTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError {
|
||||
if (!isRpcError(result)) {
|
||||
this.hoverLatches.set(result.tab_id, {
|
||||
sessionId,
|
||||
tabId: result.tab_id,
|
||||
x: result.x,
|
||||
y: result.y,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
|
||||
return this.hoverLatchesForRequest(scope).length > 0;
|
||||
}
|
||||
|
||||
private async withHoverReassert<T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options: { releaseAfter?: boolean } = {},
|
||||
): Promise<T> {
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
await this.reassertHover(scope);
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
if (options.releaseAfter) {
|
||||
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async withHoverReleaseForRequest<T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||
return work();
|
||||
}
|
||||
|
||||
private async resolveHoverLatchScope(params: {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
}): Promise<HoverLatchScope> {
|
||||
if (params.tab_id !== undefined) return params;
|
||||
const ctx = lookupSession(this.sessions, params, "hover latch");
|
||||
if (isRpcError(ctx)) return params;
|
||||
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
|
||||
if (isRpcError(target)) return params;
|
||||
return { session_id: params.session_id, tab_id: target.tabId };
|
||||
}
|
||||
|
||||
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
||||
return [...this.hoverLatches.values()].filter((latch) => {
|
||||
if (latch.sessionId !== params.session_id) return false;
|
||||
return params.tab_id === undefined || latch.tabId === params.tab_id;
|
||||
});
|
||||
}
|
||||
|
||||
private async reassertHover(params: { session_id: string; tab_id?: number }): Promise<void> {
|
||||
if (!this.cdp) return;
|
||||
await Promise.all(
|
||||
this.hoverLatchesForRequest(params).map((latch) =>
|
||||
this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: latch.x,
|
||||
y: latch.y,
|
||||
}).catch((err) => {
|
||||
console.debug("[bsk dispatcher] hover reassert failed", err);
|
||||
this.hoverLatches.delete(latch.tabId);
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
|
||||
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
|
||||
if (sessionId !== undefined && entrySessionId !== sessionId) return false;
|
||||
return tabId === undefined || entryTabId === tabId;
|
||||
};
|
||||
const tabs = new Set<number>();
|
||||
for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) {
|
||||
if (!matchesScope(bypassSessionId, bypassTabId)) continue;
|
||||
tabs.add(bypassTabId);
|
||||
this.hoverBypassTabs.delete(bypassTabId);
|
||||
}
|
||||
for (const latch of this.hoverLatches.values()) {
|
||||
if (!matchesScope(latch.sessionId, latch.tabId)) continue;
|
||||
tabs.add(latch.tabId);
|
||||
this.hoverLatches.delete(latch.tabId);
|
||||
}
|
||||
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
|
||||
}
|
||||
}
|
||||
|
||||
function isRpcError(v: unknown): v is RpcError {
|
||||
@@ -497,6 +666,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.navigate_forward":
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.hover":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
@@ -512,6 +682,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
|
||||
@@ -17,6 +17,8 @@ import type {
|
||||
ClickResult,
|
||||
FillParams,
|
||||
FillResult,
|
||||
HoverParams,
|
||||
HoverResult,
|
||||
KeyModifier,
|
||||
MouseButton,
|
||||
PressParams,
|
||||
@@ -53,9 +55,12 @@ export interface InteractionDeps {
|
||||
defaultTimeoutMs?: number;
|
||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||
|
||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||
@@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isAbortLikeError(err: unknown): boolean {
|
||||
return err instanceof DOMException && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => signal?.removeEventListener("abort", onAbort);
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
reject(new DOMException("aborted", "AbortError"));
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
@@ -286,7 +312,104 @@ export async function handleClick(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
if (automationBypassEnabled && deps.bypassOverlay) {
|
||||
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function handleHover(
|
||||
manager: SessionManager,
|
||||
params: HoverParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<HoverResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "hover");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const aborted = throwIfAborted(deps.signal);
|
||||
if (aborted) return aborted;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "hover");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
}
|
||||
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
let hoverCompleted = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
automationBypassEnabled = true;
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
modifiers,
|
||||
});
|
||||
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
|
||||
if (settleMs > 0) {
|
||||
await wait(settleMs, deps.signal);
|
||||
}
|
||||
hoverCompleted = true;
|
||||
} catch (err) {
|
||||
if (isAbortLikeError(err)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
}
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
if (
|
||||
automationBypassEnabled &&
|
||||
deps.bypassOverlay &&
|
||||
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
|
||||
) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
import {
|
||||
type ActiveScopeBlock,
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomScene,
|
||||
@@ -908,39 +910,28 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function labelForSurfaceMatch(node: VomNode): string {
|
||||
return (
|
||||
cleanAttr(node.name) ??
|
||||
cleanAttr(node.text) ??
|
||||
cleanAttr(node.attrs?.["aria-label"]) ??
|
||||
cleanAttr(node.attrs?.title) ??
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
const nodesWithLabels = nodes
|
||||
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
|
||||
.filter((entry) => entry.key.length > 0);
|
||||
const recoveredNodes = applyVomInteractionRecovery(nodes);
|
||||
const signals = capturedOnlySignals(captured.nodes);
|
||||
const used = new Set<number>();
|
||||
for (const probe of probes) {
|
||||
const probeKey = normalizeProbeKey(probe.triggerLabel);
|
||||
if (!probeKey || probe.subItems.length === 0) continue;
|
||||
const match =
|
||||
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
|
||||
nodesWithLabels.find(
|
||||
(entry) =>
|
||||
!used.has(entry.node.id) &&
|
||||
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
|
||||
);
|
||||
if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue;
|
||||
const match = findSurfaceTriggerNode(
|
||||
probe.triggerBackendNodeId,
|
||||
recoveredNodes,
|
||||
signals,
|
||||
probe.triggerPoint,
|
||||
);
|
||||
if (!match) continue;
|
||||
used.add(match.node.id);
|
||||
if (used.has(match.id)) continue;
|
||||
used.add(probe.triggerBackendNodeId);
|
||||
used.add(match.id);
|
||||
surfaces.push({
|
||||
triggerId: match.node.id,
|
||||
triggerId: match.id,
|
||||
triggerAction: probe.triggerAction,
|
||||
subItems: probe.subItems,
|
||||
});
|
||||
@@ -948,6 +939,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel)
|
||||
return surfaces;
|
||||
}
|
||||
|
||||
function findSurfaceTriggerNode(
|
||||
triggerBackendNodeId: number,
|
||||
nodes: VomNode[],
|
||||
signals: VomNodeDomSignals,
|
||||
triggerPoint?: { x: number; y: number },
|
||||
): VomNode | undefined {
|
||||
const renderedById = new Map(nodes.map((node) => [node.id, node]));
|
||||
const exact = renderedById.get(triggerBackendNodeId);
|
||||
if (exact && isSurfaceAttachableNode(exact)) return exact;
|
||||
|
||||
const domDescendant = nodes.find(
|
||||
(node) =>
|
||||
isSurfaceAttachableNode(node) &&
|
||||
(node.domParentId === triggerBackendNodeId ||
|
||||
node.domAncestorIds?.includes(triggerBackendNodeId) === true),
|
||||
);
|
||||
if (domDescendant) return domDescendant;
|
||||
|
||||
const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])];
|
||||
while (queue.length > 0) {
|
||||
const child = queue.shift() as CapturedNode;
|
||||
const rendered = renderedById.get(child.backendNodeId);
|
||||
if (rendered && isSurfaceAttachableNode(rendered)) return rendered;
|
||||
queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? []));
|
||||
}
|
||||
|
||||
let current = signals.capturedByBackendId.get(triggerBackendNodeId);
|
||||
while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) {
|
||||
const parent = renderedById.get(current.parentBackendNodeId);
|
||||
if (parent && isSurfaceAttachableNode(parent)) return parent;
|
||||
current = signals.capturedByBackendId.get(current.parentBackendNodeId);
|
||||
}
|
||||
|
||||
if (triggerPoint) {
|
||||
return findSurfaceNodeByPoint(triggerPoint, nodes, signals);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isSurfaceAttachableNode(node: VomNode): boolean {
|
||||
return isVomReferenceNode(node);
|
||||
}
|
||||
|
||||
function findSurfaceNodeByPoint(
|
||||
point: { x: number; y: number },
|
||||
nodes: VomNode[],
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode | undefined {
|
||||
let best: { node: VomNode; score: number } | undefined;
|
||||
for (const node of nodes) {
|
||||
if (!isSurfaceAttachableNode(node)) continue;
|
||||
const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect;
|
||||
if (!rect) continue;
|
||||
const contains =
|
||||
point.x >= rect.x &&
|
||||
point.x <= rect.x + rect.w &&
|
||||
point.y >= rect.y &&
|
||||
point.y <= rect.y + rect.h;
|
||||
const centerX = rect.x + rect.w / 2;
|
||||
const centerY = rect.y + rect.h / 2;
|
||||
const distance = Math.hypot(point.x - centerX, point.y - centerY);
|
||||
if (!contains && distance > 40) continue;
|
||||
const score = contains ? distance : distance + 1_000;
|
||||
if (!best || score < best.score) {
|
||||
best = { node, score };
|
||||
}
|
||||
}
|
||||
return best?.node;
|
||||
}
|
||||
|
||||
function axNodeInOverlaySubtree(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
@@ -1072,6 +1134,8 @@ export interface SnapshotDeps {
|
||||
get(tabId: number): Promise<chrome.tabs.Tab>;
|
||||
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
||||
};
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
let defaultDeps: SnapshotDeps | null = null;
|
||||
@@ -1217,9 +1281,13 @@ async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
});
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
@@ -1258,7 +1326,14 @@ async function handleVomObservation(
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
effectiveConditionalSurfaceProbe,
|
||||
deps.hoverProbeBypassOverlay,
|
||||
);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
@@ -1275,6 +1350,19 @@ async function handleVomObservation(
|
||||
ref_count: rendered.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: rendered.truncated,
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
sub_items: probe.subItems,
|
||||
...(probe.confidence ? { confidence: probe.confidence } : {}),
|
||||
})),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
|
||||
@@ -51,6 +51,156 @@ function fakeSnapshotReply() {
|
||||
};
|
||||
}
|
||||
|
||||
function hoverTriggerSnapshotReply() {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"class",
|
||||
"user-avatar",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeType: [1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("class"), i("user-avatar")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[940, 16, 32, 32],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function twoHoverTriggerSnapshotReply() {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"class",
|
||||
"user-avatar",
|
||||
"secondary-dropdown-trigger",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("class"), i("user-avatar")],
|
||||
[i("class"), i("secondary-dropdown-trigger")],
|
||||
],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[940, 16, 32, 32],
|
||||
[100, 120, 32, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function nestedHoverTriggerSnapshotReply() {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"img",
|
||||
"class",
|
||||
"tg-avatar",
|
||||
"tg-avatar__inner",
|
||||
"tg-avatar__image",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 3],
|
||||
nodeType: [1, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")],
|
||||
backendNodeId: [10, 11, 12, 13, 14],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("class"), i("tg-avatar")],
|
||||
[i("class"), i("tg-avatar__inner")],
|
||||
[i("class"), i("tg-avatar__image")],
|
||||
],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[940, 16, 32, 32],
|
||||
[942, 18, 28, 28],
|
||||
[944, 20, 24, 24],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCdp(snapshot: unknown) {
|
||||
return {
|
||||
send: vi.fn(async (_tab: number, method: string) => {
|
||||
@@ -126,11 +276,11 @@ describe("captureViewModel", () => {
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let runtimeCalls = 0;
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
@@ -141,22 +291,23 @@ describe("captureViewModel", () => {
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
runtimeCalls += 1;
|
||||
return runtimeCalls === 1
|
||||
? {
|
||||
result: {
|
||||
value: [
|
||||
{
|
||||
triggerSel: ".menu",
|
||||
affectedSub: " .items",
|
||||
label: "Products",
|
||||
x: 50,
|
||||
y: 20,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
: { result: { value: ["Shoes", "Bags"] } };
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
if (expression.includes("querySelectorAll(selectors)")) {
|
||||
hoverStateCalls += 1;
|
||||
return hoverStateCalls === 1
|
||||
? { result: { value: [] } }
|
||||
: {
|
||||
result: {
|
||||
value: [
|
||||
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
@@ -166,12 +317,18 @@ describe("captureViewModel", () => {
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
{
|
||||
triggerBackendNodeId: 12,
|
||||
triggerPoint: { x: 956, y: 32 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
confidence: "high",
|
||||
},
|
||||
]);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
||||
expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
@@ -180,6 +337,91 @@ describe("captureViewModel", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses a fresh baseline for each hover candidate", async () => {
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
if (expression.includes("querySelectorAll(selectors)")) {
|
||||
hoverStateCalls += 1;
|
||||
const visibleMenu = [
|
||||
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||
];
|
||||
return hoverStateCalls === 1
|
||||
? { result: { value: [] } }
|
||||
: { result: { value: visibleMenu } };
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect.objectContaining({
|
||||
triggerBackendNodeId: 12,
|
||||
subItems: ["My profile", "Sign out"],
|
||||
}),
|
||||
]);
|
||||
expect(hoverStateCalls).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
|
||||
it("deduplicates nested hover candidates for one visual trigger", async () => {
|
||||
let hoverMoves = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
if (expression.includes("querySelectorAll(selectors)")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
const point = params as { x?: number; y?: number };
|
||||
if (point.x !== -10 && point.x !== 0) hoverMoves += 1;
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
@@ -45,9 +46,11 @@ export interface CapturedNode {
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerLabel: string;
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export interface CapturedViewModel {
|
||||
@@ -61,6 +64,7 @@ export interface CapturedViewModel {
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
@@ -225,11 +229,12 @@ interface RuntimeEvaluateReply {
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
triggerSel: string;
|
||||
affectedSub: string;
|
||||
label: string;
|
||||
backendNodeId: number;
|
||||
label?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
score: number;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
interface CdpDomNode {
|
||||
@@ -322,17 +327,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
const MAX_HOVER_TRIGGERS = 8;
|
||||
const HOVER_SETTLE_MS = 200;
|
||||
const MAX_HOVER_TRIGGERS = 6;
|
||||
const MAX_HOVER_SURFACES = 3;
|
||||
const HOVER_SETTLE_MS = 300;
|
||||
const MAX_HOVER_SUB_ITEMS = 12;
|
||||
|
||||
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
||||
return reply.result?.value as T | undefined;
|
||||
}
|
||||
|
||||
function hoverCssScanExpression(): string {
|
||||
function hoverCssTriggerScanExpression(): string {
|
||||
return `(() => {
|
||||
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
||||
const pairs = [];
|
||||
const centres = [];
|
||||
const seenRules = new Set();
|
||||
for (const sheet of Array.from(document.styleSheets)) {
|
||||
let rules;
|
||||
@@ -348,90 +355,73 @@ function hoverCssScanExpression(): string {
|
||||
const hoverIndex = part.indexOf(":hover");
|
||||
if (hoverIndex < 0) continue;
|
||||
const triggerSel = part.slice(0, hoverIndex).trim();
|
||||
const affectedSub = part.slice(hoverIndex + 6).trim();
|
||||
if (!triggerSel) continue;
|
||||
const key = triggerSel + "||" + affectedSub;
|
||||
if (seenRules.has(key)) continue;
|
||||
seenRules.add(key);
|
||||
pairs.push({ triggerSel, affectedSub });
|
||||
if (seenRules.has(triggerSel)) continue;
|
||||
seenRules.add(triggerSel);
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof HTMLElement)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 });
|
||||
if (centres.length >= 24) return centres;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = [];
|
||||
const seenLabels = new Set();
|
||||
const isUnsafeTrigger = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return true;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||
if (el.isContentEditable) return true;
|
||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
||||
return false;
|
||||
};
|
||||
for (const pair of pairs) {
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof Element)) continue;
|
||||
if (isUnsafeTrigger(el)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
if (!label || seenLabels.has(label)) continue;
|
||||
seenLabels.add(label);
|
||||
candidates.push({
|
||||
triggerSel: pair.triggerSel,
|
||||
affectedSub: pair.affectedSub,
|
||||
label,
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
});
|
||||
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
return centres;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function hoverCollectExpression(candidate: HoverCandidate): string {
|
||||
return `(() => {
|
||||
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
|
||||
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
|
||||
const x = ${JSON.stringify(candidate.x)};
|
||||
const y = ${JSON.stringify(candidate.y)};
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
|
||||
if (!trigger) return [];
|
||||
interface HoverRuntimeItem {
|
||||
text: string;
|
||||
role: string;
|
||||
tag: string;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
|
||||
let targets = [];
|
||||
if (!affectedSub) {
|
||||
targets = [trigger];
|
||||
} else {
|
||||
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
|
||||
}
|
||||
function hoverStateExpression(): string {
|
||||
return `(() => {
|
||||
const items = [];
|
||||
const seen = new Set();
|
||||
const push = (value) => {
|
||||
const text = String(value || "").replace(/\\s+/g, " ").trim();
|
||||
const selectors = [
|
||||
"a",
|
||||
"button",
|
||||
"[role='menuitem']",
|
||||
"[role='menuitemcheckbox']",
|
||||
"[role='menuitemradio']",
|
||||
"[role='option']",
|
||||
"[role='tab']",
|
||||
"[role='link']",
|
||||
"[role='button']"
|
||||
].join(",");
|
||||
const push = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return;
|
||||
const style = getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width <= 0 || rect.height <= 0) return;
|
||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
|
||||
const text = String(
|
||||
el.getAttribute("aria-label") ||
|
||||
el.getAttribute("title") ||
|
||||
el.textContent ||
|
||||
""
|
||||
).replace(/\\s+/g, " ").trim();
|
||||
if (!text || seen.has(text)) return;
|
||||
seen.add(text);
|
||||
items.push(text);
|
||||
items.push({
|
||||
text,
|
||||
role: (el.getAttribute("role") || "").toLowerCase(),
|
||||
tag: el.tagName.toLowerCase(),
|
||||
x: rect.left,
|
||||
y: rect.top,
|
||||
});
|
||||
};
|
||||
for (const target of targets) {
|
||||
if (!(target instanceof Element)) continue;
|
||||
const style = getComputedStyle(target);
|
||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
||||
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
|
||||
if (clickable.length > 0) {
|
||||
for (const child of Array.from(clickable)) push(child.textContent);
|
||||
} else {
|
||||
push(target.textContent);
|
||||
}
|
||||
}
|
||||
return items.slice(0, 12);
|
||||
for (const el of Array.from(document.querySelectorAll(selectors))) push(el);
|
||||
return items.slice(0, 400);
|
||||
})()`;
|
||||
}
|
||||
|
||||
@@ -453,45 +443,217 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
);
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
||||
function capturedText(node: CapturedNode): string | undefined {
|
||||
const value =
|
||||
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||
const clean = value.replace(/\s+/g, " ").trim();
|
||||
return clean || undefined;
|
||||
}
|
||||
|
||||
function hasGraphicDescendant(
|
||||
node: CapturedNode,
|
||||
childrenByParentId: Map<number, CapturedNode[]>,
|
||||
depth = 0,
|
||||
): boolean {
|
||||
if (depth > 3) return false;
|
||||
for (const child of childrenByParentId.get(node.backendNodeId) ?? []) {
|
||||
const tag = child.tag.toLowerCase();
|
||||
if (["img", "svg", "use", "path", "i"].includes(tag)) return true;
|
||||
if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function roleOf(node: CapturedNode): string {
|
||||
return (node.attrs.role ?? "").toLowerCase();
|
||||
}
|
||||
|
||||
function scoreHoverCandidate(
|
||||
node: CapturedNode,
|
||||
childrenByParentId: Map<number, CapturedNode[]>,
|
||||
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||
): HoverCandidate | null {
|
||||
const rect = node.rect;
|
||||
if (!rect) return null;
|
||||
const label = capturedText(node);
|
||||
const cssHoverMatch = cssHoverPoints.some(
|
||||
(point) =>
|
||||
point.x >= rect.x &&
|
||||
point.x <= rect.x + rect.w &&
|
||||
point.y >= rect.y &&
|
||||
point.y <= rect.y + rect.h,
|
||||
);
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: node.tag,
|
||||
role: roleOf(node),
|
||||
label,
|
||||
attrs: node.attrs,
|
||||
rect,
|
||||
cursor: node.cursor,
|
||||
pointerEvents: node.pointerEvents,
|
||||
hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId),
|
||||
cssHoverMatch,
|
||||
});
|
||||
|
||||
if (!decision.eligible) return null;
|
||||
return {
|
||||
backendNodeId: node.backendNodeId,
|
||||
label,
|
||||
x: rect.x + rect.w / 2,
|
||||
y: rect.y + rect.h / 2,
|
||||
score: decision.score,
|
||||
reasons: decision.reasons,
|
||||
};
|
||||
}
|
||||
|
||||
function buildHoverCandidates(
|
||||
nodes: CapturedNode[],
|
||||
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||
): HoverCandidate[] {
|
||||
const childrenByParentId = new Map<number, CapturedNode[]>();
|
||||
const parentByBackendId = new Map<number, number | null>();
|
||||
for (const node of nodes) {
|
||||
parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId);
|
||||
if (node.parentBackendNodeId === null) continue;
|
||||
const children = childrenByParentId.get(node.parentBackendNodeId) ?? [];
|
||||
children.push(node);
|
||||
childrenByParentId.set(node.parentBackendNodeId, children);
|
||||
}
|
||||
|
||||
const candidates = nodes
|
||||
.map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints))
|
||||
.filter((candidate): candidate is HoverCandidate => candidate !== null)
|
||||
.sort((a, b) => b.score - a.score);
|
||||
|
||||
const deduped: HoverCandidate[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const candidate of candidates) {
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) {
|
||||
continue;
|
||||
}
|
||||
seen.add(candidate.backendNodeId);
|
||||
deduped.push(candidate);
|
||||
if (deduped.length >= MAX_HOVER_TRIGGERS) break;
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function sameHoverCluster(
|
||||
a: HoverCandidate,
|
||||
b: HoverCandidate,
|
||||
parentByBackendId: Map<number, number | null>,
|
||||
): boolean {
|
||||
if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true;
|
||||
return (
|
||||
isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) ||
|
||||
isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId)
|
||||
);
|
||||
}
|
||||
|
||||
function isBackendAncestor(
|
||||
ancestorId: number,
|
||||
nodeId: number,
|
||||
parentByBackendId: Map<number, number | null>,
|
||||
): boolean {
|
||||
let parentId = parentByBackendId.get(nodeId);
|
||||
let guard = 0;
|
||||
while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) {
|
||||
if (parentId === ancestorId) return true;
|
||||
parentId = parentByBackendId.get(parentId);
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] {
|
||||
const beforeKeys = new Set(before.map((item) => item.text.toLowerCase()));
|
||||
const out: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of after) {
|
||||
const text = item.text.replace(/\s+/g, " ").trim();
|
||||
const key = text.toLowerCase();
|
||||
if (!text || beforeKeys.has(key) || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(text);
|
||||
if (out.length >= MAX_HOVER_SUB_ITEMS) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function confidenceForHover(
|
||||
candidate: HoverCandidate,
|
||||
subItems: string[],
|
||||
): "high" | "medium" | "low" {
|
||||
if (subItems.length >= 2 && candidate.score >= 80) return "high";
|
||||
if (subItems.length >= 2 || candidate.score >= 80) return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
nodes: CapturedNode[],
|
||||
options: CaptureViewModelOptions,
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssScanExpression(),
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssTriggerScanExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
|
||||
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
|
||||
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (!candidate.label || seen.has(candidate.label)) continue;
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCollectExpression(candidate),
|
||||
returnByValue: true,
|
||||
});
|
||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.label);
|
||||
results.push({
|
||||
triggerLabel: candidate.label,
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
const seen = new Set<number>();
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
@@ -797,7 +959,9 @@ export async function captureViewModel(
|
||||
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
@@ -314,8 +314,21 @@ export interface SnapshotResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export type ObserveParams = SnapshotParams;
|
||||
export type ObserveResult = SnapshotResult;
|
||||
export interface ObserveParams extends SnapshotParams {
|
||||
debug_surfaces?: boolean;
|
||||
}
|
||||
|
||||
export interface ObserveResult extends SnapshotResult {
|
||||
debug?: {
|
||||
surface_probes?: Array<{
|
||||
trigger_backend_node_id: number;
|
||||
trigger_point?: { x: number; y: number };
|
||||
trigger_action: string;
|
||||
sub_items: string[];
|
||||
confidence?: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface GetHtmlParams {
|
||||
session_id: string;
|
||||
@@ -410,6 +423,25 @@ export interface ClickResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface HoverParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
modifiers?: KeyModifier[];
|
||||
settle_ms?: number;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface HoverResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface FillParams {
|
||||
session_id: string;
|
||||
value: string;
|
||||
@@ -668,6 +700,11 @@ export type DraftTraceStep =
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "hover";
|
||||
target: TargetDescriptor;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
target: TargetDescriptor;
|
||||
@@ -701,6 +738,7 @@ export type DraftTraceStep =
|
||||
export type Step =
|
||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "fill" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
|
||||
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk snapshot --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
@@ -10,8 +10,8 @@ use std::time::Duration;
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
|
||||
PressResult, SelectParams, SelectResult,
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct HoverArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||
pub target: Option<String>,
|
||||
|
||||
/// Force-treat the value as a snapshot ref (overrides positional detection).
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
/// Force-treat the value as a CSS selector.
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
|
||||
#[arg(long, default_value = "")]
|
||||
pub modifiers: String,
|
||||
|
||||
/// Wait after moving the mouse so hover-triggered UI can settle.
|
||||
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
|
||||
pub settle: u32,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(
|
||||
args.target.clone(),
|
||||
args.ref_.clone(),
|
||||
args.selector.clone(),
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let params = HoverParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
modifiers: if modifiers.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(modifiers)
|
||||
},
|
||||
settle_ms: Some(args.settle),
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: HoverResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolHover,
|
||||
params,
|
||||
"hover-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target =
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||
println!(
|
||||
"hover ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -164,6 +164,9 @@ pub enum Command {
|
||||
/// Click a snapshot ref or CSS selector.
|
||||
Click(ClickArgs),
|
||||
|
||||
/// Hover a snapshot ref or CSS selector.
|
||||
Hover(HoverArgs),
|
||||
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
|
||||
@@ -29,6 +29,10 @@ pub struct ObserveArgs {
|
||||
/// Soft cap on rendered tokens (~4 chars/token).
|
||||
#[arg(long = "max-tokens")]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// Include conditional surface probe diagnostics in JSON output.
|
||||
#[arg(long = "debug-surfaces")]
|
||||
pub debug_surfaces: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -42,6 +46,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
||||
tab_id: args.tab_id,
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
debug_surfaces: args.debug_surfaces,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
|
||||
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolNavigateForward
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
|
||||
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
|
||||
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
|
||||
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"hover",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--settle",
|
||||
"300ms",
|
||||
]);
|
||||
let Command::Hover(args) = cli.command else {
|
||||
panic!("expected hover command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("@e1"));
|
||||
assert_eq!(args.settle, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_click_count() {
|
||||
assert!(
|
||||
|
||||
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
tab_id: 13,
|
||||
truncated: false,
|
||||
dialogs: vec![],
|
||||
debug: None,
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -492,6 +493,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
tab_id: None,
|
||||
max_depth: None,
|
||||
max_tokens: None,
|
||||
debug_surfaces: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "HoverParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"ref": {
|
||||
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"settle_ms": {
|
||||
"description": "Wait after the mouse move so hover-triggered UI can settle.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"description": "Target tab. Defaults to the Agent Window's active tab.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "HoverResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tab_id",
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"x": {
|
||||
"description": "Viewport-relative hover coordinates (CSS pixels).",
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"y": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,11 @@
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"debug_surfaces": {
|
||||
"description": "Include implementation diagnostics for conditional surface probes.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"max_depth": {
|
||||
"description": "Cap the depth of the rendered VOM tree.",
|
||||
"type": [
|
||||
|
||||
@@ -8,6 +8,16 @@
|
||||
"text"
|
||||
],
|
||||
"properties": {
|
||||
"debug": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ObserveDebug"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -103,6 +113,73 @@
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"ObserveDebug": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"surface_probes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SurfaceProbeDebug"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"SurfaceProbeDebug": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"sub_items",
|
||||
"trigger_action",
|
||||
"trigger_backend_node_id"
|
||||
],
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sub_items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"trigger_action": {
|
||||
"type": "string"
|
||||
},
|
||||
"trigger_backend_node_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"trigger_point": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/SurfaceProbePoint"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"SurfaceProbePoint": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"x": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"y": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +144,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -144,6 +144,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -177,6 +177,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -83,6 +83,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -77,6 +77,8 @@ fn main() {
|
||||
|
||||
dump!(ClickParams, "tool_click_params");
|
||||
dump!(ClickResult, "tool_click_result");
|
||||
dump!(HoverParams, "tool_hover_params");
|
||||
dump!(HoverResult, "tool_hover_result");
|
||||
dump!(FillParams, "tool_fill_params");
|
||||
dump!(FillResult, "tool_fill_result");
|
||||
dump!(PressParams, "tool_press_params");
|
||||
|
||||
@@ -70,6 +70,8 @@ pub enum Method {
|
||||
ToolReload,
|
||||
#[serde(rename = "tool.click")]
|
||||
ToolClick,
|
||||
#[serde(rename = "tool.hover")]
|
||||
ToolHover,
|
||||
#[serde(rename = "tool.fill")]
|
||||
ToolFill,
|
||||
#[serde(rename = "tool.press")]
|
||||
@@ -161,7 +163,7 @@ impl Method {
|
||||
|
||||
// Transient input — no committed browser action, but still page
|
||||
// input. It must be stopped by pending user interrupts.
|
||||
Method::ToolObserve => MethodEffect::TransientInput,
|
||||
Method::ToolHover | Method::ToolObserve => MethodEffect::TransientInput,
|
||||
|
||||
// Passive reads — transparent.
|
||||
// `record_stop` / `record_await` observe / finish a recording
|
||||
@@ -259,6 +261,7 @@ mod tests {
|
||||
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
|
||||
assert!(!Method::ToolTabList.is_mutating());
|
||||
assert!(!Method::ToolSnapshot.is_mutating());
|
||||
assert!(!Method::ToolHover.is_mutating());
|
||||
assert!(!Method::ToolObserve.is_mutating());
|
||||
assert!(!Method::ToolGetHtml.is_mutating());
|
||||
assert!(!Method::ToolScreenshot.is_mutating());
|
||||
@@ -321,6 +324,7 @@ mod tests {
|
||||
#[test]
|
||||
fn effect_classifies_observe_as_transient_input() {
|
||||
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
|
||||
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
|
||||
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
|
||||
@@ -329,6 +333,7 @@ mod tests {
|
||||
#[test]
|
||||
fn interrupt_gate_includes_transient_input() {
|
||||
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
|
||||
assert!(Method::ToolHover.requires_interrupt_gate());
|
||||
assert!(Method::ToolObserve.requires_interrupt_gate());
|
||||
assert!(Method::ToolClick.requires_interrupt_gate());
|
||||
assert!(!Method::Cancel.requires_interrupt_gate());
|
||||
|
||||
@@ -85,6 +85,52 @@ pub struct ClickResult {
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverParams {
|
||||
pub session_id: String,
|
||||
/// Optional `@e<N>` ref allocated by the last observation.
|
||||
/// Mutually exclusive with `selector`.
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// Target tab. Defaults to the Agent Window's active tab.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub modifiers: Option<Vec<KeyModifier>>,
|
||||
/// Wait after the mouse move so hover-triggered UI can settle.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 0))]
|
||||
pub settle_ms: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
/// Viewport-relative hover coordinates (CSS pixels).
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// fill
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -261,6 +307,24 @@ mod tests {
|
||||
assert_eq!(p.ref_.as_deref(), Some("e1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hover_params_serialise_ref_field_name() {
|
||||
let p = HoverParams {
|
||||
session_id: "abcd".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
tab_id: Some(42),
|
||||
modifiers: Some(vec![KeyModifier::Shift]),
|
||||
settle_ms: Some(200),
|
||||
timeout_ms: Some(5_000),
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||
assert!(v.get("ref_").is_none());
|
||||
let round: HoverParams = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn modifiers_render_as_lowercase_strings() {
|
||||
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
|
||||
|
||||
@@ -74,6 +74,32 @@ pub struct ObserveParams {
|
||||
/// best-effort heuristic based on character count).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
/// Include implementation diagnostics for conditional surface probes.
|
||||
#[serde(default)]
|
||||
pub debug_surfaces: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SurfaceProbePoint {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SurfaceProbeDebug {
|
||||
pub trigger_backend_node_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub trigger_point: Option<SurfaceProbePoint>,
|
||||
pub trigger_action: String,
|
||||
pub sub_items: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub confidence: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveDebug {
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub surface_probes: Vec<SurfaceProbeDebug>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -92,6 +118,8 @@ pub struct ObserveResult {
|
||||
pub truncated: bool,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub debug: Option<ObserveDebug>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -280,6 +308,7 @@ mod tests {
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1));
|
||||
|
||||
@@ -94,6 +94,11 @@ pub enum Step {
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Hover {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
|
||||
@@ -404,7 +404,7 @@ describe("renderVom single-layer page", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [hover: Shoes | Bags | Accessories]');
|
||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export { renderVom } from "./render";
|
||||
export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render";
|
||||
export type {
|
||||
ActiveScopeBlock,
|
||||
BlockingLayer,
|
||||
|
||||
+13
-11
@@ -197,7 +197,7 @@ function shouldRender(node: VomNode): boolean {
|
||||
return cleaned(node.name) !== undefined;
|
||||
}
|
||||
|
||||
function shouldReference(node: VomNode): boolean {
|
||||
export function isVomReferenceNode(node: VomNode): boolean {
|
||||
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
|
||||
}
|
||||
|
||||
@@ -342,7 +342,7 @@ function isAncestorOf(
|
||||
return false;
|
||||
}
|
||||
|
||||
function applyRecovery(nodes: VomNode[]): VomNode[] {
|
||||
export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] {
|
||||
const parentMap = buildParentMap(nodes);
|
||||
const children = buildChildren(nodes);
|
||||
const recoveredIds = new Set<number>();
|
||||
@@ -350,7 +350,7 @@ function applyRecovery(nodes: VomNode[]): VomNode[] {
|
||||
const recoveredNameById = new Map<number, string>();
|
||||
|
||||
for (const node of nodes) {
|
||||
if (shouldReference(node)) continue;
|
||||
if (isVomReferenceNode(node)) continue;
|
||||
const role = recoveredRole(node, children);
|
||||
if (!role || !passesRecoveryGuards(node, children)) continue;
|
||||
recoveredIds.add(node.id);
|
||||
@@ -397,7 +397,7 @@ function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
|
||||
function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
|
||||
const counts = new Map<string, number>();
|
||||
for (const node of nodes) {
|
||||
if (!shouldReference(node)) continue;
|
||||
if (!isVomReferenceNode(node)) continue;
|
||||
const name = cleaned(node.name);
|
||||
if (!name) continue;
|
||||
const key = normalizeContextKey(name);
|
||||
@@ -427,7 +427,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined {
|
||||
}
|
||||
|
||||
function weakLabelContext(node: VomNode, targetName: string): string | undefined {
|
||||
if (shouldReference(node)) return undefined;
|
||||
if (isVomReferenceNode(node)) return undefined;
|
||||
return weakContextText(
|
||||
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
|
||||
targetName,
|
||||
@@ -509,7 +509,7 @@ function collectWeakLabelsFromSubtree(
|
||||
state: RenderState,
|
||||
labels: string[],
|
||||
): void {
|
||||
if (shouldReference(node)) return;
|
||||
if (isVomReferenceNode(node)) return;
|
||||
const label = weakLabelContext(node, nodeName);
|
||||
if (label) {
|
||||
labels.push(label);
|
||||
@@ -637,7 +637,9 @@ function renderNodeLine(
|
||||
if (surface && surface.subItems.length > 0) {
|
||||
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
|
||||
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
|
||||
line += ` [${surface.triggerAction}: ${items}${suffix}]`;
|
||||
const action =
|
||||
surface.triggerAction === "hover" ? "hover first" : `${surface.triggerAction} first`;
|
||||
line += ` [${action}: ${items}${suffix}]`;
|
||||
}
|
||||
|
||||
return line;
|
||||
@@ -664,7 +666,7 @@ function descendantTextCoveredByName(node: VomNode, state: RenderState): boolean
|
||||
const texts: string[] = [];
|
||||
while (stack.length > 0) {
|
||||
const child = stack.pop() as VomNode;
|
||||
if (shouldReference(child)) return false;
|
||||
if (isVomReferenceNode(child)) return false;
|
||||
const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value);
|
||||
if (text) texts.push(text);
|
||||
stack.push(...(state.children.get(child.id) ?? []));
|
||||
@@ -748,7 +750,7 @@ function renderTree(
|
||||
continue;
|
||||
}
|
||||
|
||||
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined;
|
||||
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||
const context = ref ? handleContext(node, state) : [];
|
||||
const surface = state.surfaceMap.get(node.id);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
||||
@@ -913,7 +915,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
||||
|
||||
const blockedRoots = new Set<number>();
|
||||
for (const target of nodes) {
|
||||
if (!shouldReference(target)) continue;
|
||||
if (!isVomReferenceNode(target)) continue;
|
||||
const blocked = candidates.some((candidate) =>
|
||||
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
|
||||
);
|
||||
@@ -968,7 +970,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
}
|
||||
|
||||
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
|
||||
const nodes = applyRecovery(scene.nodes);
|
||||
const nodes = applyVomInteractionRecovery(scene.nodes);
|
||||
const renderScene = { ...scene, nodes };
|
||||
const layer = detectBlockingLayer(nodes, scene.viewport);
|
||||
if (layer) return renderDoubleLayer(renderScene, layer, options);
|
||||
|
||||
+11
-7
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk snapshot --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
Reference in New Issue
Block a user