Compare commits

...
45 changed files with 4906 additions and 254 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,109 @@
import { describe, expect, it, vi } from "vitest";
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
function mockRect(
el: Element,
rect: { left: number; top: number; width: number; height: number },
): void {
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
x: rect.left,
y: rect.top,
top: rect.top,
left: rect.left,
right: rect.left + rect.width,
bottom: rect.top + rect.height,
width: rect.width,
height: rect.height,
toJSON: () => ({}),
});
}
describe("record-hover-surface", () => {
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
document.body.innerHTML = `
<ul class="header-menu tw-flex navbar-app__authentication-menu">
<li class="tg-popover">
<div class="tg-avatar tg-avatar--img"></div>
</li>
</ul>
<div class="tg-popover__popper">
<div class="tg-popover__content">
<div class="tg-popover__content-body">
<li class="header-menu__item">
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
</li>
<li class="header-menu__item">
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
</li>
</div>
</div>
</div>
`;
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
false,
);
expect(
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
).toBe(false);
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
true,
);
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
true,
);
expect(
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
).toBe(true);
});
it("does not treat elements covered by an open surface as the surface owner", () => {
document.body.innerHTML = `
<button class="avatar">image</button>
<a class="fork" href="/fork">Fork</a>
<button class="clone">Clone & Download</button>
<div class="tg-popover__content-body"></div>
`;
const avatar = document.querySelector(".avatar")!;
const fork = document.querySelector(".fork")!;
const clone = document.querySelector(".clone")!;
const surface = document.querySelector(".tg-popover__content-body")!;
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
});
it("does not treat fixed navigation containers as plain floating surfaces", () => {
document.body.innerHTML = `
<header class="navbar-app">
<a href="/fork">Fork</a>
<button>image</button>
</header>
<div class="plain-floating"></div>
`;
const navbar = document.querySelector(".navbar-app")!;
const floating = document.querySelector(".plain-floating")!;
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
const style = {
cursor: "",
display: "block",
pointerEvents: "auto",
position: el === navbar ? "fixed" : "absolute",
visibility: "visible",
} as CSSStyleDeclaration;
return style;
});
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
});
});
+489 -1
View File
@@ -1,4 +1,17 @@
import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
import {
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
type HoverTriggerDecision,
type HoverTriggerRect,
hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy";
import {
isRecordCancelMessage,
isRecordStartMessage,
@@ -15,6 +28,14 @@ import {
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import {
closestHoverSurfaceCandidate,
collectHoverSurfaceStates,
decideHoverSurfaceRelation,
type HoverSurfaceState,
isHoverSurfaceCandidateElement,
isLikelyHoverSurfaceOwner,
} from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
@@ -49,8 +70,29 @@ interface FillSession {
lastValue: string;
}
interface HoverCandidate {
element: Element;
target: TargetDescriptor;
recordedAt: number;
score: number;
eligible: boolean;
}
interface HoverSurfaceNode {
element: Element;
signature: string;
owner: HoverCandidate;
parent?: HoverSurfaceNode;
}
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
const HOVER_REPLACE_SCORE_MARGIN = 50;
const HOVER_CANDIDATE_LIMIT = 24;
const HOVER_TRIGGER_LABEL_MAX = 48;
function eventTarget(event: Event): EventTarget | null {
return event.composedPath()[0] ?? event.target;
}
@@ -122,6 +164,173 @@ function fillableValue(el: FillableElement): string {
return el.textContent ?? "";
}
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
return {
id: el.id || undefined,
class: typeof el.className === "string" ? el.className || undefined : undefined,
"data-testid": el.getAttribute("data-testid") ?? undefined,
"data-test": el.getAttribute("data-test") ?? undefined,
"data-cy": el.getAttribute("data-cy") ?? undefined,
"aria-label": el.getAttribute("aria-label") ?? undefined,
"aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
"aria-controls": el.getAttribute("aria-controls") ?? undefined,
"aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
"aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
"aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
contenteditable: el.getAttribute("contenteditable") ?? undefined,
disabled: el.hasAttribute("disabled") ? "" : undefined,
hidden: el.hasAttribute("hidden") ? "" : undefined,
inert: el.hasAttribute("inert") ? "" : undefined,
onclick: el.getAttribute("onclick") ?? undefined,
onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
onmouseover: el.getAttribute("onmouseover") ?? undefined,
role: el.getAttribute("role") ?? undefined,
tabindex: el.getAttribute("tabindex") ?? undefined,
title: el.getAttribute("title") ?? undefined,
};
}
function hoverTriggerRect(el: Element): HoverTriggerRect {
const rect = el.getBoundingClientRect();
return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
}
function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
if (!(el instanceof HTMLElement)) return {};
const style = getComputedStyle(el);
return { cursor: style.cursor, pointerEvents: style.pointerEvents };
}
function hasHoverGraphicDescendant(el: Element): boolean {
return el.querySelector("img,svg,use,path,i") !== null;
}
function normalizeLabelText(value: string): string {
return value.replace(/\s+/g, " ").trim();
}
function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string {
const normalized = normalizeLabelText(value);
if (normalized.length <= max) return normalized;
return `${normalized.slice(0, max - 1)}…`;
}
function collectHoverTriggerLabelText(root: Element): string {
let text = "";
const visit = (node: Node) => {
if (node.nodeType === Node.TEXT_NODE) {
text += ` ${node.textContent ?? ""}`;
return;
}
if (!(node instanceof Element)) return;
if (node !== root && isHoverSurfaceCandidateElement(node)) {
return;
}
for (const child of node.childNodes) visit(child);
};
for (const child of root.childNodes) visit(child);
return normalizeLabelText(text);
}
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
if (!desc.name) return desc;
const fullText = normalizeLabelText(el.textContent ?? "");
const compactName = collectHoverTriggerLabelText(el);
if (!compactName || compactName === desc.name) return desc;
const compactComparable = compactName.replace(/\s+/g, "");
const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, "");
const fullTextComparable = fullText.replace(/\s+/g, "");
if (
descComparable.startsWith(compactComparable) &&
fullTextComparable.startsWith(descComparable) &&
compactName.length < desc.name.length
) {
return { ...desc, name: truncateLabel(compactName) };
}
return desc;
}
function isWeakHoverTarget(target: TargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
function looksLikeAvatarElement(el: Element): boolean {
const className = typeof el.className === "string" ? el.className : "";
return /\b(avatar|user-avatar)\b/i.test(className);
}
function normalizeHoverTarget(
el: Element,
desc: TargetDescriptor,
decision: HoverTriggerDecision,
): TargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
if (
isWeakHoverTarget(desc) &&
(looksLikeAvatarElement(el) ||
(hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
) {
return { tag: "img", role: "img", name: "image" };
}
return desc;
}
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return {
tag: el.tagName.toLowerCase(),
role: desc.role,
label: desc.name,
attrs: hoverTriggerAttrs(el),
rect: hoverTriggerRect(el),
cursor: style.cursor,
pointerEvents: style.pointerEvents,
hasGraphicDescendant: hasHoverGraphicDescendant(el),
};
}
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
if (!(target instanceof Element)) return null;
const element = resolveHoverElement(target);
if (!element) return null;
const desc = describeTarget(element);
const signals = hoverTriggerSignals(element, desc);
if (!signals) return null;
const decision = evaluateHoverTrigger(signals);
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
if (!decision.eligible && !desc.name && !desc.role) return null;
const normalizedTarget = normalizeHoverTarget(
element,
compactHoverTargetName(element, desc),
decision,
);
const now = Date.now();
return {
element,
target: normalizedTarget,
recordedAt: now,
score: decision.score,
eligible: decision.eligible,
};
}
function shouldReplaceHoverCandidate(
current: HoverCandidate,
next: HoverCandidate,
now: number,
): boolean {
if (next.element === current.element) return false;
if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
if (current.element.contains(next.element)) return false;
if (next.element.contains(current.element)) return true;
return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
}
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
if (!session || !(target instanceof Element)) return false;
@@ -168,10 +377,17 @@ export function startRecordCapture(
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
};
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
new Map(states.map((state) => [state.element, state.signature]));
let fillSession: FillSession | null = null;
let composing = false;
let lastUrl = location.href;
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
let pendingHover: HoverCandidate | null = null;
const recentHoverCandidates: HoverCandidate[] = [];
const emittedHoverElements = new WeakSet<Element>();
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
let generatedControlClick: Element | null = null;
let navigationActionPending = false;
let navigationActionVersion = 0;
@@ -237,6 +453,223 @@ export function startRecordCapture(
});
};
const rememberHoverCandidate = (hover: HoverCandidate) => {
const duplicate = recentHoverCandidates.findIndex(
(candidate) => candidate.element === hover.element,
);
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
recentHoverCandidates.push(hover);
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
}
};
const surfaceArea = (el: Element): number => {
const rect = el.getBoundingClientRect();
return rect.width * rect.height;
};
const smallestContaining = <T>(
target: Element,
items: Iterable<T>,
elementFor: (item: T) => Element,
): T | undefined => {
let best: T | undefined;
for (const item of items) {
const element = elementFor(item);
if (!element.contains(target)) continue;
if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) {
best = item;
}
}
return best;
};
const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined =>
smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element);
const surfaceStateContaining = (
target: Element,
states: HoverSurfaceState[],
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
const inferOwnerForSurface = (
surface: Element,
now: number,
before?: HoverCandidate,
): HoverCandidate | undefined => {
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
const candidate = recentHoverCandidates[index];
if (!candidate) continue;
if (before && candidate.element === before.element) continue;
if (before && candidate.recordedAt > before.recordedAt) continue;
if (surface.contains(candidate.element)) continue;
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
}
return undefined;
};
const nodeForUnownedSurface = (
surfaceState: HoverSurfaceState,
currentStates: HoverSurfaceState[],
now: number,
): HoverSurfaceNode | undefined => {
const owner = inferOwnerForSurface(surfaceState.element, now);
if (!owner) return undefined;
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
const node: HoverSurfaceNode = {
element: surfaceState.element,
signature: surfaceState.signature,
owner,
...(parent ? { parent } : {}),
};
hoverSurfaceNodes.set(surfaceState.element, node);
return node;
};
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
const closestSurface = closestHoverSurfaceCandidate(target);
if (closestSurface) {
const owned = hoverSurfaceNodes.get(closestSurface);
if (owned) return owned;
}
const ownedContaining = ownedSurfaceContaining(target);
if (ownedContaining) return ownedContaining;
const now = Date.now();
const currentStates = collectHoverSurfaceStates();
pruneGoneHoverSurfaces(currentStates);
const surfaceState = closestSurface
? currentStates.find((state) => state.element === closestSurface)
: surfaceStateContaining(target, currentStates);
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
if (!surfaceState) return undefined;
return nodeForUnownedSurface(surfaceState, currentStates, now);
};
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
const current = new Set(currentStates.map((state) => state.element));
for (const element of hoverSurfaceNodes.keys()) {
if (!current.has(element)) hoverSurfaceNodes.delete(element);
}
};
const parentSurfaceNodeForOwner = (
owner: HoverCandidate,
currentStates: HoverSurfaceState[],
now: number,
): HoverSurfaceNode | undefined => {
const ownedParent = ownedSurfaceContaining(owner.element);
if (ownedParent) return ownedParent;
const parentState = surfaceStateContaining(owner.element, currentStates);
if (!parentState) return undefined;
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
if (!parentOwner) return undefined;
const parentNode: HoverSurfaceNode = {
element: parentState.element,
signature: parentState.signature,
owner: parentOwner,
};
hoverSurfaceNodes.set(parentState.element, parentNode);
return parentNode;
};
const createSurfaceNode = (
state: HoverSurfaceState,
owner: HoverCandidate,
currentStates: HoverSurfaceState[],
now: number,
): HoverSurfaceNode | undefined => {
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
if (parent?.element === state.element) return undefined;
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
const node: HoverSurfaceNode = {
element: state.element,
signature: state.signature,
owner,
...(parent ? { parent } : {}),
};
hoverSurfaceNodes.set(state.element, node);
return node;
};
const bindChangedHoverSurfaces = (
previousStates: Map<Element, string>,
currentStates: HoverSurfaceState[],
now: number,
) => {
for (const state of currentStates) {
if (previousStates.get(state.element) === state.signature) continue;
const owner = inferOwnerForSurface(state.element, now);
if (!owner) continue;
createSurfaceNode(state, owner, currentStates, now);
}
};
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
const now = Date.now();
const currentStates = collectHoverSurfaceStates();
pruneGoneHoverSurfaces(currentStates);
bindChangedHoverSurfaces(previousStates, currentStates, now);
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
};
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
};
const emitHoverStep = (hover: HoverCandidate) => {
if (emittedHoverElements.has(hover.element)) return;
emitStep({
op: "hover",
target: hover.target,
});
emittedHoverElements.add(hover.element);
};
const surfaceOwnerChain = (
surface: HoverSurfaceNode,
actionElement: Element,
now: number,
): HoverCandidate[] => {
const chain: HoverCandidate[] = [];
const selected = new WeakSet<Element>();
const surfaces: HoverSurfaceNode[] = [];
let node: HoverSurfaceNode | undefined = surface;
while (node && surfaces.length < 4) {
surfaces.unshift(node);
node = node.parent;
}
for (const owned of surfaces) {
const owner = owned.owner;
if (owner.element === actionElement || actionElement.contains(owner.element)) {
continue;
}
if (selected.has(owner.element)) continue;
if (emittedHoverElements.has(owner.element)) continue;
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
selected.add(owner.element);
chain.push(owner);
}
return chain;
};
const containedHoverOwnerForAction = (
actionElement: Element,
now: number,
): HoverCandidate | undefined => {
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
const candidate = recentHoverCandidates[index];
if (!candidate) continue;
if (candidate.element === actionElement) continue;
if (!candidate.element.contains(actionElement)) continue;
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
return candidate;
}
return undefined;
};
const emitClick = (event: MouseEvent) => {
// Only record clicks an LLM can re-identify (named interactive controls).
const target = describeEventTarget(eventTarget(event));
@@ -249,6 +682,54 @@ export function startRecordCapture(
});
};
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
if (!(actionTarget instanceof Element)) return;
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
const surface = ownedSurfaceForAction(actionElement);
const now = Date.now();
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
const hoverChain = surface
? surfaceOwnerChain(surface, actionElement, now)
: containedOwner
? [containedOwner]
: [];
if (hoverChain.length === 0) {
pendingHover = null;
return;
}
for (const hover of hoverChain) emitHoverStep(hover);
pendingHover = null;
};
const onMouseOver = (event: MouseEvent) => {
const target = eventTarget(event);
if (isOverlayTarget(target)) return;
const now = Date.now();
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
pendingHover = null;
}
const candidate = hoverCandidateFromEvent(target);
if (candidate) {
const previousSurfaceStates = new Map(hoverSurfaceStates);
rememberHoverCandidate(candidate);
refreshHoverSurfaces(previousSurfaceStates);
scheduleHoverSurfaceRefresh(previousSurfaceStates);
}
const hover = candidate?.eligible ? candidate : null;
if (pendingHover && target instanceof Element) {
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
return;
}
}
if (!hover) return;
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
return;
}
pendingHover = hover;
};
const onClick = (event: MouseEvent) => {
if (event.button !== 0) return;
const target = eventTarget(event);
@@ -278,6 +759,7 @@ export function startRecordCapture(
const fillable = fillableFromTarget(target);
if (fillable) {
emitHoverCandidateBeforeAction(fillable);
ensureFillSession(fillable);
return;
}
@@ -285,6 +767,7 @@ export function startRecordCapture(
if (target instanceof Element) {
const nearbyFillable = nearbyFillableFromSearchChrome(target);
if (nearbyFillable) {
emitHoverCandidateBeforeAction(nearbyFillable);
ensureFillSession(nearbyFillable);
return;
}
@@ -304,6 +787,7 @@ export function startRecordCapture(
commitFillSession();
if (target instanceof Element && target.closest("select")) return;
emitHoverCandidateBeforeAction(target);
emitClick(event);
};
@@ -345,6 +829,7 @@ export function startRecordCapture(
commitFillSession();
const target = eventTarget(event);
if (target instanceof HTMLSelectElement) {
emitHoverCandidateBeforeAction(target);
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
const labels = Array.from(target.selectedOptions).map((opt) =>
(opt.label || opt.textContent || opt.value).trim(),
@@ -398,6 +883,7 @@ export function startRecordCapture(
}
const desc = describeEventTarget(target);
if (!desc && !event.key) return;
emitHoverCandidateBeforeAction(target);
emitStep({
op: "press",
key: event.key,
@@ -408,6 +894,7 @@ export function startRecordCapture(
};
document.addEventListener("click", onClick, true);
document.addEventListener("mouseover", onMouseOver, true);
document.addEventListener("focusin", onFocusIn, true);
document.addEventListener("focusout", onFocusOut, true);
document.addEventListener("input", onInput, true);
@@ -438,6 +925,7 @@ export function startRecordCapture(
dispose() {
commitFillSession();
document.removeEventListener("click", onClick, true);
document.removeEventListener("mouseover", onMouseOver, true);
document.removeEventListener("focusin", onFocusIn, true);
document.removeEventListener("focusout", onFocusOut, true);
document.removeEventListener("input", onInput, true);
@@ -0,0 +1,237 @@
export interface HoverSurfaceDecision {
related: boolean;
reason?: string;
surface?: Element;
}
export interface HoverSurfaceContext {
triggerElement: Element;
}
export interface HoverSurfaceState {
element: Element;
signature: string;
}
const HOVER_SURFACE_SELECTOR = [
'[role="menu"]',
'[role="menubar"]',
'[role="listbox"]',
'[role="dialog"]',
'[role="tooltip"]',
'[aria-modal="true"]',
"[data-popper-placement]",
"[data-floating-ui-placement]",
"[data-headlessui-state]",
"[data-radix-popper-content-wrapper]",
].join(",");
const HOVER_SURFACE_WORD_RE =
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
const HOVER_SURFACE_ITEM_RE =
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
function elementTokenText(el: Element): string {
return [
el.id,
typeof el.className === "string" ? el.className : "",
el.getAttribute("data-testid"),
el.getAttribute("data-test"),
el.getAttribute("data-cy"),
el.getAttribute("aria-label"),
el.getAttribute("role"),
]
.filter((value): value is string => typeof value === "string" && value.length > 0)
.join(" ");
}
export function isRecognizedHoverSurfaceElement(el: Element): boolean {
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
const tokenText = elementTokenText(el);
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
return true;
}
return false;
}
function isVisibleSurfaceElement(el: Element): boolean {
if (!(el instanceof HTMLElement)) return false;
const style = getComputedStyle(el);
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
return false;
}
return rectFor(el) !== null;
}
export function closestRecognizedHoverSurface(element: Element): Element | null {
let node: Element | null = element;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
if (isRecognizedHoverSurfaceElement(node)) return node;
node = node.parentElement;
depth += 1;
}
return null;
}
export function isPositionedFloatingElement(el: Element): boolean {
if (!(el instanceof HTMLElement)) return false;
const style = getComputedStyle(el);
if (style.position !== "absolute") return false;
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
return false;
}
const rect = rectFor(el);
if (!rect) return false;
const viewportArea = window.innerWidth * window.innerHeight;
const area = rect.width * rect.height;
return (
area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO)
);
}
export function isHoverSurfaceCandidateElement(el: Element): boolean {
return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el);
}
export function closestHoverSurfaceCandidate(element: Element): Element | null {
let node: Element | null = element;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
if (isHoverSurfaceCandidateElement(node)) return node;
node = node.parentElement;
depth += 1;
}
return null;
}
function closestFloatingSurface(clickElement: Element): Element | null {
let node: Element | null = clickElement;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
if (isPositionedFloatingElement(node)) return node;
node = node.parentElement;
depth += 1;
}
return null;
}
function hoverSurfaceSignature(el: Element): string {
const rect = rectFor(el);
return [
rect
? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}`
: "",
(el.textContent ?? "").trim().slice(0, 512),
].join("|");
}
export function collectHoverSurfaceStates(): HoverSurfaceState[] {
const states: HoverSurfaceState[] = [];
for (const el of document.querySelectorAll("*")) {
if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) {
states.push({ element: el, signature: hoverSurfaceSignature(el) });
}
}
return states;
}
function rectFor(el: Element): DOMRect | null {
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return null;
return rect;
}
function surfaceRelationMetrics(surface: Element, trigger: Element) {
const surfaceRect = rectFor(surface);
const triggerRect = rectFor(trigger);
if (!surfaceRect || !triggerRect) return null;
const horizontalOverlap =
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
const verticalOverlap =
Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top);
const horizontalGap = Math.max(
0,
triggerRect.left - surfaceRect.right,
surfaceRect.left - triggerRect.right,
);
const verticalGap = Math.max(
0,
triggerRect.top - surfaceRect.bottom,
surfaceRect.top - triggerRect.bottom,
);
return {
surfaceRect,
triggerRect,
horizontalOverlap,
verticalOverlap,
horizontalGap,
verticalGap,
};
}
function isNearTrigger(surface: Element, trigger: Element): boolean {
const metrics = surfaceRelationMetrics(surface, trigger);
if (!metrics) return true;
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
return (
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP
);
}
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
const metrics = surfaceRelationMetrics(surface, trigger);
if (!metrics) return true;
const {
surfaceRect,
triggerRect,
horizontalOverlap,
verticalOverlap,
horizontalGap,
verticalGap,
} = metrics;
const opensVertically =
horizontalOverlap > 0 &&
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
const opensSideways =
verticalOverlap > 0 &&
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
const opensFromCorner =
horizontalGap > 0 &&
verticalGap > 0 &&
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
return opensVertically || opensSideways || opensFromCorner;
}
export function decideHoverSurfaceRelation(
context: HoverSurfaceContext,
clickElement: Element,
): HoverSurfaceDecision {
const clickSurface = closestRecognizedHoverSurface(clickElement);
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
}
const floatingSurface = closestFloatingSurface(clickElement);
if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) {
return {
related: true,
reason: "click-inside-near-floating-surface",
surface: floatingSurface,
};
}
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
return { related: true, reason: "click-inside-trigger-contained-surface" };
}
return { related: false, reason: "click-outside-hover-surface" };
}
@@ -0,0 +1,125 @@
import { describe, expect, it } from "vitest";
import { evaluateHoverTrigger } from "../hover-trigger-policy";
describe("evaluateHoverTrigger", () => {
it("accepts explicit popup triggers", () => {
const decision = evaluateHoverTrigger({
tag: "button",
role: "button",
label: "Open user navigation menu",
attrs: { "aria-haspopup": "menu", "aria-expanded": "false" },
rect: { x: 900, y: 8, w: 32, h: 32 },
cursor: "pointer",
pointerEvents: "auto",
});
expect(decision.eligible).toBe(true);
expect(decision.reasons).toEqual(
expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]),
);
});
it("accepts compact topbar image buttons without popup attributes", () => {
const decision = evaluateHoverTrigger({
tag: "button",
role: "button",
label: "image",
attrs: {},
rect: { x: 900, y: 8, w: 32, h: 32 },
pointerEvents: "auto",
hasGraphicDescendant: true,
});
expect(decision.eligible).toBe(true);
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
});
it("accepts compact topbar image links without popup attributes", () => {
const decision = evaluateHoverTrigger({
tag: "a",
role: "link",
label: "image",
attrs: {},
rect: { x: 900, y: 8, w: 32, h: 32 },
pointerEvents: "auto",
hasGraphicDescendant: true,
});
expect(decision.eligible).toBe(true);
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
});
it("rejects topbar menu links without surface trigger evidence", () => {
const decision = evaluateHoverTrigger({
tag: "a",
role: "link",
label: "My profile",
attrs: {},
rect: { x: 820, y: 72, w: 96, h: 32 },
cursor: "pointer",
pointerEvents: "auto",
});
expect(decision.eligible).toBe(false);
});
it("rejects text navigation items even when their attributes contain popup words", () => {
const decision = evaluateHoverTrigger({
tag: "a",
role: "link",
label: "My profile",
attrs: { class: "dropdown-item profile-link" },
rect: { x: 820, y: 72, w: 96, h: 32 },
cursor: "pointer",
pointerEvents: "auto",
});
expect(decision.eligible).toBe(false);
});
it("rejects labelled dropdown list items as hover triggers", () => {
const decision = evaluateHoverTrigger({
tag: "li",
label: "文档C+D",
attrs: { class: "dropdown-item", tabindex: "0" },
rect: { x: 820, y: 48, w: 160, h: 32 },
cursor: "pointer",
pointerEvents: "auto",
});
expect(decision.eligible).toBe(false);
});
it("rejects ordinary controls without hover popup signals", () => {
const decision = evaluateHoverTrigger({
tag: "button",
role: "button",
label: "Search",
attrs: {},
rect: { x: 100, y: 240, w: 120, h: 36 },
pointerEvents: "auto",
});
expect(decision.eligible).toBe(false);
});
it("rejects unsafe or non-visible triggers", () => {
expect(
evaluateHoverTrigger({
tag: "input",
attrs: {},
rect: { x: 10, y: 10, w: 80, h: 24 },
pointerEvents: "auto",
}).eligible,
).toBe(false);
expect(
evaluateHoverTrigger({
tag: "button",
role: "button",
attrs: { hidden: "" },
rect: { x: 10, y: 10, w: 80, h: 24 },
pointerEvents: "auto",
}).eligible,
).toBe(false);
});
});
@@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => {
});
});
it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { tag: "span", role: "button", name: "Account" },
page_url: "https://example.com/app",
},
{
op: "click",
target: { tag: "a", role: "link", name: "Profile" },
page_url: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
+14
View File
@@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null {
return null;
}
export function resolveHoverElement(target: Element): Element | null {
const clickable = resolveClickableElement(target);
if (clickable) return clickable;
let node: Element | null = target;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
if (looksClickable(node)) return node;
node = node.parentElement;
depth += 1;
}
return null;
}
/**
* Teachable clicks: the LLM must get a label it can later find via snapshot
* (visible name), or at least a form `name_attr` for checkbox/radio.
@@ -0,0 +1,178 @@
export interface HoverTriggerRect {
x: number;
y: number;
w: number;
h: number;
}
export interface HoverTriggerSignals {
tag: string;
role?: string;
label?: string;
attrs?: Record<string, string | undefined>;
rect?: HoverTriggerRect | null;
cursor?: string;
pointerEvents?: string;
hasGraphicDescendant?: boolean;
cssHoverMatch?: boolean;
}
export interface HoverTriggerDecision {
eligible: boolean;
score: number;
reasons: string[];
}
const HOVER_TRIGGER_MIN_SCORE = 45;
const HOVER_POPUP_SIGNAL_RE =
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
return signals.attrs?.[name.toLowerCase()];
}
export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean {
const tag = signals.tag.toLowerCase();
if (["input", "textarea", "select", "option"].includes(tag)) return true;
if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true;
if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true;
return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true";
}
export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
const attrs = signals.attrs ?? {};
const haystack = [
attrs.id,
attrs.class,
attrs["data-testid"],
attrs["data-test"],
attrs["data-cy"],
attrs["aria-label"],
attrs["aria-haspopup"],
attrs["aria-controls"],
attrs.title,
signals.role === "button" ? signals.label : undefined,
]
.filter(Boolean)
.join(" ")
.toLowerCase();
return HOVER_POPUP_SIGNAL_RE.test(haystack);
}
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
return (
signals.cssHoverMatch === true ||
attr(signals, "aria-haspopup") !== undefined ||
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
);
}
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
const tag = signals.tag.toLowerCase();
const role = (signals.role ?? "").toLowerCase();
return (
["button", "a", "summary"].includes(tag) ||
["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
attr(signals, "tabindex") !== undefined ||
signals.cursor === "pointer" ||
attr(signals, "onclick") !== undefined ||
attr(signals, "onmouseenter") !== undefined ||
attr(signals, "onmouseover") !== undefined ||
attr(signals, "aria-haspopup") !== undefined ||
attr(signals, "aria-expanded") === "false"
);
}
export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision {
const rect = signals.rect;
if (
!rect ||
rect.w <= 0 ||
rect.h <= 0 ||
signals.pointerEvents === "none" ||
attr(signals, "hidden") !== undefined ||
attr(signals, "inert") !== undefined ||
(attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" ||
isUnsafeHoverTrigger(signals)
) {
return { eligible: false, score: 0, reasons: [] };
}
const area = rect.w * rect.h;
if (area <= 0 || area > 160_000) {
return { eligible: false, score: 0, reasons: [] };
}
const reasons: string[] = [];
let score = 0;
const tag = signals.tag.toLowerCase();
const role = (signals.role ?? "").toLowerCase();
const label = signals.label?.trim();
const graphic = signals.hasGraphicDescendant === true;
const directInteractive = hasDirectHoverInteractiveSignal(signals);
const popupSignal = hasHoverPopupSignal(signals);
const isTextNavigationItem =
(tag === "a" ||
tag === "li" ||
role === "link" ||
role === "option" ||
role.startsWith("menuitem") ||
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
!graphic &&
!!label &&
label.toLowerCase() !== "image";
const compactTopbarIcon =
(role === "button" || role === "link" || tag === "button" || tag === "a") &&
graphic &&
rect.y < 120 &&
rect.w <= 96 &&
rect.h <= 96 &&
(!label || label.toLowerCase() === "image");
const surfaceTriggerEvidence =
(popupSignal && !isTextNavigationItem) ||
compactTopbarIcon ||
hasStrongHoverExpansionSignal(signals);
if (!directInteractive || !surfaceTriggerEvidence) {
return { eligible: false, score: 0, reasons: [] };
}
if (attr(signals, "aria-haspopup") !== undefined) {
score += 80;
reasons.push("aria-haspopup");
}
if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") {
score += 55;
reasons.push("collapsed");
}
if (popupSignal) {
score += 45;
reasons.push("popup-signal");
}
if (tag === "button" || role === "button") {
score += 30;
reasons.push("button");
}
if (signals.cursor === "pointer") {
score += 25;
reasons.push("pointer");
}
if (graphic && (!label || label.toLowerCase() === "image")) {
score += 40;
reasons.push("icon-only");
}
if (rect.y < 120) {
score += 25;
reasons.push("topbar");
}
if (rect.w <= 80 && rect.h <= 80) {
score += 20;
reasons.push("compact");
}
if (signals.cssHoverMatch) {
score += 40;
reasons.push("css-hover");
}
return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons };
}
+1 -1
View File
@@ -33,7 +33,7 @@ export interface RecordStartMessage {
}
export interface RecordStepPayload {
op: "click" | "fill" | "press" | "select" | "navigate";
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
target?: TargetDescriptor;
value?: string;
key?: string;
@@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "hover":
return payload.target
? {
op: "hover",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
+7
View File
@@ -139,6 +139,13 @@ function toV2Step(
},
effectForNavigation(step.navigated_to, urlToId),
);
case "hover":
return {
op: "hover",
id,
page,
target: step.target,
};
case "fill":
return {
op: "fill",
@@ -9,6 +9,8 @@ import type {
} from "@/transport/types";
import { ToolDispatcher } from "../dispatcher";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
function fakeTransport() {
const handlers = new Set<FrameHandler>();
const stateHandlers = new Set<ConnectionStateHandler>();
@@ -129,7 +131,7 @@ describe("ToolDispatcher", () => {
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
dispatcher.start();
deliver(makeRequest("tool.console", { session_id: "aa11" }));
@@ -172,7 +174,7 @@ describe("ToolDispatcher", () => {
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
dispatcher.start();
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
@@ -311,6 +313,277 @@ describe("ToolDispatcher", () => {
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
});
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
vi.stubGlobal("chrome", {
tabs: {
sendMessage: vi.fn(async () => undefined),
},
});
const { transport } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
const order: string[] = [];
const cdp = {
send: vi.fn(async <T>() => {
order.push("hover");
return {} as T;
}),
detachSession: vi.fn(async () => {}),
ensureNetworkCapture: vi.fn(async () => {}),
networkEntriesSince: vi.fn(() => ({
tab_id: 7,
entries: [],
next_since: 0,
truncated: false,
})),
setDeviceMetricsOverride: vi.fn(async () => {}),
clearDeviceMetricsOverride: vi.fn(async () => {}),
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
(
dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
).rememberHover("aa11", {
tab_id: 7,
x: 10,
y: 20,
});
const helpers = dispatcher as unknown as {
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
options?: { releaseAfter?: boolean },
) => Promise<T>;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
order.push("observe");
return {};
});
expect(order).toEqual(["hover", "observe"]);
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
await helpers.setHoverBypass("aa11", 7, true);
await helpers.withHoverReassert(
{ session_id: "aa11", tab_id: 7 },
async () => {
order.push("click");
return {};
},
{ releaseAfter: true },
);
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 10,
y: 20,
});
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
});
it("does not reassert or disable hover latches from another session", async () => {
vi.stubGlobal("chrome", {
tabs: {
sendMessage: vi.fn(async () => undefined),
},
});
const { transport } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
ensureNetworkCapture: vi.fn(async () => {}),
networkEntriesSince: vi.fn(() => ({
tab_id: 7,
entries: [],
next_since: 0,
truncated: false,
})),
setDeviceMetricsOverride: vi.fn(async () => {}),
clearDeviceMetricsOverride: vi.fn(async () => {}),
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
options?: { releaseAfter?: boolean },
) => Promise<T>;
releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
};
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await helpers.setHoverBypass("aa11", 7, true);
await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
releaseAfter: true,
});
expect(cdp.send).not.toHaveBeenCalled();
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
await helpers.releaseHoverLatch("aa11", 7);
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
});
it("transfers hover overlay bypass ownership between sessions", async () => {
vi.stubGlobal("chrome", {
tabs: {
sendMessage: vi.fn(async () => undefined),
},
});
const { transport } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
const helpers = dispatcher as unknown as {
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
await helpers.setHoverBypass("aa11", 7, true);
await helpers.setHoverBypass("bb22", 7, true);
await helpers.setHoverBypass("aa11", 7, false);
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
await helpers.setHoverBypass("bb22", 7, false);
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
});
it("limits default-target hover reassertion to the active tab", async () => {
vi.stubGlobal("chrome", {
tabs: {
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
sendMessage: vi.fn(async () => undefined),
},
});
const { transport } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
ensureNetworkCapture: vi.fn(async () => {}),
networkEntriesSince: vi.fn(() => ({
tab_id: 9,
entries: [],
next_since: 0,
truncated: false,
})),
setDeviceMetricsOverride: vi.fn(async () => {}),
clearDeviceMetricsOverride: vi.fn(async () => {}),
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
expect(cdp.send).toHaveBeenCalledTimes(1);
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 30,
y: 40,
});
});
it("releases the current session hover latch before navigation-style work", async () => {
vi.stubGlobal("chrome", {
tabs: {
sendMessage: vi.fn(async () => undefined),
},
});
const { transport } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
withHoverReleaseForRequest: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await helpers.setHoverBypass("aa11", 7, true);
await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
});
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
const { transport, deliver } = fakeTransport();
const sessions = new SessionManager({
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
import {
handleClick,
handleFill,
handleHover,
handlePress,
handleSelect,
modifiersBitfield,
@@ -435,6 +436,50 @@ describe("handleClick", () => {
});
});
describe("handleHover", () => {
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Runtime.evaluate": (params: unknown) => {
const expr = String((params as { expression?: string })?.expression ?? "");
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
}
if (expr.includes("hitIndex")) {
return {
result: {
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
},
};
}
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
},
"Input.dispatchMouseEvent": () => ({}),
});
const res = await handleHover(
sm,
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
{
cdp: fake.cdp,
tabsApi: fake.tabsApi,
bypassOverlay,
keepOverlayBypassAfterHover: true,
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
expect(bypassOverlay).toHaveBeenCalledTimes(1);
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
});
});
describe("handleFill", () => {
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
@@ -1430,7 +1430,7 @@ describe("buildVomScene", () => {
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
});
it("maps hover surface probes to matching node labels", () => {
it("maps hover surface probes to matching backend node ids", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
@@ -1451,7 +1451,7 @@ describe("buildVomScene", () => {
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
],
nodes: [
{
@@ -1480,7 +1480,260 @@ describe("buildVomScene", () => {
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
});
it("maps hover surface probes to rendered descendants in the same trigger subtree", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "image" },
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
});
it("deduplicates hover surface probes by original trigger backend id", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "image" },
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
]);
});
it("maps hover surface probes to DOM-recovered custom controls", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "img" },
name: { type: "computedString", value: "image" },
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 20,
triggerPoint: { x: 915, y: 25 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
cursor: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
],
});
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
]);
expect(renderVom(scene).text).toContain(
'@e1 button "image" [hover first: My profile | Sign out]',
);
});
it("does not attach hover probes to distant geometry matches", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "link" },
name: { type: "computedString", value: "0" },
backendDOMNodeId: 42,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 999,
triggerPoint: { x: 900, y: 20 },
triggerAction: "hover",
subItems: ["My profile"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 42,
parentBackendNodeId: 10,
tag: "a",
attrs: {},
rect: { x: 860, y: 120, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
expect(scene.surfaces).toBeUndefined();
expect(renderVom(scene).text).not.toContain("[hover first:");
});
it("enriches names and active scope signals from AX properties", () => {
@@ -1953,7 +2206,7 @@ describe("handleSnapshot", () => {
});
const res = await handleObserve(
sm,
{ session_id: "aa11" },
{ session_id: "aa11", debug_surfaces: true },
{
cdp: {
send: send as unknown as <T = unknown>(
@@ -1974,6 +2227,7 @@ describe("handleSnapshot", () => {
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.debug).toEqual({ surface_probes: [] });
expect(send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
+234 -53
View File
@@ -8,6 +8,8 @@ import type {
EvaluateParams,
FillParams,
GetHtmlParams,
HoverParams,
HoverResult,
NavigateBackParams,
NavigateForwardParams,
NavigateParams,
@@ -33,7 +35,7 @@ import { handleConsole } from "./console";
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import {
handleNavigate,
handleNavigateBack,
@@ -56,7 +58,7 @@ import {
type SessionStartParams,
type SessionStopParams,
} from "./session";
import { chromeTabsApi } from "./shared";
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
import {
type BorrowConfirmationApprover,
handleTabBorrow,
@@ -81,6 +83,18 @@ type DispatcherCdpRunner = CdpRunner &
detachSession(sessionId: string): Promise<void>;
};
interface HoverLatch {
sessionId: string;
tabId: number;
x: number;
y: number;
}
interface HoverLatchScope {
session_id: string;
tab_id?: number;
}
export interface DispatcherDeps {
transport: Transport;
sessions: SessionManager;
@@ -126,6 +140,8 @@ export class ToolDispatcher {
private readonly approveBorrow?: BorrowConfirmationApprover;
private readonly helpNotificationCopy?: () => { title: string; body: string };
private subscription: { dispose(): void } | null = null;
private readonly hoverBypassTabs = new Map<number, string>();
private readonly hoverLatches = new Map<number, HoverLatch>();
/**
* Per-rpc-id `AbortController` registry. Populated inside
* [`dispatch`] before we await the tool handler and torn down in
@@ -271,16 +287,20 @@ export class ToolDispatcher {
switch (req.method) {
case "tool.session_start":
return handleSessionStart(this.sessions, req.params as SessionStartParams);
case "tool.session_stop":
case "tool.session_stop": {
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
cdp: this.cdp,
});
}
case "tool.tab_list":
return handleTabList(this.sessions, req.params as TabListParams);
case "tool.tab_create":
return handleTabCreate(this.sessions, req.params as TabCreateParams);
case "tool.tab_close":
return handleTabClose(this.sessions, req.params as TabCloseParams);
return this.withHoverReleaseForRequest(req.params as TabCloseParams, () =>
handleTabClose(this.sessions, req.params as TabCloseParams),
);
case "tool.tab_select":
return handleTabSelect(this.sessions, req.params as TabSelectParams);
case "tool.tab_borrow":
@@ -319,17 +339,31 @@ export class ToolDispatcher {
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
);
case "tool.snapshot":
return handleSnapshot(
this.sessions,
req.params as SnapshotParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
return this.withHoverReassert(req.params as SnapshotParams, () =>
handleSnapshot(
this.sessions,
req.params as SnapshotParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
),
);
case "tool.observe":
return handleObserve(
this.sessions,
req.params as ObserveParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
case "tool.observe": {
const params = req.params as ObserveParams;
const hoverScope = await this.resolveHoverLatchScope(params);
return this.withHoverReassert(params, () =>
handleObserve(
this.sessions,
params,
this.cdp
? {
cdp: this.cdp,
tabsApi: chromeTabsCaptureApi,
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
hoverProbeBypassOverlay: bypassOverlay,
}
: undefined,
),
);
}
case "tool.get_html":
return handleGetHtml(
this.sessions,
@@ -337,68 +371,95 @@ export class ToolDispatcher {
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
);
case "tool.navigate":
return handleNavigate(
this.sessions,
req.params as NavigateParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as NavigateParams, () =>
handleNavigate(
this.sessions,
req.params as NavigateParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.navigate_back":
return handleNavigateBack(
this.sessions,
req.params as NavigateBackParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () =>
handleNavigateBack(
this.sessions,
req.params as NavigateBackParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.navigate_forward":
return handleNavigateForward(
this.sessions,
req.params as NavigateForwardParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () =>
handleNavigateForward(
this.sessions,
req.params as NavigateForwardParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.reload":
return handleReload(
this.sessions,
req.params as ReloadParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as ReloadParams, () =>
handleReload(
this.sessions,
req.params as ReloadParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.click":
return handleClick(
this.sessions,
return this.withHoverReassert(
req.params as ClickParams,
() =>
handleClick(
this.sessions,
req.params as ClickParams,
this.cdp
? {
cdp: this.cdp,
tabsApi: chromeTabsApi,
signal,
bypassOverlay,
}
: undefined,
),
{ releaseAfter: true },
);
case "tool.hover": {
const result = await handleHover(
this.sessions,
req.params as HoverParams,
this.cdp
? {
cdp: this.cdp,
tabsApi: chromeTabsApi,
signal,
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
bypassOverlay: (tabId, enabled) =>
this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled),
keepOverlayBypassAfterHover: true,
}
: undefined,
);
return this.rememberHover((req.params as HoverParams).session_id, result);
}
case "tool.fill":
return handleFill(
this.sessions,
req.params as FillParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as FillParams, () =>
handleFill(
this.sessions,
req.params as FillParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.press":
return handlePress(
this.sessions,
req.params as PressParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as PressParams, () =>
handlePress(
this.sessions,
req.params as PressParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.select":
return handleSelect(
this.sessions,
req.params as SelectParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
return this.withHoverReleaseForRequest(req.params as SelectParams, () =>
handleSelect(
this.sessions,
req.params as SelectParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.evaluate":
return handleEvaluate(
@@ -471,6 +532,114 @@ export class ToolDispatcher {
} satisfies RpcError;
}
}
private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise<void> {
const owner = this.hoverBypassTabs.get(tabId);
if (enabled) {
if (owner === sessionId) return;
if (owner === undefined) await bypassOverlay(tabId, true);
this.hoverBypassTabs.set(tabId, sessionId);
} else {
if (owner !== sessionId) return;
await bypassOverlay(tabId, false);
this.hoverBypassTabs.delete(tabId);
}
}
private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError {
if (!isRpcError(result)) {
this.hoverLatches.set(result.tab_id, {
sessionId,
tabId: result.tab_id,
x: result.x,
y: result.y,
});
}
return result;
}
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
return this.hoverLatchesForRequest(scope).length > 0;
}
private async withHoverReassert<T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
options: { releaseAfter?: boolean } = {},
): Promise<T> {
const scope = await this.resolveHoverLatchScope(params);
await this.reassertHover(scope);
try {
return await work();
} finally {
if (options.releaseAfter) {
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
}
}
}
private async withHoverReleaseForRequest<T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
): Promise<T> {
const scope = await this.resolveHoverLatchScope(params);
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
return work();
}
private async resolveHoverLatchScope(params: {
session_id: string;
tab_id?: number;
}): Promise<HoverLatchScope> {
if (params.tab_id !== undefined) return params;
const ctx = lookupSession(this.sessions, params, "hover latch");
if (isRpcError(ctx)) return params;
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
if (isRpcError(target)) return params;
return { session_id: params.session_id, tab_id: target.tabId };
}
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
return [...this.hoverLatches.values()].filter((latch) => {
if (latch.sessionId !== params.session_id) return false;
return params.tab_id === undefined || latch.tabId === params.tab_id;
});
}
private async reassertHover(params: { session_id: string; tab_id?: number }): Promise<void> {
if (!this.cdp) return;
await Promise.all(
this.hoverLatchesForRequest(params).map((latch) =>
this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: latch.x,
y: latch.y,
}).catch((err) => {
console.debug("[bsk dispatcher] hover reassert failed", err);
this.hoverLatches.delete(latch.tabId);
}),
),
);
}
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
if (sessionId !== undefined && entrySessionId !== sessionId) return false;
return tabId === undefined || entryTabId === tabId;
};
const tabs = new Set<number>();
for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) {
if (!matchesScope(bypassSessionId, bypassTabId)) continue;
tabs.add(bypassTabId);
this.hoverBypassTabs.delete(bypassTabId);
}
for (const latch of this.hoverLatches.values()) {
if (!matchesScope(latch.sessionId, latch.tabId)) continue;
tabs.add(latch.tabId);
this.hoverLatches.delete(latch.tabId);
}
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
}
}
function isRpcError(v: unknown): v is RpcError {
@@ -497,6 +666,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.navigate_forward":
case "tool.reload":
case "tool.click":
case "tool.hover":
case "tool.fill":
case "tool.press":
case "tool.select":
@@ -512,6 +682,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
}
}
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
}
/**
* Resolves never; rejects with `AbortLikeError` as soon as the signal
* fires (or immediately if it is already aborted). Used by the
+124 -1
View File
@@ -17,6 +17,8 @@ import type {
ClickResult,
FillParams,
FillResult,
HoverParams,
HoverResult,
KeyModifier,
MouseButton,
PressParams,
@@ -53,9 +55,12 @@ export interface InteractionDeps {
defaultTimeoutMs?: number;
/** Temporarily disable overlay click blocker during CDP automation. */
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
/** Keep hover hit-testing active for the caller's next observation/action. */
keepOverlayBypassAfterHover?: boolean;
}
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_HOVER_SETTLE_MS = 200;
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
@@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
return null;
}
function isAbortLikeError(err: unknown): boolean {
return err instanceof DOMException && err.name === "AbortError";
}
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
await new Promise<void>((resolve, reject) => {
const cleanup = () => signal?.removeEventListener("abort", onAbort);
const timer = setTimeout(() => {
cleanup();
resolve();
}, ms);
const onAbort = () => {
clearTimeout(timer);
cleanup();
reject(new DOMException("aborted", "AbortError"));
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
/**
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
* `RpcError` if the caller supplied neither (or both), or if neither
@@ -286,7 +312,104 @@ export async function handleClick(
message: err instanceof Error ? err.message : String(err),
};
} finally {
if (automationBypassEnabled && deps.bypassOverlay) {
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
try {
await deps.bypassOverlay(target.tabId, false);
} catch (err) {
console.debug("[bsk interaction] overlay bypass disable failed", err);
}
}
}
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
x: centre.x,
y: centre.y,
});
}
// ---------------------------------------------------------------------------
// tool.hover
// ---------------------------------------------------------------------------
export async function handleHover(
manager: SessionManager,
params: HoverParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<HoverResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "hover");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const aborted = throwIfAborted(deps.signal);
if (aborted) return aborted;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "hover");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
if (isRpcError(node)) return node;
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
if (scrollErr) return scrollErr;
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
}
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
if (isRpcError(centre)) return centre;
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "hover aborted" };
}
const modifiers = modifiersBitfield(params.modifiers);
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
let automationBypassEnabled = false;
let hoverCompleted = false;
if (overlayBlocking && deps.bypassOverlay) {
try {
await deps.bypassOverlay(target.tabId, true);
automationBypassEnabled = true;
} catch (err) {
console.debug("[bsk interaction] overlay bypass enable failed", err);
}
}
try {
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: centre.x,
y: centre.y,
modifiers,
});
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
if (settleMs > 0) {
await wait(settleMs, deps.signal);
}
hoverCompleted = true;
} catch (err) {
if (isAbortLikeError(err)) {
return { code: "cancelled", message: "hover aborted" };
}
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
} finally {
if (
automationBypassEnabled &&
deps.bypassOverlay &&
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
) {
try {
await deps.bypassOverlay(target.tabId, false);
} catch (err) {
+114 -26
View File
@@ -5,7 +5,9 @@
import {
type ActiveScopeBlock,
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
renderVom,
type VomNode,
type VomScene,
@@ -908,39 +910,28 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
return blocks;
}
function labelForSurfaceMatch(node: VomNode): string {
return (
cleanAttr(node.name) ??
cleanAttr(node.text) ??
cleanAttr(node.attrs?.["aria-label"]) ??
cleanAttr(node.attrs?.title) ??
""
);
}
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
const probes = captured.surfaceProbes ?? [];
if (probes.length === 0) return [];
const surfaces: CondSurface[] = [];
const nodesWithLabels = nodes
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
.filter((entry) => entry.key.length > 0);
const recoveredNodes = applyVomInteractionRecovery(nodes);
const signals = capturedOnlySignals(captured.nodes);
const used = new Set<number>();
for (const probe of probes) {
const probeKey = normalizeProbeKey(probe.triggerLabel);
if (!probeKey || probe.subItems.length === 0) continue;
const match =
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
nodesWithLabels.find(
(entry) =>
!used.has(entry.node.id) &&
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
);
if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue;
const match = findSurfaceTriggerNode(
probe.triggerBackendNodeId,
recoveredNodes,
signals,
probe.triggerPoint,
);
if (!match) continue;
used.add(match.node.id);
if (used.has(match.id)) continue;
used.add(probe.triggerBackendNodeId);
used.add(match.id);
surfaces.push({
triggerId: match.node.id,
triggerId: match.id,
triggerAction: probe.triggerAction,
subItems: probe.subItems,
});
@@ -948,6 +939,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel)
return surfaces;
}
function findSurfaceTriggerNode(
triggerBackendNodeId: number,
nodes: VomNode[],
signals: VomNodeDomSignals,
triggerPoint?: { x: number; y: number },
): VomNode | undefined {
const renderedById = new Map(nodes.map((node) => [node.id, node]));
const exact = renderedById.get(triggerBackendNodeId);
if (exact && isSurfaceAttachableNode(exact)) return exact;
const domDescendant = nodes.find(
(node) =>
isSurfaceAttachableNode(node) &&
(node.domParentId === triggerBackendNodeId ||
node.domAncestorIds?.includes(triggerBackendNodeId) === true),
);
if (domDescendant) return domDescendant;
const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])];
while (queue.length > 0) {
const child = queue.shift() as CapturedNode;
const rendered = renderedById.get(child.backendNodeId);
if (rendered && isSurfaceAttachableNode(rendered)) return rendered;
queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? []));
}
let current = signals.capturedByBackendId.get(triggerBackendNodeId);
while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) {
const parent = renderedById.get(current.parentBackendNodeId);
if (parent && isSurfaceAttachableNode(parent)) return parent;
current = signals.capturedByBackendId.get(current.parentBackendNodeId);
}
if (triggerPoint) {
return findSurfaceNodeByPoint(triggerPoint, nodes, signals);
}
return undefined;
}
function isSurfaceAttachableNode(node: VomNode): boolean {
return isVomReferenceNode(node);
}
function findSurfaceNodeByPoint(
point: { x: number; y: number },
nodes: VomNode[],
signals: VomNodeDomSignals,
): VomNode | undefined {
let best: { node: VomNode; score: number } | undefined;
for (const node of nodes) {
if (!isSurfaceAttachableNode(node)) continue;
const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect;
if (!rect) continue;
const contains =
point.x >= rect.x &&
point.x <= rect.x + rect.w &&
point.y >= rect.y &&
point.y <= rect.y + rect.h;
const centerX = rect.x + rect.w / 2;
const centerY = rect.y + rect.h / 2;
const distance = Math.hypot(point.x - centerX, point.y - centerY);
if (!contains && distance > 40) continue;
const score = contains ? distance : distance + 1_000;
if (!best || score < best.score) {
best = { node, score };
}
}
return best?.node;
}
function axNodeInOverlaySubtree(
axNode: CdpAxNode,
axById: Map<string, CdpAxNode>,
@@ -1072,6 +1134,8 @@ export interface SnapshotDeps {
get(tabId: number): Promise<chrome.tabs.Tab>;
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
};
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
let defaultDeps: SnapshotDeps | null = null;
@@ -1217,9 +1281,13 @@ async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe,
hoverProbeBypassOverlay,
});
} catch {
return fallbackCapturedViewModel(cdp, tabId);
}
@@ -1258,7 +1326,14 @@ async function handleVomObservation(
{},
);
const axNodes = result.nodes ?? [];
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom(
deps.cdp,
target.tabId,
effectiveConditionalSurfaceProbe,
deps.hoverProbeBypassOverlay,
);
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
const rendered = renderVom(scene, {
maxDepth: params.max_depth,
@@ -1275,6 +1350,19 @@ async function handleVomObservation(
ref_count: rendered.refs.length,
tab_id: target.tabId,
truncated: rendered.truncated,
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? {
debug: {
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
trigger_backend_node_id: probe.triggerBackendNodeId,
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
trigger_action: probe.triggerAction,
sub_items: probe.subItems,
...(probe.confidence ? { confidence: probe.confidence } : {}),
})),
},
}
: {}),
});
} catch (err) {
return {
@@ -51,6 +51,156 @@ function fakeSnapshotReply() {
};
}
function hoverTriggerSnapshotReply() {
const S = [
"html",
"body",
"button",
"class",
"user-avatar",
"position",
"static",
"pointer-events",
"auto",
"cursor",
"pointer",
];
const i = (s: string) => S.indexOf(s);
return {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1],
nodeName: [i("html"), i("body"), i("button")],
backendNodeId: [10, 11, 12],
attributes: [[], [], [i("class"), i("user-avatar")]],
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
],
bounds: [
[0, 0, 1000, 800],
[940, 16, 32, 32],
],
paintOrders: [0, 1],
},
},
],
};
}
function twoHoverTriggerSnapshotReply() {
const S = [
"html",
"body",
"button",
"class",
"user-avatar",
"secondary-dropdown-trigger",
"position",
"static",
"pointer-events",
"auto",
"cursor",
"pointer",
];
const i = (s: string) => S.indexOf(s);
return {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button")],
backendNodeId: [10, 11, 12, 13],
attributes: [
[],
[],
[i("class"), i("user-avatar")],
[i("class"), i("secondary-dropdown-trigger")],
],
},
layout: {
nodeIndex: [1, 2, 3],
styles: [
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
],
bounds: [
[0, 0, 1000, 800],
[940, 16, 32, 32],
[100, 120, 32, 32],
],
paintOrders: [0, 1, 2],
},
},
],
};
}
function nestedHoverTriggerSnapshotReply() {
const S = [
"html",
"body",
"div",
"img",
"class",
"tg-avatar",
"tg-avatar__inner",
"tg-avatar__image",
"position",
"static",
"pointer-events",
"auto",
"cursor",
"pointer",
];
const i = (s: string) => S.indexOf(s);
return {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")],
backendNodeId: [10, 11, 12, 13, 14],
attributes: [
[],
[],
[i("class"), i("tg-avatar")],
[i("class"), i("tg-avatar__inner")],
[i("class"), i("tg-avatar__image")],
],
},
layout: {
nodeIndex: [1, 2, 3, 4],
styles: [
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
],
bounds: [
[0, 0, 1000, 800],
[940, 16, 32, 32],
[942, 18, 28, 28],
[944, 20, 24, 24],
],
paintOrders: [0, 1, 2, 3],
},
},
],
};
}
function makeCdp(snapshot: unknown) {
return {
send: vi.fn(async (_tab: number, method: string) => {
@@ -126,11 +276,11 @@ describe("captureViewModel", () => {
});
it("runs hover surface probes when explicitly enabled", async () => {
let runtimeCalls = 0;
let hoverStateCalls = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
@@ -141,22 +291,23 @@ describe("captureViewModel", () => {
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
runtimeCalls += 1;
return runtimeCalls === 1
? {
result: {
value: [
{
triggerSel: ".menu",
affectedSub: " .items",
label: "Products",
x: 50,
y: 20,
},
],
},
}
: { result: { value: ["Shoes", "Bags"] } };
if (expression.includes("document.styleSheets")) {
return { result: { value: [] } };
}
if (expression.includes("querySelectorAll(selectors)")) {
hoverStateCalls += 1;
return hoverStateCalls === 1
? { result: { value: [] } }
: {
result: {
value: [
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
],
},
};
}
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
}
if (method === "Input.dispatchMouseEvent") return {};
throw new Error(`unexpected ${method}`);
@@ -166,12 +317,18 @@ describe("captureViewModel", () => {
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(result.surfaceProbes).toEqual([
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
{
triggerBackendNodeId: 12,
triggerPoint: { x: 956, y: 32 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
confidence: "high",
},
]);
expect(cdp.send).toHaveBeenCalledWith(
4,
"Input.dispatchMouseEvent",
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }),
);
expect(cdp.send).toHaveBeenCalledWith(
4,
@@ -180,6 +337,91 @@ describe("captureViewModel", () => {
);
});
it("uses a fresh baseline for each hover candidate", async () => {
let hoverStateCalls = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
if (expression.includes("document.styleSheets")) {
return { result: { value: [] } };
}
if (expression.includes("querySelectorAll(selectors)")) {
hoverStateCalls += 1;
const visibleMenu = [
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
];
return hoverStateCalls === 1
? { result: { value: [] } }
: { result: { value: visibleMenu } };
}
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
}
if (method === "Input.dispatchMouseEvent") return {};
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(result.surfaceProbes).toEqual([
expect.objectContaining({
triggerBackendNodeId: 12,
subItems: ["My profile", "Sign out"],
}),
]);
expect(hoverStateCalls).toBeGreaterThanOrEqual(4);
});
it("deduplicates nested hover candidates for one visual trigger", async () => {
let hoverMoves = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
if (expression.includes("document.styleSheets")) {
return { result: { value: [] } };
}
if (expression.includes("querySelectorAll(selectors)")) {
return { result: { value: [] } };
}
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
}
if (method === "Input.dispatchMouseEvent") {
const point = params as { x?: number; y?: number };
if (point.x !== -10 && point.x !== 0) hoverMoves += 1;
return {};
}
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(hoverMoves).toBe(1);
});
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
// DOMSnapshot inlines an open shadow root's content as descendants of the
// host. The agent's WXT overlay host carries a fixed full-viewport
+277 -113
View File
@@ -5,6 +5,7 @@
// `styles` columns are [position, pointer-events, cursor] in that order.
import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import type { CdpRunner } from "../shared";
@@ -45,9 +46,11 @@ export interface CapturedNode {
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerLabel: string;
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export interface CapturedViewModel {
@@ -61,6 +64,7 @@ export interface CapturedViewModel {
export interface CaptureViewModelOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
@@ -225,11 +229,12 @@ interface RuntimeEvaluateReply {
}
interface HoverCandidate {
triggerSel: string;
affectedSub: string;
label: string;
backendNodeId: number;
label?: string;
x: number;
y: number;
score: number;
reasons: string[];
}
interface CdpDomNode {
@@ -322,17 +327,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
}
const MAX_HOVER_PROBE_MS = 2_000;
const MAX_HOVER_TRIGGERS = 8;
const HOVER_SETTLE_MS = 200;
const MAX_HOVER_TRIGGERS = 6;
const MAX_HOVER_SURFACES = 3;
const HOVER_SETTLE_MS = 300;
const MAX_HOVER_SUB_ITEMS = 12;
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
return reply.result?.value as T | undefined;
}
function hoverCssScanExpression(): string {
function hoverCssTriggerScanExpression(): string {
return `(() => {
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
const pairs = [];
const centres = [];
const seenRules = new Set();
for (const sheet of Array.from(document.styleSheets)) {
let rules;
@@ -348,90 +355,73 @@ function hoverCssScanExpression(): string {
const hoverIndex = part.indexOf(":hover");
if (hoverIndex < 0) continue;
const triggerSel = part.slice(0, hoverIndex).trim();
const affectedSub = part.slice(hoverIndex + 6).trim();
if (!triggerSel) continue;
const key = triggerSel + "||" + affectedSub;
if (seenRules.has(key)) continue;
seenRules.add(key);
pairs.push({ triggerSel, affectedSub });
if (seenRules.has(triggerSel)) continue;
seenRules.add(triggerSel);
let elements;
try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof HTMLElement)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 });
if (centres.length >= 24) return centres;
}
}
}
}
const candidates = [];
const seenLabels = new Set();
const isUnsafeTrigger = (el) => {
if (!(el instanceof HTMLElement)) return true;
const tag = el.tagName.toLowerCase();
if (["input", "textarea", "select", "option"].includes(tag)) return true;
if (el.isContentEditable) return true;
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
return false;
};
for (const pair of pairs) {
let elements;
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof Element)) continue;
if (isUnsafeTrigger(el)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
if (!label || seenLabels.has(label)) continue;
seenLabels.add(label);
candidates.push({
triggerSel: pair.triggerSel,
affectedSub: pair.affectedSub,
label,
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
});
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
}
}
return candidates;
return centres;
})()`;
}
function hoverCollectExpression(candidate: HoverCandidate): string {
return `(() => {
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
const x = ${JSON.stringify(candidate.x)};
const y = ${JSON.stringify(candidate.y)};
const hit = document.elementFromPoint(x, y);
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
if (!trigger) return [];
interface HoverRuntimeItem {
text: string;
role: string;
tag: string;
x: number;
y: number;
}
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
let targets = [];
if (!affectedSub) {
targets = [trigger];
} else {
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
}
function hoverStateExpression(): string {
return `(() => {
const items = [];
const seen = new Set();
const push = (value) => {
const text = String(value || "").replace(/\\s+/g, " ").trim();
const selectors = [
"a",
"button",
"[role='menuitem']",
"[role='menuitemcheckbox']",
"[role='menuitemradio']",
"[role='option']",
"[role='tab']",
"[role='link']",
"[role='button']"
].join(",");
const push = (el) => {
if (!(el instanceof HTMLElement)) return;
const style = getComputedStyle(el);
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return;
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
const text = String(
el.getAttribute("aria-label") ||
el.getAttribute("title") ||
el.textContent ||
""
).replace(/\\s+/g, " ").trim();
if (!text || seen.has(text)) return;
seen.add(text);
items.push(text);
items.push({
text,
role: (el.getAttribute("role") || "").toLowerCase(),
tag: el.tagName.toLowerCase(),
x: rect.left,
y: rect.top,
});
};
for (const target of targets) {
if (!(target instanceof Element)) continue;
const style = getComputedStyle(target);
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
if (clickable.length > 0) {
for (const child of Array.from(clickable)) push(child.textContent);
} else {
push(target.textContent);
}
}
return items.slice(0, 12);
for (const el of Array.from(document.querySelectorAll(selectors))) push(el);
return items.slice(0, 400);
})()`;
}
@@ -453,45 +443,217 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
);
}
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
function capturedText(node: CapturedNode): string | undefined {
const value =
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
const clean = value.replace(/\s+/g, " ").trim();
return clean || undefined;
}
function hasGraphicDescendant(
node: CapturedNode,
childrenByParentId: Map<number, CapturedNode[]>,
depth = 0,
): boolean {
if (depth > 3) return false;
for (const child of childrenByParentId.get(node.backendNodeId) ?? []) {
const tag = child.tag.toLowerCase();
if (["img", "svg", "use", "path", "i"].includes(tag)) return true;
if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true;
}
return false;
}
function roleOf(node: CapturedNode): string {
return (node.attrs.role ?? "").toLowerCase();
}
function scoreHoverCandidate(
node: CapturedNode,
childrenByParentId: Map<number, CapturedNode[]>,
cssHoverPoints: Array<{ x: number; y: number }>,
): HoverCandidate | null {
const rect = node.rect;
if (!rect) return null;
const label = capturedText(node);
const cssHoverMatch = cssHoverPoints.some(
(point) =>
point.x >= rect.x &&
point.x <= rect.x + rect.w &&
point.y >= rect.y &&
point.y <= rect.y + rect.h,
);
const decision = evaluateHoverTrigger({
tag: node.tag,
role: roleOf(node),
label,
attrs: node.attrs,
rect,
cursor: node.cursor,
pointerEvents: node.pointerEvents,
hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId),
cssHoverMatch,
});
if (!decision.eligible) return null;
return {
backendNodeId: node.backendNodeId,
label,
x: rect.x + rect.w / 2,
y: rect.y + rect.h / 2,
score: decision.score,
reasons: decision.reasons,
};
}
function buildHoverCandidates(
nodes: CapturedNode[],
cssHoverPoints: Array<{ x: number; y: number }>,
): HoverCandidate[] {
const childrenByParentId = new Map<number, CapturedNode[]>();
const parentByBackendId = new Map<number, number | null>();
for (const node of nodes) {
parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId);
if (node.parentBackendNodeId === null) continue;
const children = childrenByParentId.get(node.parentBackendNodeId) ?? [];
children.push(node);
childrenByParentId.set(node.parentBackendNodeId, children);
}
const candidates = nodes
.map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints))
.filter((candidate): candidate is HoverCandidate => candidate !== null)
.sort((a, b) => b.score - a.score);
const deduped: HoverCandidate[] = [];
const seen = new Set<number>();
for (const candidate of candidates) {
if (seen.has(candidate.backendNodeId)) continue;
if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) {
continue;
}
seen.add(candidate.backendNodeId);
deduped.push(candidate);
if (deduped.length >= MAX_HOVER_TRIGGERS) break;
}
return deduped;
}
function sameHoverCluster(
a: HoverCandidate,
b: HoverCandidate,
parentByBackendId: Map<number, number | null>,
): boolean {
if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true;
return (
isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) ||
isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId)
);
}
function isBackendAncestor(
ancestorId: number,
nodeId: number,
parentByBackendId: Map<number, number | null>,
): boolean {
let parentId = parentByBackendId.get(nodeId);
let guard = 0;
while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) {
if (parentId === ancestorId) return true;
parentId = parentByBackendId.get(parentId);
guard += 1;
}
return false;
}
function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] {
const beforeKeys = new Set(before.map((item) => item.text.toLowerCase()));
const out: string[] = [];
const seen = new Set<string>();
for (const item of after) {
const text = item.text.replace(/\s+/g, " ").trim();
const key = text.toLowerCase();
if (!text || beforeKeys.has(key) || seen.has(key)) continue;
seen.add(key);
out.push(text);
if (out.length >= MAX_HOVER_SUB_ITEMS) break;
}
return out;
}
function confidenceForHover(
candidate: HoverCandidate,
subItems: string[],
): "high" | "medium" | "low" {
if (subItems.length >= 2 && candidate.score >= 80) return "high";
if (subItems.length >= 2 || candidate.score >= 80) return "medium";
return "low";
}
async function probeHoverSurfaces(
cdp: CdpRunner,
tabId: number,
nodes: CapturedNode[],
options: CaptureViewModelOptions,
): Promise<CapturedSurfaceProbe[]> {
const started = Date.now();
try {
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverCssScanExpression(),
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverCssTriggerScanExpression(),
returnByValue: true,
});
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
if (candidates.length === 0) return [];
const results: CapturedSurfaceProbe[] = [];
const seen = new Set<string>();
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
if (!candidate.label || seen.has(candidate.label)) continue;
try {
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
await wait(HOVER_SETTLE_MS);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverCollectExpression(candidate),
returnByValue: true,
});
const subItems = (runtimeValue<string[]>(collected) ?? [])
.map((item) => item.replace(/\s+/g, " ").trim())
.filter(Boolean);
if (subItems.length === 0) continue;
seen.add(candidate.label);
results.push({
triggerLabel: candidate.label,
triggerAction: "hover",
subItems,
});
} catch {
continue;
} finally {
await clearHover(cdp, tabId);
const seen = new Set<number>();
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
try {
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
if (results.length >= MAX_HOVER_SURFACES) break;
if (seen.has(candidate.backendNodeId)) continue;
try {
await clearHover(cdp, tabId);
await wait(HOVER_SETTLE_MS);
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
await wait(HOVER_SETTLE_MS);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
const subItems = diffHoverItems(
baselineItems,
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
);
if (subItems.length === 0) continue;
seen.add(candidate.backendNodeId);
results.push({
triggerBackendNodeId: candidate.backendNodeId,
triggerPoint: { x: candidate.x, y: candidate.y },
triggerAction: "hover",
subItems,
confidence: confidenceForHover(candidate, subItems),
});
} catch {
continue;
} finally {
await clearHover(cdp, tabId);
}
}
} finally {
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
}
return results;
} catch (err) {
@@ -797,7 +959,9 @@ export async function captureViewModel(
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
const surfaceProbes = options.conditionalSurfaceProbe
? await probeHoverSurfaces(cdp, tabId, nodes, options)
: [];
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
}
+40 -2
View File
@@ -314,8 +314,21 @@ export interface SnapshotResult {
dialogs?: JavaScriptDialogInfo[];
}
export type ObserveParams = SnapshotParams;
export type ObserveResult = SnapshotResult;
export interface ObserveParams extends SnapshotParams {
debug_surfaces?: boolean;
}
export interface ObserveResult extends SnapshotResult {
debug?: {
surface_probes?: Array<{
trigger_backend_node_id: number;
trigger_point?: { x: number; y: number };
trigger_action: string;
sub_items: string[];
confidence?: string;
}>;
};
}
export interface GetHtmlParams {
session_id: string;
@@ -410,6 +423,25 @@ export interface ClickResult {
dialogs?: JavaScriptDialogInfo[];
}
export interface HoverParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
modifiers?: KeyModifier[];
settle_ms?: number;
timeout_ms?: number;
}
export interface HoverResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
x: number;
y: number;
dialogs?: JavaScriptDialogInfo[];
}
export interface FillParams {
session_id: string;
value: string;
@@ -668,6 +700,11 @@ export type DraftTraceStep =
navigated_to?: string;
page_url?: string;
}
| {
op: "hover";
target: TargetDescriptor;
page_url?: string;
}
| {
op: "fill";
target: TargetDescriptor;
@@ -701,6 +738,7 @@ export type DraftTraceStep =
export type Step =
| ({ op: "navigate" } & StepCommon & { to: string })
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
| ({ op: "fill" } & StepCommon & {
target: TargetDescriptor;
value: string;
+11 -7
View File
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
```
bsk navigate <url> --session <id>
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
bsk snapshot --session <id> → again after navigation / DOM change
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
## Observation priority
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
1. `bsk snapshot` — strict static page understanding and interaction planning
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
2. `bsk snapshot` — strict static accessibility tree fallback
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
+87 -2
View File
@@ -10,8 +10,8 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
PressResult, SelectParams, SelectResult,
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk hover
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct HoverArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
/// Force-treat the value as a snapshot ref (overrides positional detection).
#[arg(long = "ref")]
pub ref_: Option<String>,
/// Force-treat the value as a CSS selector.
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
#[arg(long, default_value = "")]
pub modifiers: String,
/// Wait after moving the mouse so hover-triggered UI can settle.
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
pub settle: u32,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(
args.target.clone(),
args.ref_.clone(),
args.selector.clone(),
)?;
let modifiers = parse_modifiers(&args.modifiers)
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
let params = HoverParams {
session_id: args.session.clone(),
ref_,
selector,
tab_id: args.tab_id,
modifiers: if modifiers.is_empty() {
None
} else {
Some(modifiers)
},
settle_ms: Some(args.settle),
timeout_ms: Some(args.timeout),
};
let reply: HoverResult = call(
info.sock_path,
Method::ToolHover,
params,
"hover-1",
args.timeout,
)?;
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"hover ok tab={} target={target} at=({}, {})",
reply.tab_id, reply.x, reply.y
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk fill
// ---------------------------------------------------------------------------
+4 -1
View File
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
@@ -164,6 +164,9 @@ pub enum Command {
/// Click a snapshot ref or CSS selector.
Click(ClickArgs),
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
+5
View File
@@ -29,6 +29,10 @@ pub struct ObserveArgs {
/// Soft cap on rendered tokens (~4 chars/token).
#[arg(long = "max-tokens")]
pub max_tokens: Option<u32>,
/// Include conditional surface probe diagnostics in JSON output.
#[arg(long = "debug-surfaces")]
pub debug_surfaces: bool,
}
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
@@ -42,6 +46,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
tab_id: args.tab_id,
max_depth: args.max_depth,
max_tokens: args.max_tokens,
debug_surfaces: args.debug_surfaces,
};
let reply: ObserveResult = call(sock, params)?;
match format {
+1
View File
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
+1
View File
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
+18
View File
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
assert_eq!(args.click_count, 2);
}
#[test]
fn parses_hover_with_settle() {
let cli = parse(&[
"bsk",
"hover",
"@e1",
"--session",
"s1",
"--settle",
"300ms",
]);
let Command::Hover(args) = cli.command else {
panic!("expected hover command");
};
assert_eq!(args.target.as_deref(), Some("@e1"));
assert_eq!(args.settle, 300);
}
#[test]
fn rejects_zero_click_count() {
assert!(
+2
View File
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
tab_id: 13,
truncated: false,
dialogs: vec![],
debug: None,
})
.unwrap(),
)
@@ -492,6 +493,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
tab_id: None,
max_depth: None,
max_tokens: None,
debug_surfaces: false,
},
)
.await
@@ -0,0 +1,72 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "HoverParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"settle_ms": {
"description": "Wait after the mouse move so hover-triggered UI can settle.",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
},
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
}
}
}
@@ -0,0 +1,113 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "HoverResult",
"type": "object",
"required": [
"tab_id",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"x": {
"description": "Viewport-relative hover coordinates (CSS pixels).",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -7,6 +7,11 @@
"session_id"
],
"properties": {
"debug_surfaces": {
"description": "Include implementation diagnostics for conditional surface probes.",
"default": false,
"type": "boolean"
},
"max_depth": {
"description": "Cap the depth of the rendered VOM tree.",
"type": [
@@ -8,6 +8,16 @@
"text"
],
"properties": {
"debug": {
"anyOf": [
{
"$ref": "#/definitions/ObserveDebug"
},
{
"type": "null"
}
]
},
"dialogs": {
"type": "array",
"items": {
@@ -103,6 +113,73 @@
"prompt",
"beforeunload"
]
},
"ObserveDebug": {
"type": "object",
"properties": {
"surface_probes": {
"type": "array",
"items": {
"$ref": "#/definitions/SurfaceProbeDebug"
}
}
}
},
"SurfaceProbeDebug": {
"type": "object",
"required": [
"sub_items",
"trigger_action",
"trigger_backend_node_id"
],
"properties": {
"confidence": {
"type": [
"string",
"null"
]
},
"sub_items": {
"type": "array",
"items": {
"type": "string"
}
},
"trigger_action": {
"type": "string"
},
"trigger_backend_node_id": {
"type": "integer",
"format": "int64"
},
"trigger_point": {
"anyOf": [
{
"$ref": "#/definitions/SurfaceProbePoint"
},
{
"type": "null"
}
]
}
}
},
"SurfaceProbePoint": {
"type": "object",
"required": [
"x",
"y"
],
"properties": {
"x": {
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
}
}
}
}
@@ -144,6 +144,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -144,6 +144,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+40
View File
@@ -177,6 +177,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -83,6 +83,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -77,6 +77,8 @@ fn main() {
dump!(ClickParams, "tool_click_params");
dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params");
+6 -1
View File
@@ -70,6 +70,8 @@ pub enum Method {
ToolReload,
#[serde(rename = "tool.click")]
ToolClick,
#[serde(rename = "tool.hover")]
ToolHover,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.press")]
@@ -161,7 +163,7 @@ impl Method {
// Transient input — no committed browser action, but still page
// input. It must be stopped by pending user interrupts.
Method::ToolObserve => MethodEffect::TransientInput,
Method::ToolHover | Method::ToolObserve => MethodEffect::TransientInput,
// Passive reads — transparent.
// `record_stop` / `record_await` observe / finish a recording
@@ -259,6 +261,7 @@ mod tests {
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
assert!(!Method::ToolTabList.is_mutating());
assert!(!Method::ToolSnapshot.is_mutating());
assert!(!Method::ToolHover.is_mutating());
assert!(!Method::ToolObserve.is_mutating());
assert!(!Method::ToolGetHtml.is_mutating());
assert!(!Method::ToolScreenshot.is_mutating());
@@ -321,6 +324,7 @@ mod tests {
#[test]
fn effect_classifies_observe_as_transient_input() {
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
@@ -329,6 +333,7 @@ mod tests {
#[test]
fn interrupt_gate_includes_transient_input() {
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate());
@@ -85,6 +85,52 @@ pub struct ClickResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// hover
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct HoverParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub modifiers: Option<Vec<KeyModifier>>,
/// Wait after the mouse move so hover-triggered UI can settle.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 0))]
pub settle_ms: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct HoverResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Viewport-relative hover coordinates (CSS pixels).
pub x: f64,
pub y: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// fill
// ---------------------------------------------------------------------------
@@ -261,6 +307,24 @@ mod tests {
assert_eq!(p.ref_.as_deref(), Some("e1"));
}
#[test]
fn hover_params_serialise_ref_field_name() {
let p = HoverParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
modifiers: Some(vec![KeyModifier::Shift]),
settle_ms: Some(200),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(v.get("ref_").is_none());
let round: HoverParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
@@ -74,6 +74,32 @@ pub struct ObserveParams {
/// best-effort heuristic based on character count).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
/// Include implementation diagnostics for conditional surface probes.
#[serde(default)]
pub debug_surfaces: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SurfaceProbePoint {
pub x: f64,
pub y: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SurfaceProbeDebug {
pub trigger_backend_node_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trigger_point: Option<SurfaceProbePoint>,
pub trigger_action: String,
pub sub_items: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub confidence: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ObserveDebug {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub surface_probes: Vec<SurfaceProbeDebug>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -92,6 +118,8 @@ pub struct ObserveResult {
pub truncated: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub debug: Option<ObserveDebug>,
}
// ---------------------------------------------------------------------------
@@ -280,6 +308,7 @@ mod tests {
tab_id: 42,
truncated: false,
dialogs: Vec::new(),
debug: None,
};
let v = serde_json::to_value(&r).unwrap();
assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1));
+5
View File
@@ -94,6 +94,11 @@ pub enum Step {
common: StepCommon,
target: TargetDescriptor,
},
Hover {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
},
Fill {
#[serde(flatten)]
common: StepCommon,
+1 -1
View File
@@ -404,7 +404,7 @@ describe("renderVom single-layer page", () => {
],
});
expect(out.text).toContain('@e1 button "Products" [hover: Shoes | Bags | Accessories]');
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
+1 -1
View File
@@ -1,4 +1,4 @@
export { renderVom } from "./render";
export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render";
export type {
ActiveScopeBlock,
BlockingLayer,
+13 -11
View File
@@ -197,7 +197,7 @@ function shouldRender(node: VomNode): boolean {
return cleaned(node.name) !== undefined;
}
function shouldReference(node: VomNode): boolean {
export function isVomReferenceNode(node: VomNode): boolean {
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
}
@@ -342,7 +342,7 @@ function isAncestorOf(
return false;
}
function applyRecovery(nodes: VomNode[]): VomNode[] {
export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] {
const parentMap = buildParentMap(nodes);
const children = buildChildren(nodes);
const recoveredIds = new Set<number>();
@@ -350,7 +350,7 @@ function applyRecovery(nodes: VomNode[]): VomNode[] {
const recoveredNameById = new Map<number, string>();
for (const node of nodes) {
if (shouldReference(node)) continue;
if (isVomReferenceNode(node)) continue;
const role = recoveredRole(node, children);
if (!role || !passesRecoveryGuards(node, children)) continue;
recoveredIds.add(node.id);
@@ -397,7 +397,7 @@ function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
const counts = new Map<string, number>();
for (const node of nodes) {
if (!shouldReference(node)) continue;
if (!isVomReferenceNode(node)) continue;
const name = cleaned(node.name);
if (!name) continue;
const key = normalizeContextKey(name);
@@ -427,7 +427,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined {
}
function weakLabelContext(node: VomNode, targetName: string): string | undefined {
if (shouldReference(node)) return undefined;
if (isVomReferenceNode(node)) return undefined;
return weakContextText(
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
targetName,
@@ -509,7 +509,7 @@ function collectWeakLabelsFromSubtree(
state: RenderState,
labels: string[],
): void {
if (shouldReference(node)) return;
if (isVomReferenceNode(node)) return;
const label = weakLabelContext(node, nodeName);
if (label) {
labels.push(label);
@@ -637,7 +637,9 @@ function renderNodeLine(
if (surface && surface.subItems.length > 0) {
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
line += ` [${surface.triggerAction}: ${items}${suffix}]`;
const action =
surface.triggerAction === "hover" ? "hover first" : `${surface.triggerAction} first`;
line += ` [${action}: ${items}${suffix}]`;
}
return line;
@@ -664,7 +666,7 @@ function descendantTextCoveredByName(node: VomNode, state: RenderState): boolean
const texts: string[] = [];
while (stack.length > 0) {
const child = stack.pop() as VomNode;
if (shouldReference(child)) return false;
if (isVomReferenceNode(child)) return false;
const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value);
if (text) texts.push(text);
stack.push(...(state.children.get(child.id) ?? []));
@@ -748,7 +750,7 @@ function renderTree(
continue;
}
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined;
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
const context = ref ? handleContext(node, state) : [];
const surface = state.surfaceMap.get(node.id);
const line = renderNodeLine(node, depth, ref, context, surface);
@@ -913,7 +915,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
const blockedRoots = new Set<number>();
for (const target of nodes) {
if (!shouldReference(target)) continue;
if (!isVomReferenceNode(target)) continue;
const blocked = candidates.some((candidate) =>
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
);
@@ -968,7 +970,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
}
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
const nodes = applyRecovery(scene.nodes);
const nodes = applyVomInteractionRecovery(scene.nodes);
const renderScene = { ...scene, nodes };
const layer = detectBlockingLayer(nodes, scene.viewport);
if (layer) return renderDoubleLayer(renderScene, layer, options);
+11 -7
View File
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
```
bsk navigate <url> --session <id>
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
bsk snapshot --session <id> → again after navigation / DOM change
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
## Observation priority
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
1. `bsk snapshot` — strict static page understanding and interaction planning
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
2. `bsk snapshot` — strict static accessibility tree fallback
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |