mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
566b6c17fd | ||
|
|
2ccc0aafb9 | ||
|
|
eeab97bcdd | ||
|
|
6c610faed9 | ||
|
|
d685c75d25 | ||
|
|
73724617f9 | ||
|
|
4ba6fb45e1 | ||
|
|
9d4ac90592 | ||
|
|
cde6f22ecf | ||
|
|
d24bb6aa93 | ||
|
|
dbc4845c0d | ||
|
|
91225e5fdb |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
|
||||||
|
|
||||||
|
function mockRect(
|
||||||
|
el: Element,
|
||||||
|
rect: { left: number; top: number; width: number; height: number },
|
||||||
|
): void {
|
||||||
|
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
top: rect.top,
|
||||||
|
left: rect.left,
|
||||||
|
right: rect.left + rect.width,
|
||||||
|
bottom: rect.top + rect.height,
|
||||||
|
width: rect.width,
|
||||||
|
height: rect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record-hover-surface", () => {
|
||||||
|
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<ul class="header-menu tw-flex navbar-app__authentication-menu">
|
||||||
|
<li class="tg-popover">
|
||||||
|
<div class="tg-avatar tg-avatar--img"></div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div class="tg-popover__popper">
|
||||||
|
<div class="tg-popover__content">
|
||||||
|
<div class="tg-popover__content-body">
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
|
||||||
|
</li>
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
|
||||||
|
</li>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
|
||||||
|
).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat elements covered by an open surface as the surface owner", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button class="avatar">image</button>
|
||||||
|
<a class="fork" href="/fork">Fork</a>
|
||||||
|
<button class="clone">Clone & Download</button>
|
||||||
|
<div class="tg-popover__content-body"></div>
|
||||||
|
`;
|
||||||
|
const avatar = document.querySelector(".avatar")!;
|
||||||
|
const fork = document.querySelector(".fork")!;
|
||||||
|
const clone = document.querySelector(".clone")!;
|
||||||
|
const surface = document.querySelector(".tg-popover__content-body")!;
|
||||||
|
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
|
||||||
|
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
|
||||||
|
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
|
||||||
|
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
|
||||||
|
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat fixed navigation containers as plain floating surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<header class="navbar-app">
|
||||||
|
<a href="/fork">Fork</a>
|
||||||
|
<button>image</button>
|
||||||
|
</header>
|
||||||
|
<div class="plain-floating"></div>
|
||||||
|
`;
|
||||||
|
const navbar = document.querySelector(".navbar-app")!;
|
||||||
|
const floating = document.querySelector(".plain-floating")!;
|
||||||
|
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
|
||||||
|
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||||
|
const style = {
|
||||||
|
cursor: "",
|
||||||
|
display: "block",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
position: el === navbar ? "fixed" : "absolute",
|
||||||
|
visibility: "visible",
|
||||||
|
} as CSSStyleDeclaration;
|
||||||
|
return style;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,17 @@
|
|||||||
import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
|
import {
|
||||||
|
describeEventTarget,
|
||||||
|
describeTarget,
|
||||||
|
resolveClickableElement,
|
||||||
|
resolveHoverElement,
|
||||||
|
type TargetDescriptor,
|
||||||
|
} from "@/lib/describe-target";
|
||||||
|
import {
|
||||||
|
evaluateHoverTrigger,
|
||||||
|
type HoverTriggerDecision,
|
||||||
|
type HoverTriggerRect,
|
||||||
|
hasDirectHoverInteractiveSignal,
|
||||||
|
hasStrongHoverExpansionSignal,
|
||||||
|
} from "@/lib/hover-trigger-policy";
|
||||||
import {
|
import {
|
||||||
isRecordCancelMessage,
|
isRecordCancelMessage,
|
||||||
isRecordStartMessage,
|
isRecordStartMessage,
|
||||||
@@ -15,6 +28,14 @@ import {
|
|||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||||
|
import {
|
||||||
|
closestHoverSurfaceCandidate,
|
||||||
|
collectHoverSurfaceStates,
|
||||||
|
decideHoverSurfaceRelation,
|
||||||
|
type HoverSurfaceState,
|
||||||
|
isHoverSurfaceCandidateElement,
|
||||||
|
isLikelyHoverSurfaceOwner,
|
||||||
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
const failedStepDeliveries = new Set<string>();
|
const failedStepDeliveries = new Set<string>();
|
||||||
@@ -49,8 +70,29 @@ interface FillSession {
|
|||||||
lastValue: string;
|
lastValue: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface HoverCandidate {
|
||||||
|
element: Element;
|
||||||
|
target: TargetDescriptor;
|
||||||
|
recordedAt: number;
|
||||||
|
score: number;
|
||||||
|
eligible: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HoverSurfaceNode {
|
||||||
|
element: Element;
|
||||||
|
signature: string;
|
||||||
|
owner: HoverCandidate;
|
||||||
|
parent?: HoverSurfaceNode;
|
||||||
|
}
|
||||||
|
|
||||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||||
|
|
||||||
|
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||||
|
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
|
||||||
|
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||||
|
const HOVER_CANDIDATE_LIMIT = 24;
|
||||||
|
const HOVER_TRIGGER_LABEL_MAX = 48;
|
||||||
|
|
||||||
function eventTarget(event: Event): EventTarget | null {
|
function eventTarget(event: Event): EventTarget | null {
|
||||||
return event.composedPath()[0] ?? event.target;
|
return event.composedPath()[0] ?? event.target;
|
||||||
}
|
}
|
||||||
@@ -122,6 +164,173 @@ function fillableValue(el: FillableElement): string {
|
|||||||
return el.textContent ?? "";
|
return el.textContent ?? "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
|
||||||
|
return {
|
||||||
|
id: el.id || undefined,
|
||||||
|
class: typeof el.className === "string" ? el.className || undefined : undefined,
|
||||||
|
"data-testid": el.getAttribute("data-testid") ?? undefined,
|
||||||
|
"data-test": el.getAttribute("data-test") ?? undefined,
|
||||||
|
"data-cy": el.getAttribute("data-cy") ?? undefined,
|
||||||
|
"aria-label": el.getAttribute("aria-label") ?? undefined,
|
||||||
|
"aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
|
||||||
|
"aria-controls": el.getAttribute("aria-controls") ?? undefined,
|
||||||
|
"aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
|
||||||
|
"aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
|
||||||
|
"aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
|
||||||
|
contenteditable: el.getAttribute("contenteditable") ?? undefined,
|
||||||
|
disabled: el.hasAttribute("disabled") ? "" : undefined,
|
||||||
|
hidden: el.hasAttribute("hidden") ? "" : undefined,
|
||||||
|
inert: el.hasAttribute("inert") ? "" : undefined,
|
||||||
|
onclick: el.getAttribute("onclick") ?? undefined,
|
||||||
|
onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
|
||||||
|
onmouseover: el.getAttribute("onmouseover") ?? undefined,
|
||||||
|
role: el.getAttribute("role") ?? undefined,
|
||||||
|
tabindex: el.getAttribute("tabindex") ?? undefined,
|
||||||
|
title: el.getAttribute("title") ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerRect(el: Element): HoverTriggerRect {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
|
||||||
|
if (!(el instanceof HTMLElement)) return {};
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
return { cursor: style.cursor, pointerEvents: style.pointerEvents };
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasHoverGraphicDescendant(el: Element): boolean {
|
||||||
|
return el.querySelector("img,svg,use,path,i") !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeLabelText(value: string): string {
|
||||||
|
return value.replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string {
|
||||||
|
const normalized = normalizeLabelText(value);
|
||||||
|
if (normalized.length <= max) return normalized;
|
||||||
|
return `${normalized.slice(0, max - 1)}…`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectHoverTriggerLabelText(root: Element): string {
|
||||||
|
let text = "";
|
||||||
|
const visit = (node: Node) => {
|
||||||
|
if (node.nodeType === Node.TEXT_NODE) {
|
||||||
|
text += ` ${node.textContent ?? ""}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!(node instanceof Element)) return;
|
||||||
|
if (node !== root && isHoverSurfaceCandidateElement(node)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const child of node.childNodes) visit(child);
|
||||||
|
};
|
||||||
|
for (const child of root.childNodes) visit(child);
|
||||||
|
return normalizeLabelText(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||||
|
if (!desc.name) return desc;
|
||||||
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
|
if (!compactName || compactName === desc.name) return desc;
|
||||||
|
const compactComparable = compactName.replace(/\s+/g, "");
|
||||||
|
const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, "");
|
||||||
|
const fullTextComparable = fullText.replace(/\s+/g, "");
|
||||||
|
if (
|
||||||
|
descComparable.startsWith(compactComparable) &&
|
||||||
|
fullTextComparable.startsWith(descComparable) &&
|
||||||
|
compactName.length < desc.name.length
|
||||||
|
) {
|
||||||
|
return { ...desc, name: truncateLabel(compactName) };
|
||||||
|
}
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||||
|
return !target.role && !target.name && target.tag === "div";
|
||||||
|
}
|
||||||
|
|
||||||
|
function looksLikeAvatarElement(el: Element): boolean {
|
||||||
|
const className = typeof el.className === "string" ? el.className : "";
|
||||||
|
return /\b(avatar|user-avatar)\b/i.test(className);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeHoverTarget(
|
||||||
|
el: Element,
|
||||||
|
desc: TargetDescriptor,
|
||||||
|
decision: HoverTriggerDecision,
|
||||||
|
): TargetDescriptor {
|
||||||
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
|
return { ...desc, name: "image" };
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
isWeakHoverTarget(desc) &&
|
||||||
|
(looksLikeAvatarElement(el) ||
|
||||||
|
(hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
|
||||||
|
) {
|
||||||
|
return { tag: "img", role: "img", name: "image" };
|
||||||
|
}
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||||
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
|
const style = hoverTriggerStyle(el);
|
||||||
|
return {
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
role: desc.role,
|
||||||
|
label: desc.name,
|
||||||
|
attrs: hoverTriggerAttrs(el),
|
||||||
|
rect: hoverTriggerRect(el),
|
||||||
|
cursor: style.cursor,
|
||||||
|
pointerEvents: style.pointerEvents,
|
||||||
|
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
|
||||||
|
if (!(target instanceof Element)) return null;
|
||||||
|
const element = resolveHoverElement(target);
|
||||||
|
if (!element) return null;
|
||||||
|
const desc = describeTarget(element);
|
||||||
|
const signals = hoverTriggerSignals(element, desc);
|
||||||
|
if (!signals) return null;
|
||||||
|
const decision = evaluateHoverTrigger(signals);
|
||||||
|
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
|
||||||
|
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
|
||||||
|
if (!decision.eligible && !desc.name && !desc.role) return null;
|
||||||
|
const normalizedTarget = normalizeHoverTarget(
|
||||||
|
element,
|
||||||
|
compactHoverTargetName(element, desc),
|
||||||
|
decision,
|
||||||
|
);
|
||||||
|
const now = Date.now();
|
||||||
|
return {
|
||||||
|
element,
|
||||||
|
target: normalizedTarget,
|
||||||
|
recordedAt: now,
|
||||||
|
score: decision.score,
|
||||||
|
eligible: decision.eligible,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldReplaceHoverCandidate(
|
||||||
|
current: HoverCandidate,
|
||||||
|
next: HoverCandidate,
|
||||||
|
now: number,
|
||||||
|
): boolean {
|
||||||
|
if (next.element === current.element) return false;
|
||||||
|
if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
|
||||||
|
if (current.element.contains(next.element)) return false;
|
||||||
|
if (next.element.contains(current.element)) return true;
|
||||||
|
return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
|
||||||
|
}
|
||||||
|
|
||||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||||
if (!session || !(target instanceof Element)) return false;
|
if (!session || !(target instanceof Element)) return false;
|
||||||
@@ -168,10 +377,17 @@ export function startRecordCapture(
|
|||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
};
|
};
|
||||||
|
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||||
|
new Map(states.map((state) => [state.element, state.signature]));
|
||||||
let fillSession: FillSession | null = null;
|
let fillSession: FillSession | null = null;
|
||||||
let composing = false;
|
let composing = false;
|
||||||
let lastUrl = location.href;
|
let lastUrl = location.href;
|
||||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||||
|
let pendingHover: HoverCandidate | null = null;
|
||||||
|
const recentHoverCandidates: HoverCandidate[] = [];
|
||||||
|
const emittedHoverElements = new WeakSet<Element>();
|
||||||
|
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
|
||||||
|
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
|
||||||
let generatedControlClick: Element | null = null;
|
let generatedControlClick: Element | null = null;
|
||||||
let navigationActionPending = false;
|
let navigationActionPending = false;
|
||||||
let navigationActionVersion = 0;
|
let navigationActionVersion = 0;
|
||||||
@@ -237,6 +453,223 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const rememberHoverCandidate = (hover: HoverCandidate) => {
|
||||||
|
const duplicate = recentHoverCandidates.findIndex(
|
||||||
|
(candidate) => candidate.element === hover.element,
|
||||||
|
);
|
||||||
|
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
|
||||||
|
recentHoverCandidates.push(hover);
|
||||||
|
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
|
||||||
|
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const surfaceArea = (el: Element): number => {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return rect.width * rect.height;
|
||||||
|
};
|
||||||
|
|
||||||
|
const smallestContaining = <T>(
|
||||||
|
target: Element,
|
||||||
|
items: Iterable<T>,
|
||||||
|
elementFor: (item: T) => Element,
|
||||||
|
): T | undefined => {
|
||||||
|
let best: T | undefined;
|
||||||
|
for (const item of items) {
|
||||||
|
const element = elementFor(item);
|
||||||
|
if (!element.contains(target)) continue;
|
||||||
|
if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) {
|
||||||
|
best = item;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined =>
|
||||||
|
smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element);
|
||||||
|
|
||||||
|
const surfaceStateContaining = (
|
||||||
|
target: Element,
|
||||||
|
states: HoverSurfaceState[],
|
||||||
|
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
|
||||||
|
|
||||||
|
const inferOwnerForSurface = (
|
||||||
|
surface: Element,
|
||||||
|
now: number,
|
||||||
|
before?: HoverCandidate,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (before && candidate.element === before.element) continue;
|
||||||
|
if (before && candidate.recordedAt > before.recordedAt) continue;
|
||||||
|
if (surface.contains(candidate.element)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
const nodeForUnownedSurface = (
|
||||||
|
surfaceState: HoverSurfaceState,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const owner = inferOwnerForSurface(surfaceState.element, now);
|
||||||
|
if (!owner) return undefined;
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: surfaceState.element,
|
||||||
|
signature: surfaceState.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(surfaceState.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
|
||||||
|
const closestSurface = closestHoverSurfaceCandidate(target);
|
||||||
|
if (closestSurface) {
|
||||||
|
const owned = hoverSurfaceNodes.get(closestSurface);
|
||||||
|
if (owned) return owned;
|
||||||
|
}
|
||||||
|
const ownedContaining = ownedSurfaceContaining(target);
|
||||||
|
if (ownedContaining) return ownedContaining;
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
const surfaceState = closestSurface
|
||||||
|
? currentStates.find((state) => state.element === closestSurface)
|
||||||
|
: surfaceStateContaining(target, currentStates);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
if (!surfaceState) return undefined;
|
||||||
|
return nodeForUnownedSurface(surfaceState, currentStates, now);
|
||||||
|
};
|
||||||
|
|
||||||
|
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
|
||||||
|
const current = new Set(currentStates.map((state) => state.element));
|
||||||
|
for (const element of hoverSurfaceNodes.keys()) {
|
||||||
|
if (!current.has(element)) hoverSurfaceNodes.delete(element);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const parentSurfaceNodeForOwner = (
|
||||||
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const ownedParent = ownedSurfaceContaining(owner.element);
|
||||||
|
if (ownedParent) return ownedParent;
|
||||||
|
const parentState = surfaceStateContaining(owner.element, currentStates);
|
||||||
|
if (!parentState) return undefined;
|
||||||
|
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
|
||||||
|
if (!parentOwner) return undefined;
|
||||||
|
const parentNode: HoverSurfaceNode = {
|
||||||
|
element: parentState.element,
|
||||||
|
signature: parentState.signature,
|
||||||
|
owner: parentOwner,
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(parentState.element, parentNode);
|
||||||
|
return parentNode;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createSurfaceNode = (
|
||||||
|
state: HoverSurfaceState,
|
||||||
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
if (parent?.element === state.element) return undefined;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
|
||||||
|
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: state.element,
|
||||||
|
signature: state.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(state.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const bindChangedHoverSurfaces = (
|
||||||
|
previousStates: Map<Element, string>,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
) => {
|
||||||
|
for (const state of currentStates) {
|
||||||
|
if (previousStates.get(state.element) === state.signature) continue;
|
||||||
|
const owner = inferOwnerForSurface(state.element, now);
|
||||||
|
if (!owner) continue;
|
||||||
|
createSurfaceNode(state, owner, currentStates, now);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
bindChangedHoverSurfaces(previousStates, currentStates, now);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
};
|
||||||
|
|
||||||
|
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
|
||||||
|
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitHoverStep = (hover: HoverCandidate) => {
|
||||||
|
if (emittedHoverElements.has(hover.element)) return;
|
||||||
|
emitStep({
|
||||||
|
op: "hover",
|
||||||
|
target: hover.target,
|
||||||
|
});
|
||||||
|
emittedHoverElements.add(hover.element);
|
||||||
|
};
|
||||||
|
|
||||||
|
const surfaceOwnerChain = (
|
||||||
|
surface: HoverSurfaceNode,
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate[] => {
|
||||||
|
const chain: HoverCandidate[] = [];
|
||||||
|
const selected = new WeakSet<Element>();
|
||||||
|
const surfaces: HoverSurfaceNode[] = [];
|
||||||
|
let node: HoverSurfaceNode | undefined = surface;
|
||||||
|
while (node && surfaces.length < 4) {
|
||||||
|
surfaces.unshift(node);
|
||||||
|
node = node.parent;
|
||||||
|
}
|
||||||
|
for (const owned of surfaces) {
|
||||||
|
const owner = owned.owner;
|
||||||
|
if (owner.element === actionElement || actionElement.contains(owner.element)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (selected.has(owner.element)) continue;
|
||||||
|
if (emittedHoverElements.has(owner.element)) continue;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
selected.add(owner.element);
|
||||||
|
chain.push(owner);
|
||||||
|
}
|
||||||
|
return chain;
|
||||||
|
};
|
||||||
|
|
||||||
|
const containedHoverOwnerForAction = (
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (candidate.element === actionElement) continue;
|
||||||
|
if (!candidate.element.contains(actionElement)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
const emitClick = (event: MouseEvent) => {
|
const emitClick = (event: MouseEvent) => {
|
||||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||||
const target = describeEventTarget(eventTarget(event));
|
const target = describeEventTarget(eventTarget(event));
|
||||||
@@ -249,6 +682,54 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
|
||||||
|
if (!(actionTarget instanceof Element)) return;
|
||||||
|
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
|
||||||
|
const surface = ownedSurfaceForAction(actionElement);
|
||||||
|
const now = Date.now();
|
||||||
|
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
|
||||||
|
const hoverChain = surface
|
||||||
|
? surfaceOwnerChain(surface, actionElement, now)
|
||||||
|
: containedOwner
|
||||||
|
? [containedOwner]
|
||||||
|
: [];
|
||||||
|
if (hoverChain.length === 0) {
|
||||||
|
pendingHover = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const hover of hoverChain) emitHoverStep(hover);
|
||||||
|
pendingHover = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const onMouseOver = (event: MouseEvent) => {
|
||||||
|
const target = eventTarget(event);
|
||||||
|
if (isOverlayTarget(target)) return;
|
||||||
|
const now = Date.now();
|
||||||
|
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
||||||
|
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||||
|
pendingHover = null;
|
||||||
|
}
|
||||||
|
const candidate = hoverCandidateFromEvent(target);
|
||||||
|
if (candidate) {
|
||||||
|
const previousSurfaceStates = new Map(hoverSurfaceStates);
|
||||||
|
rememberHoverCandidate(candidate);
|
||||||
|
refreshHoverSurfaces(previousSurfaceStates);
|
||||||
|
scheduleHoverSurfaceRefresh(previousSurfaceStates);
|
||||||
|
}
|
||||||
|
const hover = candidate?.eligible ? candidate : null;
|
||||||
|
if (pendingHover && target instanceof Element) {
|
||||||
|
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
|
||||||
|
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!hover) return;
|
||||||
|
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pendingHover = hover;
|
||||||
|
};
|
||||||
|
|
||||||
const onClick = (event: MouseEvent) => {
|
const onClick = (event: MouseEvent) => {
|
||||||
if (event.button !== 0) return;
|
if (event.button !== 0) return;
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
@@ -278,6 +759,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
const fillable = fillableFromTarget(target);
|
const fillable = fillableFromTarget(target);
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
|
emitHoverCandidateBeforeAction(fillable);
|
||||||
ensureFillSession(fillable);
|
ensureFillSession(fillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -285,6 +767,7 @@ export function startRecordCapture(
|
|||||||
if (target instanceof Element) {
|
if (target instanceof Element) {
|
||||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||||
if (nearbyFillable) {
|
if (nearbyFillable) {
|
||||||
|
emitHoverCandidateBeforeAction(nearbyFillable);
|
||||||
ensureFillSession(nearbyFillable);
|
ensureFillSession(nearbyFillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -304,6 +787,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
if (target instanceof Element && target.closest("select")) return;
|
if (target instanceof Element && target.closest("select")) return;
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitClick(event);
|
emitClick(event);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -345,6 +829,7 @@ export function startRecordCapture(
|
|||||||
commitFillSession();
|
commitFillSession();
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
if (target instanceof HTMLSelectElement) {
|
if (target instanceof HTMLSelectElement) {
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||||
(opt.label || opt.textContent || opt.value).trim(),
|
(opt.label || opt.textContent || opt.value).trim(),
|
||||||
@@ -398,6 +883,7 @@ export function startRecordCapture(
|
|||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
@@ -408,6 +894,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
|
|
||||||
document.addEventListener("click", onClick, true);
|
document.addEventListener("click", onClick, true);
|
||||||
|
document.addEventListener("mouseover", onMouseOver, true);
|
||||||
document.addEventListener("focusin", onFocusIn, true);
|
document.addEventListener("focusin", onFocusIn, true);
|
||||||
document.addEventListener("focusout", onFocusOut, true);
|
document.addEventListener("focusout", onFocusOut, true);
|
||||||
document.addEventListener("input", onInput, true);
|
document.addEventListener("input", onInput, true);
|
||||||
@@ -438,6 +925,7 @@ export function startRecordCapture(
|
|||||||
dispose() {
|
dispose() {
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
document.removeEventListener("click", onClick, true);
|
document.removeEventListener("click", onClick, true);
|
||||||
|
document.removeEventListener("mouseover", onMouseOver, true);
|
||||||
document.removeEventListener("focusin", onFocusIn, true);
|
document.removeEventListener("focusin", onFocusIn, true);
|
||||||
document.removeEventListener("focusout", onFocusOut, true);
|
document.removeEventListener("focusout", onFocusOut, true);
|
||||||
document.removeEventListener("input", onInput, true);
|
document.removeEventListener("input", onInput, true);
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
export interface HoverSurfaceDecision {
|
||||||
|
related: boolean;
|
||||||
|
reason?: string;
|
||||||
|
surface?: Element;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverSurfaceContext {
|
||||||
|
triggerElement: Element;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverSurfaceState {
|
||||||
|
element: Element;
|
||||||
|
signature: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOVER_SURFACE_SELECTOR = [
|
||||||
|
'[role="menu"]',
|
||||||
|
'[role="menubar"]',
|
||||||
|
'[role="listbox"]',
|
||||||
|
'[role="dialog"]',
|
||||||
|
'[role="tooltip"]',
|
||||||
|
'[aria-modal="true"]',
|
||||||
|
"[data-popper-placement]",
|
||||||
|
"[data-floating-ui-placement]",
|
||||||
|
"[data-headlessui-state]",
|
||||||
|
"[data-radix-popper-content-wrapper]",
|
||||||
|
].join(",");
|
||||||
|
|
||||||
|
const HOVER_SURFACE_WORD_RE =
|
||||||
|
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
|
||||||
|
const HOVER_SURFACE_ITEM_RE =
|
||||||
|
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
|
||||||
|
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
||||||
|
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
|
||||||
|
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
||||||
|
|
||||||
|
function elementTokenText(el: Element): string {
|
||||||
|
return [
|
||||||
|
el.id,
|
||||||
|
typeof el.className === "string" ? el.className : "",
|
||||||
|
el.getAttribute("data-testid"),
|
||||||
|
el.getAttribute("data-test"),
|
||||||
|
el.getAttribute("data-cy"),
|
||||||
|
el.getAttribute("aria-label"),
|
||||||
|
el.getAttribute("role"),
|
||||||
|
]
|
||||||
|
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecognizedHoverSurfaceElement(el: Element): boolean {
|
||||||
|
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
|
||||||
|
const tokenText = elementTokenText(el);
|
||||||
|
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isVisibleSurfaceElement(el: Element): boolean {
|
||||||
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return rectFor(el) !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestRecognizedHoverSurface(element: Element): Element | null {
|
||||||
|
let node: Element | null = element;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isRecognizedHoverSurfaceElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPositionedFloatingElement(el: Element): boolean {
|
||||||
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (style.position !== "absolute") return false;
|
||||||
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const rect = rectFor(el);
|
||||||
|
if (!rect) return false;
|
||||||
|
const viewportArea = window.innerWidth * window.innerHeight;
|
||||||
|
const area = rect.width * rect.height;
|
||||||
|
return (
|
||||||
|
area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHoverSurfaceCandidateElement(el: Element): boolean {
|
||||||
|
return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestHoverSurfaceCandidate(element: Element): Element | null {
|
||||||
|
let node: Element | null = element;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isHoverSurfaceCandidateElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function closestFloatingSurface(clickElement: Element): Element | null {
|
||||||
|
let node: Element | null = clickElement;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isPositionedFloatingElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverSurfaceSignature(el: Element): string {
|
||||||
|
const rect = rectFor(el);
|
||||||
|
return [
|
||||||
|
rect
|
||||||
|
? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}`
|
||||||
|
: "",
|
||||||
|
(el.textContent ?? "").trim().slice(0, 512),
|
||||||
|
].join("|");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function collectHoverSurfaceStates(): HoverSurfaceState[] {
|
||||||
|
const states: HoverSurfaceState[] = [];
|
||||||
|
for (const el of document.querySelectorAll("*")) {
|
||||||
|
if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) {
|
||||||
|
states.push({ element: el, signature: hoverSurfaceSignature(el) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return states;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectFor(el: Element): DOMRect | null {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 || rect.height <= 0) return null;
|
||||||
|
return rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
function surfaceRelationMetrics(surface: Element, trigger: Element) {
|
||||||
|
const surfaceRect = rectFor(surface);
|
||||||
|
const triggerRect = rectFor(trigger);
|
||||||
|
if (!surfaceRect || !triggerRect) return null;
|
||||||
|
const horizontalOverlap =
|
||||||
|
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
|
||||||
|
const verticalOverlap =
|
||||||
|
Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top);
|
||||||
|
const horizontalGap = Math.max(
|
||||||
|
0,
|
||||||
|
triggerRect.left - surfaceRect.right,
|
||||||
|
surfaceRect.left - triggerRect.right,
|
||||||
|
);
|
||||||
|
const verticalGap = Math.max(
|
||||||
|
0,
|
||||||
|
triggerRect.top - surfaceRect.bottom,
|
||||||
|
surfaceRect.top - triggerRect.bottom,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isNearTrigger(surface: Element, trigger: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
|
||||||
|
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
|
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
|
return (
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
} = metrics;
|
||||||
|
const opensVertically =
|
||||||
|
horizontalOverlap > 0 &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
|
||||||
|
const opensSideways =
|
||||||
|
verticalOverlap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
|
||||||
|
const opensFromCorner =
|
||||||
|
horizontalGap > 0 &&
|
||||||
|
verticalGap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
|
||||||
|
return opensVertically || opensSideways || opensFromCorner;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decideHoverSurfaceRelation(
|
||||||
|
context: HoverSurfaceContext,
|
||||||
|
clickElement: Element,
|
||||||
|
): HoverSurfaceDecision {
|
||||||
|
const clickSurface = closestRecognizedHoverSurface(clickElement);
|
||||||
|
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
||||||
|
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||||
|
}
|
||||||
|
|
||||||
|
const floatingSurface = closestFloatingSurface(clickElement);
|
||||||
|
if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) {
|
||||||
|
return {
|
||||||
|
related: true,
|
||||||
|
reason: "click-inside-near-floating-surface",
|
||||||
|
surface: floatingSurface,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
|
||||||
|
return { related: true, reason: "click-inside-trigger-contained-surface" };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { related: false, reason: "click-outside-hover-surface" };
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { evaluateHoverTrigger } from "../hover-trigger-policy";
|
||||||
|
|
||||||
|
describe("evaluateHoverTrigger", () => {
|
||||||
|
it("accepts explicit popup triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "Open user navigation menu",
|
||||||
|
attrs: { "aria-haspopup": "menu", "aria-expanded": "false" },
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(
|
||||||
|
expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts compact topbar image buttons without popup attributes", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "image",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
hasGraphicDescendant: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts compact topbar image links without popup attributes", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "image",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
hasGraphicDescendant: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects topbar menu links without surface trigger evidence", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "My profile",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects text navigation items even when their attributes contain popup words", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "My profile",
|
||||||
|
attrs: { class: "dropdown-item profile-link" },
|
||||||
|
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects labelled dropdown list items as hover triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "li",
|
||||||
|
label: "文档C+D",
|
||||||
|
attrs: { class: "dropdown-item", tabindex: "0" },
|
||||||
|
rect: { x: 820, y: 48, w: 160, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects ordinary controls without hover popup signals", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "Search",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 100, y: 240, w: 120, h: 36 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects unsafe or non-visible triggers", () => {
|
||||||
|
expect(
|
||||||
|
evaluateHoverTrigger({
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
}).eligible,
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
attrs: { hidden: "" },
|
||||||
|
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
}).eligible,
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "span", role: "button", name: "Account" },
|
||||||
|
page_url: "https://example.com/app",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "a", role: "link", name: "Profile" },
|
||||||
|
page_url: "https://example.com/app",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"https://example.com/app",
|
||||||
|
);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
page: "p1",
|
||||||
|
target: { name: "Account" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
resolveTraceStartUrl(
|
||||||
|
|||||||
@@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveHoverElement(target: Element): Element | null {
|
||||||
|
const clickable = resolveClickableElement(target);
|
||||||
|
if (clickable) return clickable;
|
||||||
|
|
||||||
|
let node: Element | null = target;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
|
||||||
|
if (looksClickable(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
export interface HoverTriggerRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
w: number;
|
||||||
|
h: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverTriggerSignals {
|
||||||
|
tag: string;
|
||||||
|
role?: string;
|
||||||
|
label?: string;
|
||||||
|
attrs?: Record<string, string | undefined>;
|
||||||
|
rect?: HoverTriggerRect | null;
|
||||||
|
cursor?: string;
|
||||||
|
pointerEvents?: string;
|
||||||
|
hasGraphicDescendant?: boolean;
|
||||||
|
cssHoverMatch?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverTriggerDecision {
|
||||||
|
eligible: boolean;
|
||||||
|
score: number;
|
||||||
|
reasons: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOVER_TRIGGER_MIN_SCORE = 45;
|
||||||
|
const HOVER_POPUP_SIGNAL_RE =
|
||||||
|
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
||||||
|
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
|
||||||
|
|
||||||
|
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
||||||
|
return signals.attrs?.[name.toLowerCase()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean {
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||||
|
if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true;
|
||||||
|
if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true;
|
||||||
|
return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
const attrs = signals.attrs ?? {};
|
||||||
|
const haystack = [
|
||||||
|
attrs.id,
|
||||||
|
attrs.class,
|
||||||
|
attrs["data-testid"],
|
||||||
|
attrs["data-test"],
|
||||||
|
attrs["data-cy"],
|
||||||
|
attrs["aria-label"],
|
||||||
|
attrs["aria-haspopup"],
|
||||||
|
attrs["aria-controls"],
|
||||||
|
attrs.title,
|
||||||
|
signals.role === "button" ? signals.label : undefined,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ")
|
||||||
|
.toLowerCase();
|
||||||
|
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
return (
|
||||||
|
signals.cssHoverMatch === true ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
|
return (
|
||||||
|
["button", "a", "summary"].includes(tag) ||
|
||||||
|
["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
|
||||||
|
attr(signals, "tabindex") !== undefined ||
|
||||||
|
signals.cursor === "pointer" ||
|
||||||
|
attr(signals, "onclick") !== undefined ||
|
||||||
|
attr(signals, "onmouseenter") !== undefined ||
|
||||||
|
attr(signals, "onmouseover") !== undefined ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
attr(signals, "aria-expanded") === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision {
|
||||||
|
const rect = signals.rect;
|
||||||
|
if (
|
||||||
|
!rect ||
|
||||||
|
rect.w <= 0 ||
|
||||||
|
rect.h <= 0 ||
|
||||||
|
signals.pointerEvents === "none" ||
|
||||||
|
attr(signals, "hidden") !== undefined ||
|
||||||
|
attr(signals, "inert") !== undefined ||
|
||||||
|
(attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" ||
|
||||||
|
isUnsafeHoverTrigger(signals)
|
||||||
|
) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const area = rect.w * rect.h;
|
||||||
|
if (area <= 0 || area > 160_000) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const reasons: string[] = [];
|
||||||
|
let score = 0;
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
|
const label = signals.label?.trim();
|
||||||
|
const graphic = signals.hasGraphicDescendant === true;
|
||||||
|
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
const popupSignal = hasHoverPopupSignal(signals);
|
||||||
|
const isTextNavigationItem =
|
||||||
|
(tag === "a" ||
|
||||||
|
tag === "li" ||
|
||||||
|
role === "link" ||
|
||||||
|
role === "option" ||
|
||||||
|
role.startsWith("menuitem") ||
|
||||||
|
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
|
||||||
|
!graphic &&
|
||||||
|
!!label &&
|
||||||
|
label.toLowerCase() !== "image";
|
||||||
|
const compactTopbarIcon =
|
||||||
|
(role === "button" || role === "link" || tag === "button" || tag === "a") &&
|
||||||
|
graphic &&
|
||||||
|
rect.y < 120 &&
|
||||||
|
rect.w <= 96 &&
|
||||||
|
rect.h <= 96 &&
|
||||||
|
(!label || label.toLowerCase() === "image");
|
||||||
|
const surfaceTriggerEvidence =
|
||||||
|
(popupSignal && !isTextNavigationItem) ||
|
||||||
|
compactTopbarIcon ||
|
||||||
|
hasStrongHoverExpansionSignal(signals);
|
||||||
|
if (!directInteractive || !surfaceTriggerEvidence) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attr(signals, "aria-haspopup") !== undefined) {
|
||||||
|
score += 80;
|
||||||
|
reasons.push("aria-haspopup");
|
||||||
|
}
|
||||||
|
if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") {
|
||||||
|
score += 55;
|
||||||
|
reasons.push("collapsed");
|
||||||
|
}
|
||||||
|
if (popupSignal) {
|
||||||
|
score += 45;
|
||||||
|
reasons.push("popup-signal");
|
||||||
|
}
|
||||||
|
if (tag === "button" || role === "button") {
|
||||||
|
score += 30;
|
||||||
|
reasons.push("button");
|
||||||
|
}
|
||||||
|
if (signals.cursor === "pointer") {
|
||||||
|
score += 25;
|
||||||
|
reasons.push("pointer");
|
||||||
|
}
|
||||||
|
if (graphic && (!label || label.toLowerCase() === "image")) {
|
||||||
|
score += 40;
|
||||||
|
reasons.push("icon-only");
|
||||||
|
}
|
||||||
|
if (rect.y < 120) {
|
||||||
|
score += 25;
|
||||||
|
reasons.push("topbar");
|
||||||
|
}
|
||||||
|
if (rect.w <= 80 && rect.h <= 80) {
|
||||||
|
score += 20;
|
||||||
|
reasons.push("compact");
|
||||||
|
}
|
||||||
|
if (signals.cssHoverMatch) {
|
||||||
|
score += 40;
|
||||||
|
reasons.push("css-hover");
|
||||||
|
}
|
||||||
|
|
||||||
|
return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons };
|
||||||
|
}
|
||||||
@@ -33,7 +33,7 @@ export interface RecordStartMessage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
|
|||||||
@@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
|
case "hover":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "hover",
|
||||||
|
target: payload.target,
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
case "fill":
|
case "fill":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
|
|||||||
@@ -139,6 +139,13 @@ function toV2Step(
|
|||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigated_to, urlToId),
|
||||||
);
|
);
|
||||||
|
case "hover":
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
id,
|
||||||
|
page,
|
||||||
|
target: step.target,
|
||||||
|
};
|
||||||
case "fill":
|
case "fill":
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import type {
|
|||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
|
|
||||||
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
function fakeTransport() {
|
function fakeTransport() {
|
||||||
const handlers = new Set<FrameHandler>();
|
const handlers = new Set<FrameHandler>();
|
||||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||||
@@ -129,7 +131,7 @@ describe("ToolDispatcher", () => {
|
|||||||
setUserAgentOverride: vi.fn(async () => {}),
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
||||||
@@ -172,7 +174,7 @@ describe("ToolDispatcher", () => {
|
|||||||
setUserAgentOverride: vi.fn(async () => {}),
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
@@ -311,6 +313,277 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const order: string[] = [];
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => {
|
||||||
|
order.push("hover");
|
||||||
|
return {} as T;
|
||||||
|
}),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
|
||||||
|
(
|
||||||
|
dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
|
||||||
|
).rememberHover("aa11", {
|
||||||
|
tab_id: 7,
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options?: { releaseAfter?: boolean },
|
||||||
|
) => Promise<T>;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
|
||||||
|
order.push("observe");
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
expect(order).toEqual(["hover", "observe"]);
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
await helpers.withHoverReassert(
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
async () => {
|
||||||
|
order.push("click");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
{ releaseAfter: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reassert or disable hover latches from another session", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options?: { releaseAfter?: boolean },
|
||||||
|
) => Promise<T>;
|
||||||
|
releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
|
||||||
|
releaseAfter: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.releaseHoverLatch("aa11", 7);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
await helpers.setHoverBypass("bb22", 7, true);
|
||||||
|
await helpers.setHoverBypass("aa11", 7, false);
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("bb22", 7, false);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("limits default-target hover reassertion to the active tab", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
|
||||||
|
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 9,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenCalledTimes(1);
|
||||||
|
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 30,
|
||||||
|
y: 40,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases the current session hover latch before navigation-style work", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReleaseForRequest: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
|
||||||
|
await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||||
const { transport, deliver } = fakeTransport();
|
const { transport, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
|
|||||||
import {
|
import {
|
||||||
handleClick,
|
handleClick,
|
||||||
handleFill,
|
handleFill,
|
||||||
|
handleHover,
|
||||||
handlePress,
|
handlePress,
|
||||||
handleSelect,
|
handleSelect,
|
||||||
modifiersBitfield,
|
modifiersBitfield,
|
||||||
@@ -435,6 +436,50 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleHover", () => {
|
||||||
|
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
|
||||||
|
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||||
|
"Runtime.evaluate": (params: unknown) => {
|
||||||
|
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||||
|
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||||
|
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||||
|
}
|
||||||
|
if (expr.includes("hitIndex")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||||
|
},
|
||||||
|
"Input.dispatchMouseEvent": () => ({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleHover(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
|
||||||
|
{
|
||||||
|
cdp: fake.cdp,
|
||||||
|
tabsApi: fake.tabsApi,
|
||||||
|
bypassOverlay,
|
||||||
|
keepOverlayBypassAfterHover: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||||
|
expect(bypassOverlay).toHaveBeenCalledTimes(1);
|
||||||
|
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("handleFill", () => {
|
describe("handleFill", () => {
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
|||||||
@@ -1430,7 +1430,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
|
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps hover surface probes to matching node labels", () => {
|
it("maps hover surface probes to matching backend node ids", () => {
|
||||||
const axNodes: CdpAxNode[] = [
|
const axNodes: CdpAxNode[] = [
|
||||||
{
|
{
|
||||||
nodeId: "1",
|
nodeId: "1",
|
||||||
@@ -1451,7 +1451,7 @@ describe("buildVomScene", () => {
|
|||||||
iframeNodes: new Map(),
|
iframeNodes: new Map(),
|
||||||
excludedBackendNodeIds: new Set(),
|
excludedBackendNodeIds: new Set(),
|
||||||
surfaceProbes: [
|
surfaceProbes: [
|
||||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||||
],
|
],
|
||||||
nodes: [
|
nodes: [
|
||||||
{
|
{
|
||||||
@@ -1480,7 +1480,260 @@ describe("buildVomScene", () => {
|
|||||||
expect(scene.surfaces).toEqual([
|
expect(scene.surfaces).toEqual([
|
||||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||||
]);
|
]);
|
||||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps hover surface probes to rendered descendants in the same trigger subtree", () => {
|
||||||
|
const axNodes: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "1",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 10,
|
||||||
|
childIds: ["2"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "2",
|
||||||
|
parentId: "1",
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "image" },
|
||||||
|
backendDOMNodeId: 21,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const scene = buildVomScene(axNodes, {
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
surfaceProbes: [
|
||||||
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||||
|
],
|
||||||
|
nodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 10,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "body",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 20,
|
||||||
|
parentBackendNodeId: 10,
|
||||||
|
tag: "div",
|
||||||
|
attrs: { class: "tg-avatar" },
|
||||||
|
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 21,
|
||||||
|
parentBackendNodeId: 20,
|
||||||
|
tag: "div",
|
||||||
|
attrs: { class: "tg-avatar__inner" },
|
||||||
|
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||||
|
paintOrder: 2,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates hover surface probes by original trigger backend id", () => {
|
||||||
|
const axNodes: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "1",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 10,
|
||||||
|
childIds: ["2"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "2",
|
||||||
|
parentId: "1",
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "image" },
|
||||||
|
backendDOMNodeId: 21,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const scene = buildVomScene(axNodes, {
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
surfaceProbes: [
|
||||||
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||||
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||||
|
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||||
|
],
|
||||||
|
nodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 10,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "body",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 20,
|
||||||
|
parentBackendNodeId: 10,
|
||||||
|
tag: "div",
|
||||||
|
attrs: { class: "tg-avatar" },
|
||||||
|
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 21,
|
||||||
|
parentBackendNodeId: 20,
|
||||||
|
tag: "div",
|
||||||
|
attrs: { class: "tg-avatar__inner" },
|
||||||
|
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||||
|
paintOrder: 2,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.surfaces).toEqual([
|
||||||
|
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps hover surface probes to DOM-recovered custom controls", () => {
|
||||||
|
const axNodes: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "1",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 10,
|
||||||
|
childIds: ["2"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "2",
|
||||||
|
parentId: "1",
|
||||||
|
role: { type: "role", value: "img" },
|
||||||
|
name: { type: "computedString", value: "image" },
|
||||||
|
backendDOMNodeId: 21,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const scene = buildVomScene(axNodes, {
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
surfaceProbes: [
|
||||||
|
{
|
||||||
|
triggerBackendNodeId: 20,
|
||||||
|
triggerPoint: { x: 915, y: 25 },
|
||||||
|
triggerAction: "hover",
|
||||||
|
subItems: ["My profile", "Sign out"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
nodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 10,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "body",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
cursor: "auto",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 20,
|
||||||
|
parentBackendNodeId: 10,
|
||||||
|
tag: "div",
|
||||||
|
attrs: { class: "tg-avatar" },
|
||||||
|
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
cursor: "pointer",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 21,
|
||||||
|
parentBackendNodeId: 20,
|
||||||
|
tag: "div",
|
||||||
|
attrs: { class: "tg-avatar__inner" },
|
||||||
|
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||||
|
paintOrder: 2,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
cursor: "pointer",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.surfaces).toEqual([
|
||||||
|
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||||
|
]);
|
||||||
|
expect(renderVom(scene).text).toContain(
|
||||||
|
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not attach hover probes to distant geometry matches", () => {
|
||||||
|
const axNodes: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "1",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 10,
|
||||||
|
childIds: ["2"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "2",
|
||||||
|
parentId: "1",
|
||||||
|
role: { type: "role", value: "link" },
|
||||||
|
name: { type: "computedString", value: "0" },
|
||||||
|
backendDOMNodeId: 42,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const scene = buildVomScene(axNodes, {
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
surfaceProbes: [
|
||||||
|
{
|
||||||
|
triggerBackendNodeId: 999,
|
||||||
|
triggerPoint: { x: 900, y: 20 },
|
||||||
|
triggerAction: "hover",
|
||||||
|
subItems: ["My profile"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
nodes: [
|
||||||
|
{
|
||||||
|
backendNodeId: 10,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "body",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
backendNodeId: 42,
|
||||||
|
parentBackendNodeId: 10,
|
||||||
|
tag: "a",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scene.surfaces).toBeUndefined();
|
||||||
|
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("enriches names and active scope signals from AX properties", () => {
|
it("enriches names and active scope signals from AX properties", () => {
|
||||||
@@ -1953,7 +2206,7 @@ describe("handleSnapshot", () => {
|
|||||||
});
|
});
|
||||||
const res = await handleObserve(
|
const res = await handleObserve(
|
||||||
sm,
|
sm,
|
||||||
{ session_id: "aa11" },
|
{ session_id: "aa11", debug_surfaces: true },
|
||||||
{
|
{
|
||||||
cdp: {
|
cdp: {
|
||||||
send: send as unknown as <T = unknown>(
|
send: send as unknown as <T = unknown>(
|
||||||
@@ -1974,6 +2227,7 @@ describe("handleSnapshot", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res.debug).toEqual({ surface_probes: [] });
|
||||||
expect(send).toHaveBeenCalledWith(
|
expect(send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
"Runtime.evaluate",
|
"Runtime.evaluate",
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import type {
|
|||||||
EvaluateParams,
|
EvaluateParams,
|
||||||
FillParams,
|
FillParams,
|
||||||
GetHtmlParams,
|
GetHtmlParams,
|
||||||
|
HoverParams,
|
||||||
|
HoverResult,
|
||||||
NavigateBackParams,
|
NavigateBackParams,
|
||||||
NavigateForwardParams,
|
NavigateForwardParams,
|
||||||
NavigateParams,
|
NavigateParams,
|
||||||
@@ -33,7 +35,7 @@ import { handleConsole } from "./console";
|
|||||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
handleNavigateBack,
|
handleNavigateBack,
|
||||||
@@ -56,7 +58,7 @@ import {
|
|||||||
type SessionStartParams,
|
type SessionStartParams,
|
||||||
type SessionStopParams,
|
type SessionStopParams,
|
||||||
} from "./session";
|
} from "./session";
|
||||||
import { chromeTabsApi } from "./shared";
|
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||||
import {
|
import {
|
||||||
type BorrowConfirmationApprover,
|
type BorrowConfirmationApprover,
|
||||||
handleTabBorrow,
|
handleTabBorrow,
|
||||||
@@ -81,6 +83,18 @@ type DispatcherCdpRunner = CdpRunner &
|
|||||||
detachSession(sessionId: string): Promise<void>;
|
detachSession(sessionId: string): Promise<void>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
interface HoverLatch {
|
||||||
|
sessionId: string;
|
||||||
|
tabId: number;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HoverLatchScope {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DispatcherDeps {
|
export interface DispatcherDeps {
|
||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
@@ -126,6 +140,8 @@ export class ToolDispatcher {
|
|||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||||
private subscription: { dispose(): void } | null = null;
|
private subscription: { dispose(): void } | null = null;
|
||||||
|
private readonly hoverBypassTabs = new Map<number, string>();
|
||||||
|
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||||
/**
|
/**
|
||||||
* Per-rpc-id `AbortController` registry. Populated inside
|
* Per-rpc-id `AbortController` registry. Populated inside
|
||||||
* [`dispatch`] before we await the tool handler and torn down in
|
* [`dispatch`] before we await the tool handler and torn down in
|
||||||
@@ -271,16 +287,20 @@ export class ToolDispatcher {
|
|||||||
switch (req.method) {
|
switch (req.method) {
|
||||||
case "tool.session_start":
|
case "tool.session_start":
|
||||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
||||||
case "tool.session_stop":
|
case "tool.session_stop": {
|
||||||
|
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
case "tool.tab_list":
|
case "tool.tab_list":
|
||||||
return handleTabList(this.sessions, req.params as TabListParams);
|
return handleTabList(this.sessions, req.params as TabListParams);
|
||||||
case "tool.tab_create":
|
case "tool.tab_create":
|
||||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
||||||
case "tool.tab_close":
|
case "tool.tab_close":
|
||||||
return handleTabClose(this.sessions, req.params as TabCloseParams);
|
return this.withHoverReleaseForRequest(req.params as TabCloseParams, () =>
|
||||||
|
handleTabClose(this.sessions, req.params as TabCloseParams),
|
||||||
|
);
|
||||||
case "tool.tab_select":
|
case "tool.tab_select":
|
||||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
||||||
case "tool.tab_borrow":
|
case "tool.tab_borrow":
|
||||||
@@ -319,17 +339,31 @@ export class ToolDispatcher {
|
|||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||||
);
|
);
|
||||||
case "tool.snapshot":
|
case "tool.snapshot":
|
||||||
return handleSnapshot(
|
return this.withHoverReassert(req.params as SnapshotParams, () =>
|
||||||
this.sessions,
|
handleSnapshot(
|
||||||
req.params as SnapshotParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
req.params as SnapshotParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.observe":
|
case "tool.observe": {
|
||||||
return handleObserve(
|
const params = req.params as ObserveParams;
|
||||||
this.sessions,
|
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||||
req.params as ObserveParams,
|
return this.withHoverReassert(params, () =>
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
handleObserve(
|
||||||
|
this.sessions,
|
||||||
|
params,
|
||||||
|
this.cdp
|
||||||
|
? {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsCaptureApi,
|
||||||
|
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||||
|
hoverProbeBypassOverlay: bypassOverlay,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
|
}
|
||||||
case "tool.get_html":
|
case "tool.get_html":
|
||||||
return handleGetHtml(
|
return handleGetHtml(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
@@ -337,68 +371,95 @@ export class ToolDispatcher {
|
|||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
);
|
);
|
||||||
case "tool.navigate":
|
case "tool.navigate":
|
||||||
return handleNavigate(
|
return this.withHoverReleaseForRequest(req.params as NavigateParams, () =>
|
||||||
this.sessions,
|
handleNavigate(
|
||||||
req.params as NavigateParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as NavigateParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.navigate_back":
|
case "tool.navigate_back":
|
||||||
return handleNavigateBack(
|
return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () =>
|
||||||
this.sessions,
|
handleNavigateBack(
|
||||||
req.params as NavigateBackParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as NavigateBackParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.navigate_forward":
|
case "tool.navigate_forward":
|
||||||
return handleNavigateForward(
|
return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () =>
|
||||||
this.sessions,
|
handleNavigateForward(
|
||||||
req.params as NavigateForwardParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as NavigateForwardParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.reload":
|
case "tool.reload":
|
||||||
return handleReload(
|
return this.withHoverReleaseForRequest(req.params as ReloadParams, () =>
|
||||||
this.sessions,
|
handleReload(
|
||||||
req.params as ReloadParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as ReloadParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.click":
|
case "tool.click":
|
||||||
return handleClick(
|
return this.withHoverReassert(
|
||||||
this.sessions,
|
|
||||||
req.params as ClickParams,
|
req.params as ClickParams,
|
||||||
|
() =>
|
||||||
|
handleClick(
|
||||||
|
this.sessions,
|
||||||
|
req.params as ClickParams,
|
||||||
|
this.cdp
|
||||||
|
? {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsApi,
|
||||||
|
signal,
|
||||||
|
bypassOverlay,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
),
|
||||||
|
{ releaseAfter: true },
|
||||||
|
);
|
||||||
|
case "tool.hover": {
|
||||||
|
const result = await handleHover(
|
||||||
|
this.sessions,
|
||||||
|
req.params as HoverParams,
|
||||||
this.cdp
|
this.cdp
|
||||||
? {
|
? {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
signal,
|
signal,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
bypassOverlay: (tabId, enabled) =>
|
||||||
try {
|
this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled),
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
keepOverlayBypassAfterHover: true,
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
);
|
);
|
||||||
|
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||||
|
}
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
return handleFill(
|
return this.withHoverReleaseForRequest(req.params as FillParams, () =>
|
||||||
this.sessions,
|
handleFill(
|
||||||
req.params as FillParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as FillParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
return handlePress(
|
return this.withHoverReleaseForRequest(req.params as PressParams, () =>
|
||||||
this.sessions,
|
handlePress(
|
||||||
req.params as PressParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as PressParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
return handleSelect(
|
return this.withHoverReleaseForRequest(req.params as SelectParams, () =>
|
||||||
this.sessions,
|
handleSelect(
|
||||||
req.params as SelectParams,
|
this.sessions,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
req.params as SelectParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
case "tool.evaluate":
|
case "tool.evaluate":
|
||||||
return handleEvaluate(
|
return handleEvaluate(
|
||||||
@@ -471,6 +532,114 @@ export class ToolDispatcher {
|
|||||||
} satisfies RpcError;
|
} satisfies RpcError;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise<void> {
|
||||||
|
const owner = this.hoverBypassTabs.get(tabId);
|
||||||
|
if (enabled) {
|
||||||
|
if (owner === sessionId) return;
|
||||||
|
if (owner === undefined) await bypassOverlay(tabId, true);
|
||||||
|
this.hoverBypassTabs.set(tabId, sessionId);
|
||||||
|
} else {
|
||||||
|
if (owner !== sessionId) return;
|
||||||
|
await bypassOverlay(tabId, false);
|
||||||
|
this.hoverBypassTabs.delete(tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError {
|
||||||
|
if (!isRpcError(result)) {
|
||||||
|
this.hoverLatches.set(result.tab_id, {
|
||||||
|
sessionId,
|
||||||
|
tabId: result.tab_id,
|
||||||
|
x: result.x,
|
||||||
|
y: result.y,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
|
||||||
|
return this.hoverLatchesForRequest(scope).length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHoverReassert<T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options: { releaseAfter?: boolean } = {},
|
||||||
|
): Promise<T> {
|
||||||
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
await this.reassertHover(scope);
|
||||||
|
try {
|
||||||
|
return await work();
|
||||||
|
} finally {
|
||||||
|
if (options.releaseAfter) {
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHoverReleaseForRequest<T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
|
return work();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveHoverLatchScope(params: {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}): Promise<HoverLatchScope> {
|
||||||
|
if (params.tab_id !== undefined) return params;
|
||||||
|
const ctx = lookupSession(this.sessions, params, "hover latch");
|
||||||
|
if (isRpcError(ctx)) return params;
|
||||||
|
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
|
||||||
|
if (isRpcError(target)) return params;
|
||||||
|
return { session_id: params.session_id, tab_id: target.tabId };
|
||||||
|
}
|
||||||
|
|
||||||
|
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
||||||
|
return [...this.hoverLatches.values()].filter((latch) => {
|
||||||
|
if (latch.sessionId !== params.session_id) return false;
|
||||||
|
return params.tab_id === undefined || latch.tabId === params.tab_id;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async reassertHover(params: { session_id: string; tab_id?: number }): Promise<void> {
|
||||||
|
if (!this.cdp) return;
|
||||||
|
await Promise.all(
|
||||||
|
this.hoverLatchesForRequest(params).map((latch) =>
|
||||||
|
this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: latch.x,
|
||||||
|
y: latch.y,
|
||||||
|
}).catch((err) => {
|
||||||
|
console.debug("[bsk dispatcher] hover reassert failed", err);
|
||||||
|
this.hoverLatches.delete(latch.tabId);
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
|
||||||
|
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
|
||||||
|
if (sessionId !== undefined && entrySessionId !== sessionId) return false;
|
||||||
|
return tabId === undefined || entryTabId === tabId;
|
||||||
|
};
|
||||||
|
const tabs = new Set<number>();
|
||||||
|
for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) {
|
||||||
|
if (!matchesScope(bypassSessionId, bypassTabId)) continue;
|
||||||
|
tabs.add(bypassTabId);
|
||||||
|
this.hoverBypassTabs.delete(bypassTabId);
|
||||||
|
}
|
||||||
|
for (const latch of this.hoverLatches.values()) {
|
||||||
|
if (!matchesScope(latch.sessionId, latch.tabId)) continue;
|
||||||
|
tabs.add(latch.tabId);
|
||||||
|
this.hoverLatches.delete(latch.tabId);
|
||||||
|
}
|
||||||
|
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function isRpcError(v: unknown): v is RpcError {
|
function isRpcError(v: unknown): v is RpcError {
|
||||||
@@ -497,6 +666,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
|||||||
case "tool.navigate_forward":
|
case "tool.navigate_forward":
|
||||||
case "tool.reload":
|
case "tool.reload":
|
||||||
case "tool.click":
|
case "tool.click":
|
||||||
|
case "tool.hover":
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
@@ -512,6 +682,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||||
* fires (or immediately if it is already aborted). Used by the
|
* fires (or immediately if it is already aborted). Used by the
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ import type {
|
|||||||
ClickResult,
|
ClickResult,
|
||||||
FillParams,
|
FillParams,
|
||||||
FillResult,
|
FillResult,
|
||||||
|
HoverParams,
|
||||||
|
HoverResult,
|
||||||
KeyModifier,
|
KeyModifier,
|
||||||
MouseButton,
|
MouseButton,
|
||||||
PressParams,
|
PressParams,
|
||||||
@@ -53,9 +55,12 @@ export interface InteractionDeps {
|
|||||||
defaultTimeoutMs?: number;
|
defaultTimeoutMs?: number;
|
||||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||||
|
keepOverlayBypassAfterHover?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||||
|
|
||||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||||
@@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAbortLikeError(err: unknown): boolean {
|
||||||
|
return err instanceof DOMException && err.name === "AbortError";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const cleanup = () => signal?.removeEventListener("abort", onAbort);
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
cleanup();
|
||||||
|
reject(new DOMException("aborted", "AbortError"));
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
||||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||||
@@ -286,7 +312,104 @@ export async function handleClick(
|
|||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
if (automationBypassEnabled && deps.bypassOverlay) {
|
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(target.tabId, false);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
used_ref: node.usedRef,
|
||||||
|
used_selector: node.usedSelector,
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// tool.hover
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function handleHover(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: HoverParams,
|
||||||
|
deps: InteractionDeps = getDefaultDeps(),
|
||||||
|
): Promise<HoverResult | RpcError> {
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "hover");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
const aborted = throwIfAborted(deps.signal);
|
||||||
|
if (aborted) return aborted;
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "hover");
|
||||||
|
if (denied) return denied;
|
||||||
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||||
|
if (isRpcError(node)) return node;
|
||||||
|
|
||||||
|
try {
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
|
if (scrollErr) return scrollErr;
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
|
if (isRpcError(centre)) return centre;
|
||||||
|
|
||||||
|
if (throwIfAborted(deps.signal)) {
|
||||||
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const modifiers = modifiersBitfield(params.modifiers);
|
||||||
|
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||||
|
let automationBypassEnabled = false;
|
||||||
|
let hoverCompleted = false;
|
||||||
|
if (overlayBlocking && deps.bypassOverlay) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(target.tabId, true);
|
||||||
|
automationBypassEnabled = true;
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
modifiers,
|
||||||
|
});
|
||||||
|
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
|
||||||
|
if (settleMs > 0) {
|
||||||
|
await wait(settleMs, deps.signal);
|
||||||
|
}
|
||||||
|
hoverCompleted = true;
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortLikeError(err)) {
|
||||||
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
if (
|
||||||
|
automationBypassEnabled &&
|
||||||
|
deps.bypassOverlay &&
|
||||||
|
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
|
||||||
|
) {
|
||||||
try {
|
try {
|
||||||
await deps.bypassOverlay(target.tabId, false);
|
await deps.bypassOverlay(target.tabId, false);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -5,7 +5,9 @@
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
type ActiveScopeBlock,
|
type ActiveScopeBlock,
|
||||||
|
applyVomInteractionRecovery,
|
||||||
type CondSurface,
|
type CondSurface,
|
||||||
|
isVomReferenceNode,
|
||||||
renderVom,
|
renderVom,
|
||||||
type VomNode,
|
type VomNode,
|
||||||
type VomScene,
|
type VomScene,
|
||||||
@@ -908,39 +910,28 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
|||||||
return blocks;
|
return blocks;
|
||||||
}
|
}
|
||||||
|
|
||||||
function labelForSurfaceMatch(node: VomNode): string {
|
|
||||||
return (
|
|
||||||
cleanAttr(node.name) ??
|
|
||||||
cleanAttr(node.text) ??
|
|
||||||
cleanAttr(node.attrs?.["aria-label"]) ??
|
|
||||||
cleanAttr(node.attrs?.title) ??
|
|
||||||
""
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||||
const probes = captured.surfaceProbes ?? [];
|
const probes = captured.surfaceProbes ?? [];
|
||||||
if (probes.length === 0) return [];
|
if (probes.length === 0) return [];
|
||||||
|
|
||||||
const surfaces: CondSurface[] = [];
|
const surfaces: CondSurface[] = [];
|
||||||
const nodesWithLabels = nodes
|
const recoveredNodes = applyVomInteractionRecovery(nodes);
|
||||||
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
|
const signals = capturedOnlySignals(captured.nodes);
|
||||||
.filter((entry) => entry.key.length > 0);
|
|
||||||
const used = new Set<number>();
|
const used = new Set<number>();
|
||||||
for (const probe of probes) {
|
for (const probe of probes) {
|
||||||
const probeKey = normalizeProbeKey(probe.triggerLabel);
|
if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue;
|
||||||
if (!probeKey || probe.subItems.length === 0) continue;
|
const match = findSurfaceTriggerNode(
|
||||||
const match =
|
probe.triggerBackendNodeId,
|
||||||
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
|
recoveredNodes,
|
||||||
nodesWithLabels.find(
|
signals,
|
||||||
(entry) =>
|
probe.triggerPoint,
|
||||||
!used.has(entry.node.id) &&
|
);
|
||||||
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
|
|
||||||
);
|
|
||||||
if (!match) continue;
|
if (!match) continue;
|
||||||
used.add(match.node.id);
|
if (used.has(match.id)) continue;
|
||||||
|
used.add(probe.triggerBackendNodeId);
|
||||||
|
used.add(match.id);
|
||||||
surfaces.push({
|
surfaces.push({
|
||||||
triggerId: match.node.id,
|
triggerId: match.id,
|
||||||
triggerAction: probe.triggerAction,
|
triggerAction: probe.triggerAction,
|
||||||
subItems: probe.subItems,
|
subItems: probe.subItems,
|
||||||
});
|
});
|
||||||
@@ -948,6 +939,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel)
|
|||||||
return surfaces;
|
return surfaces;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function findSurfaceTriggerNode(
|
||||||
|
triggerBackendNodeId: number,
|
||||||
|
nodes: VomNode[],
|
||||||
|
signals: VomNodeDomSignals,
|
||||||
|
triggerPoint?: { x: number; y: number },
|
||||||
|
): VomNode | undefined {
|
||||||
|
const renderedById = new Map(nodes.map((node) => [node.id, node]));
|
||||||
|
const exact = renderedById.get(triggerBackendNodeId);
|
||||||
|
if (exact && isSurfaceAttachableNode(exact)) return exact;
|
||||||
|
|
||||||
|
const domDescendant = nodes.find(
|
||||||
|
(node) =>
|
||||||
|
isSurfaceAttachableNode(node) &&
|
||||||
|
(node.domParentId === triggerBackendNodeId ||
|
||||||
|
node.domAncestorIds?.includes(triggerBackendNodeId) === true),
|
||||||
|
);
|
||||||
|
if (domDescendant) return domDescendant;
|
||||||
|
|
||||||
|
const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])];
|
||||||
|
while (queue.length > 0) {
|
||||||
|
const child = queue.shift() as CapturedNode;
|
||||||
|
const rendered = renderedById.get(child.backendNodeId);
|
||||||
|
if (rendered && isSurfaceAttachableNode(rendered)) return rendered;
|
||||||
|
queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? []));
|
||||||
|
}
|
||||||
|
|
||||||
|
let current = signals.capturedByBackendId.get(triggerBackendNodeId);
|
||||||
|
while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) {
|
||||||
|
const parent = renderedById.get(current.parentBackendNodeId);
|
||||||
|
if (parent && isSurfaceAttachableNode(parent)) return parent;
|
||||||
|
current = signals.capturedByBackendId.get(current.parentBackendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (triggerPoint) {
|
||||||
|
return findSurfaceNodeByPoint(triggerPoint, nodes, signals);
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSurfaceAttachableNode(node: VomNode): boolean {
|
||||||
|
return isVomReferenceNode(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findSurfaceNodeByPoint(
|
||||||
|
point: { x: number; y: number },
|
||||||
|
nodes: VomNode[],
|
||||||
|
signals: VomNodeDomSignals,
|
||||||
|
): VomNode | undefined {
|
||||||
|
let best: { node: VomNode; score: number } | undefined;
|
||||||
|
for (const node of nodes) {
|
||||||
|
if (!isSurfaceAttachableNode(node)) continue;
|
||||||
|
const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect;
|
||||||
|
if (!rect) continue;
|
||||||
|
const contains =
|
||||||
|
point.x >= rect.x &&
|
||||||
|
point.x <= rect.x + rect.w &&
|
||||||
|
point.y >= rect.y &&
|
||||||
|
point.y <= rect.y + rect.h;
|
||||||
|
const centerX = rect.x + rect.w / 2;
|
||||||
|
const centerY = rect.y + rect.h / 2;
|
||||||
|
const distance = Math.hypot(point.x - centerX, point.y - centerY);
|
||||||
|
if (!contains && distance > 40) continue;
|
||||||
|
const score = contains ? distance : distance + 1_000;
|
||||||
|
if (!best || score < best.score) {
|
||||||
|
best = { node, score };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best?.node;
|
||||||
|
}
|
||||||
|
|
||||||
function axNodeInOverlaySubtree(
|
function axNodeInOverlaySubtree(
|
||||||
axNode: CdpAxNode,
|
axNode: CdpAxNode,
|
||||||
axById: Map<string, CdpAxNode>,
|
axById: Map<string, CdpAxNode>,
|
||||||
@@ -1072,6 +1134,8 @@ export interface SnapshotDeps {
|
|||||||
get(tabId: number): Promise<chrome.tabs.Tab>;
|
get(tabId: number): Promise<chrome.tabs.Tab>;
|
||||||
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
||||||
};
|
};
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
let defaultDeps: SnapshotDeps | null = null;
|
let defaultDeps: SnapshotDeps | null = null;
|
||||||
@@ -1217,9 +1281,13 @@ async function captureForVom(
|
|||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
conditionalSurfaceProbe: boolean,
|
conditionalSurfaceProbe: boolean,
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
try {
|
try {
|
||||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
return await captureViewModel(cdp, tabId, {
|
||||||
|
conditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay,
|
||||||
|
});
|
||||||
} catch {
|
} catch {
|
||||||
return fallbackCapturedViewModel(cdp, tabId);
|
return fallbackCapturedViewModel(cdp, tabId);
|
||||||
}
|
}
|
||||||
@@ -1258,7 +1326,14 @@ async function handleVomObservation(
|
|||||||
{},
|
{},
|
||||||
);
|
);
|
||||||
const axNodes = result.nodes ?? [];
|
const axNodes = result.nodes ?? [];
|
||||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
const effectiveConditionalSurfaceProbe =
|
||||||
|
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||||
|
const captured = await captureForVom(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
effectiveConditionalSurfaceProbe,
|
||||||
|
deps.hoverProbeBypassOverlay,
|
||||||
|
);
|
||||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||||
const rendered = renderVom(scene, {
|
const rendered = renderVom(scene, {
|
||||||
maxDepth: params.max_depth,
|
maxDepth: params.max_depth,
|
||||||
@@ -1275,6 +1350,19 @@ async function handleVomObservation(
|
|||||||
ref_count: rendered.refs.length,
|
ref_count: rendered.refs.length,
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
truncated: rendered.truncated,
|
truncated: rendered.truncated,
|
||||||
|
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||||
|
? {
|
||||||
|
debug: {
|
||||||
|
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||||
|
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||||
|
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||||
|
trigger_action: probe.triggerAction,
|
||||||
|
sub_items: probe.subItems,
|
||||||
|
...(probe.confidence ? { confidence: probe.confidence } : {}),
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -51,6 +51,156 @@ function fakeSnapshotReply() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hoverTriggerSnapshotReply() {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"button",
|
||||||
|
"class",
|
||||||
|
"user-avatar",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"cursor",
|
||||||
|
"pointer",
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1],
|
||||||
|
nodeType: [1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button")],
|
||||||
|
backendNodeId: [10, 11, 12],
|
||||||
|
attributes: [[], [], [i("class"), i("user-avatar")]],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[940, 16, 32, 32],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function twoHoverTriggerSnapshotReply() {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"button",
|
||||||
|
"class",
|
||||||
|
"user-avatar",
|
||||||
|
"secondary-dropdown-trigger",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"cursor",
|
||||||
|
"pointer",
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1],
|
||||||
|
nodeType: [1, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button"), i("button")],
|
||||||
|
backendNodeId: [10, 11, 12, 13],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[i("class"), i("user-avatar")],
|
||||||
|
[i("class"), i("secondary-dropdown-trigger")],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[940, 16, 32, 32],
|
||||||
|
[100, 120, 32, 32],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function nestedHoverTriggerSnapshotReply() {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"div",
|
||||||
|
"img",
|
||||||
|
"class",
|
||||||
|
"tg-avatar",
|
||||||
|
"tg-avatar__inner",
|
||||||
|
"tg-avatar__image",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"cursor",
|
||||||
|
"pointer",
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 2, 3],
|
||||||
|
nodeType: [1, 1, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[i("class"), i("tg-avatar")],
|
||||||
|
[i("class"), i("tg-avatar__inner")],
|
||||||
|
[i("class"), i("tg-avatar__image")],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3, 4],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[940, 16, 32, 32],
|
||||||
|
[942, 18, 28, 28],
|
||||||
|
[944, 20, 24, 24],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2, 3],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function makeCdp(snapshot: unknown) {
|
function makeCdp(snapshot: unknown) {
|
||||||
return {
|
return {
|
||||||
send: vi.fn(async (_tab: number, method: string) => {
|
send: vi.fn(async (_tab: number, method: string) => {
|
||||||
@@ -126,11 +276,11 @@ describe("captureViewModel", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("runs hover surface probes when explicitly enabled", async () => {
|
it("runs hover surface probes when explicitly enabled", async () => {
|
||||||
let runtimeCalls = 0;
|
let hoverStateCalls = 0;
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
if (method === "DOMSnapshot.enable") return {};
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
|
||||||
if (method === "Page.getLayoutMetrics") {
|
if (method === "Page.getLayoutMetrics") {
|
||||||
return {
|
return {
|
||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
@@ -141,22 +291,23 @@ describe("captureViewModel", () => {
|
|||||||
if (expression.includes("input,textarea,select")) {
|
if (expression.includes("input,textarea,select")) {
|
||||||
return { result: { value: { controls: [], childFrames: [] } } };
|
return { result: { value: { controls: [], childFrames: [] } } };
|
||||||
}
|
}
|
||||||
runtimeCalls += 1;
|
if (expression.includes("document.styleSheets")) {
|
||||||
return runtimeCalls === 1
|
return { result: { value: [] } };
|
||||||
? {
|
}
|
||||||
result: {
|
if (expression.includes("querySelectorAll(selectors)")) {
|
||||||
value: [
|
hoverStateCalls += 1;
|
||||||
{
|
return hoverStateCalls === 1
|
||||||
triggerSel: ".menu",
|
? { result: { value: [] } }
|
||||||
affectedSub: " .items",
|
: {
|
||||||
label: "Products",
|
result: {
|
||||||
x: 50,
|
value: [
|
||||||
y: 20,
|
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||||
},
|
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
: { result: { value: ["Shoes", "Bags"] } };
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||||
}
|
}
|
||||||
if (method === "Input.dispatchMouseEvent") return {};
|
if (method === "Input.dispatchMouseEvent") return {};
|
||||||
throw new Error(`unexpected ${method}`);
|
throw new Error(`unexpected ${method}`);
|
||||||
@@ -166,12 +317,18 @@ describe("captureViewModel", () => {
|
|||||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||||
|
|
||||||
expect(result.surfaceProbes).toEqual([
|
expect(result.surfaceProbes).toEqual([
|
||||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
{
|
||||||
|
triggerBackendNodeId: 12,
|
||||||
|
triggerPoint: { x: 956, y: 32 },
|
||||||
|
triggerAction: "hover",
|
||||||
|
subItems: ["My profile", "Sign out"],
|
||||||
|
confidence: "high",
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
expect(cdp.send).toHaveBeenCalledWith(
|
expect(cdp.send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
"Input.dispatchMouseEvent",
|
"Input.dispatchMouseEvent",
|
||||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }),
|
||||||
);
|
);
|
||||||
expect(cdp.send).toHaveBeenCalledWith(
|
expect(cdp.send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
@@ -180,6 +337,91 @@ describe("captureViewModel", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses a fresh baseline for each hover candidate", async () => {
|
||||||
|
let hoverStateCalls = 0;
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||||
|
if (expression.includes("input,textarea,select")) {
|
||||||
|
return { result: { value: { controls: [], childFrames: [] } } };
|
||||||
|
}
|
||||||
|
if (expression.includes("document.styleSheets")) {
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
if (expression.includes("querySelectorAll(selectors)")) {
|
||||||
|
hoverStateCalls += 1;
|
||||||
|
const visibleMenu = [
|
||||||
|
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||||
|
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||||
|
];
|
||||||
|
return hoverStateCalls === 1
|
||||||
|
? { result: { value: [] } }
|
||||||
|
: { result: { value: visibleMenu } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchMouseEvent") return {};
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||||
|
|
||||||
|
expect(result.surfaceProbes).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
triggerBackendNodeId: 12,
|
||||||
|
subItems: ["My profile", "Sign out"],
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(hoverStateCalls).toBeGreaterThanOrEqual(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates nested hover candidates for one visual trigger", async () => {
|
||||||
|
let hoverMoves = 0;
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||||
|
if (expression.includes("input,textarea,select")) {
|
||||||
|
return { result: { value: { controls: [], childFrames: [] } } };
|
||||||
|
}
|
||||||
|
if (expression.includes("document.styleSheets")) {
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
if (expression.includes("querySelectorAll(selectors)")) {
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchMouseEvent") {
|
||||||
|
const point = params as { x?: number; y?: number };
|
||||||
|
if (point.x !== -10 && point.x !== 0) hoverMoves += 1;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
|
};
|
||||||
|
|
||||||
|
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||||
|
|
||||||
|
expect(hoverMoves).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||||
|
|
||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||||
|
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
@@ -45,9 +46,11 @@ export interface CapturedNode {
|
|||||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||||
|
|
||||||
export interface CapturedSurfaceProbe {
|
export interface CapturedSurfaceProbe {
|
||||||
triggerLabel: string;
|
triggerBackendNodeId: number;
|
||||||
|
triggerPoint?: { x: number; y: number };
|
||||||
triggerAction: "hover" | "focus" | string;
|
triggerAction: "hover" | "focus" | string;
|
||||||
subItems: string[];
|
subItems: string[];
|
||||||
|
confidence?: "high" | "medium" | "low";
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CapturedViewModel {
|
export interface CapturedViewModel {
|
||||||
@@ -61,6 +64,7 @@ export interface CapturedViewModel {
|
|||||||
|
|
||||||
export interface CaptureViewModelOptions {
|
export interface CaptureViewModelOptions {
|
||||||
conditionalSurfaceProbe?: boolean;
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||||
@@ -225,11 +229,12 @@ interface RuntimeEvaluateReply {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
triggerSel: string;
|
backendNodeId: number;
|
||||||
affectedSub: string;
|
label?: string;
|
||||||
label: string;
|
|
||||||
x: number;
|
x: number;
|
||||||
y: number;
|
y: number;
|
||||||
|
score: number;
|
||||||
|
reasons: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
interface CdpDomNode {
|
interface CdpDomNode {
|
||||||
@@ -322,17 +327,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
|||||||
}
|
}
|
||||||
|
|
||||||
const MAX_HOVER_PROBE_MS = 2_000;
|
const MAX_HOVER_PROBE_MS = 2_000;
|
||||||
const MAX_HOVER_TRIGGERS = 8;
|
const MAX_HOVER_TRIGGERS = 6;
|
||||||
const HOVER_SETTLE_MS = 200;
|
const MAX_HOVER_SURFACES = 3;
|
||||||
|
const HOVER_SETTLE_MS = 300;
|
||||||
|
const MAX_HOVER_SUB_ITEMS = 12;
|
||||||
|
|
||||||
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
||||||
return reply.result?.value as T | undefined;
|
return reply.result?.value as T | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverCssScanExpression(): string {
|
function hoverCssTriggerScanExpression(): string {
|
||||||
return `(() => {
|
return `(() => {
|
||||||
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
||||||
const pairs = [];
|
const centres = [];
|
||||||
const seenRules = new Set();
|
const seenRules = new Set();
|
||||||
for (const sheet of Array.from(document.styleSheets)) {
|
for (const sheet of Array.from(document.styleSheets)) {
|
||||||
let rules;
|
let rules;
|
||||||
@@ -348,90 +355,73 @@ function hoverCssScanExpression(): string {
|
|||||||
const hoverIndex = part.indexOf(":hover");
|
const hoverIndex = part.indexOf(":hover");
|
||||||
if (hoverIndex < 0) continue;
|
if (hoverIndex < 0) continue;
|
||||||
const triggerSel = part.slice(0, hoverIndex).trim();
|
const triggerSel = part.slice(0, hoverIndex).trim();
|
||||||
const affectedSub = part.slice(hoverIndex + 6).trim();
|
|
||||||
if (!triggerSel) continue;
|
if (!triggerSel) continue;
|
||||||
const key = triggerSel + "||" + affectedSub;
|
if (seenRules.has(triggerSel)) continue;
|
||||||
if (seenRules.has(key)) continue;
|
seenRules.add(triggerSel);
|
||||||
seenRules.add(key);
|
let elements;
|
||||||
pairs.push({ triggerSel, affectedSub });
|
try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; }
|
||||||
|
for (const el of elements) {
|
||||||
|
if (!(el instanceof HTMLElement)) continue;
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||||
|
centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 });
|
||||||
|
if (centres.length >= 24) return centres;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return centres;
|
||||||
const candidates = [];
|
|
||||||
const seenLabels = new Set();
|
|
||||||
const isUnsafeTrigger = (el) => {
|
|
||||||
if (!(el instanceof HTMLElement)) return true;
|
|
||||||
const tag = el.tagName.toLowerCase();
|
|
||||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
|
||||||
if (el.isContentEditable) return true;
|
|
||||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
|
||||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
for (const pair of pairs) {
|
|
||||||
let elements;
|
|
||||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
|
||||||
for (const el of elements) {
|
|
||||||
if (!(el instanceof Element)) continue;
|
|
||||||
if (isUnsafeTrigger(el)) continue;
|
|
||||||
const rect = el.getBoundingClientRect();
|
|
||||||
const style = getComputedStyle(el);
|
|
||||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
|
||||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
|
||||||
if (!label || seenLabels.has(label)) continue;
|
|
||||||
seenLabels.add(label);
|
|
||||||
candidates.push({
|
|
||||||
triggerSel: pair.triggerSel,
|
|
||||||
affectedSub: pair.affectedSub,
|
|
||||||
label,
|
|
||||||
x: rect.left + rect.width / 2,
|
|
||||||
y: rect.top + rect.height / 2,
|
|
||||||
});
|
|
||||||
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return candidates;
|
|
||||||
})()`;
|
})()`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverCollectExpression(candidate: HoverCandidate): string {
|
interface HoverRuntimeItem {
|
||||||
return `(() => {
|
text: string;
|
||||||
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
|
role: string;
|
||||||
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
|
tag: string;
|
||||||
const x = ${JSON.stringify(candidate.x)};
|
x: number;
|
||||||
const y = ${JSON.stringify(candidate.y)};
|
y: number;
|
||||||
const hit = document.elementFromPoint(x, y);
|
}
|
||||||
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
|
|
||||||
if (!trigger) return [];
|
|
||||||
|
|
||||||
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
|
function hoverStateExpression(): string {
|
||||||
let targets = [];
|
return `(() => {
|
||||||
if (!affectedSub) {
|
|
||||||
targets = [trigger];
|
|
||||||
} else {
|
|
||||||
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
|
|
||||||
}
|
|
||||||
const items = [];
|
const items = [];
|
||||||
const seen = new Set();
|
const seen = new Set();
|
||||||
const push = (value) => {
|
const selectors = [
|
||||||
const text = String(value || "").replace(/\\s+/g, " ").trim();
|
"a",
|
||||||
|
"button",
|
||||||
|
"[role='menuitem']",
|
||||||
|
"[role='menuitemcheckbox']",
|
||||||
|
"[role='menuitemradio']",
|
||||||
|
"[role='option']",
|
||||||
|
"[role='tab']",
|
||||||
|
"[role='link']",
|
||||||
|
"[role='button']"
|
||||||
|
].join(",");
|
||||||
|
const push = (el) => {
|
||||||
|
if (!(el instanceof HTMLElement)) return;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 || rect.height <= 0) return;
|
||||||
|
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
|
||||||
|
const text = String(
|
||||||
|
el.getAttribute("aria-label") ||
|
||||||
|
el.getAttribute("title") ||
|
||||||
|
el.textContent ||
|
||||||
|
""
|
||||||
|
).replace(/\\s+/g, " ").trim();
|
||||||
if (!text || seen.has(text)) return;
|
if (!text || seen.has(text)) return;
|
||||||
seen.add(text);
|
seen.add(text);
|
||||||
items.push(text);
|
items.push({
|
||||||
|
text,
|
||||||
|
role: (el.getAttribute("role") || "").toLowerCase(),
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
for (const target of targets) {
|
for (const el of Array.from(document.querySelectorAll(selectors))) push(el);
|
||||||
if (!(target instanceof Element)) continue;
|
return items.slice(0, 400);
|
||||||
const style = getComputedStyle(target);
|
|
||||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
|
||||||
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
|
|
||||||
if (clickable.length > 0) {
|
|
||||||
for (const child of Array.from(clickable)) push(child.textContent);
|
|
||||||
} else {
|
|
||||||
push(target.textContent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items.slice(0, 12);
|
|
||||||
})()`;
|
})()`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -453,45 +443,217 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
function capturedText(node: CapturedNode): string | undefined {
|
||||||
|
const value =
|
||||||
|
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||||
|
const clean = value.replace(/\s+/g, " ").trim();
|
||||||
|
return clean || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasGraphicDescendant(
|
||||||
|
node: CapturedNode,
|
||||||
|
childrenByParentId: Map<number, CapturedNode[]>,
|
||||||
|
depth = 0,
|
||||||
|
): boolean {
|
||||||
|
if (depth > 3) return false;
|
||||||
|
for (const child of childrenByParentId.get(node.backendNodeId) ?? []) {
|
||||||
|
const tag = child.tag.toLowerCase();
|
||||||
|
if (["img", "svg", "use", "path", "i"].includes(tag)) return true;
|
||||||
|
if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function roleOf(node: CapturedNode): string {
|
||||||
|
return (node.attrs.role ?? "").toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreHoverCandidate(
|
||||||
|
node: CapturedNode,
|
||||||
|
childrenByParentId: Map<number, CapturedNode[]>,
|
||||||
|
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||||
|
): HoverCandidate | null {
|
||||||
|
const rect = node.rect;
|
||||||
|
if (!rect) return null;
|
||||||
|
const label = capturedText(node);
|
||||||
|
const cssHoverMatch = cssHoverPoints.some(
|
||||||
|
(point) =>
|
||||||
|
point.x >= rect.x &&
|
||||||
|
point.x <= rect.x + rect.w &&
|
||||||
|
point.y >= rect.y &&
|
||||||
|
point.y <= rect.y + rect.h,
|
||||||
|
);
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: node.tag,
|
||||||
|
role: roleOf(node),
|
||||||
|
label,
|
||||||
|
attrs: node.attrs,
|
||||||
|
rect,
|
||||||
|
cursor: node.cursor,
|
||||||
|
pointerEvents: node.pointerEvents,
|
||||||
|
hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId),
|
||||||
|
cssHoverMatch,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!decision.eligible) return null;
|
||||||
|
return {
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
label,
|
||||||
|
x: rect.x + rect.w / 2,
|
||||||
|
y: rect.y + rect.h / 2,
|
||||||
|
score: decision.score,
|
||||||
|
reasons: decision.reasons,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildHoverCandidates(
|
||||||
|
nodes: CapturedNode[],
|
||||||
|
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||||
|
): HoverCandidate[] {
|
||||||
|
const childrenByParentId = new Map<number, CapturedNode[]>();
|
||||||
|
const parentByBackendId = new Map<number, number | null>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId);
|
||||||
|
if (node.parentBackendNodeId === null) continue;
|
||||||
|
const children = childrenByParentId.get(node.parentBackendNodeId) ?? [];
|
||||||
|
children.push(node);
|
||||||
|
childrenByParentId.set(node.parentBackendNodeId, children);
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates = nodes
|
||||||
|
.map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints))
|
||||||
|
.filter((candidate): candidate is HoverCandidate => candidate !== null)
|
||||||
|
.sort((a, b) => b.score - a.score);
|
||||||
|
|
||||||
|
const deduped: HoverCandidate[] = [];
|
||||||
|
const seen = new Set<number>();
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (seen.has(candidate.backendNodeId)) continue;
|
||||||
|
if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seen.add(candidate.backendNodeId);
|
||||||
|
deduped.push(candidate);
|
||||||
|
if (deduped.length >= MAX_HOVER_TRIGGERS) break;
|
||||||
|
}
|
||||||
|
return deduped;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameHoverCluster(
|
||||||
|
a: HoverCandidate,
|
||||||
|
b: HoverCandidate,
|
||||||
|
parentByBackendId: Map<number, number | null>,
|
||||||
|
): boolean {
|
||||||
|
if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true;
|
||||||
|
return (
|
||||||
|
isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) ||
|
||||||
|
isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isBackendAncestor(
|
||||||
|
ancestorId: number,
|
||||||
|
nodeId: number,
|
||||||
|
parentByBackendId: Map<number, number | null>,
|
||||||
|
): boolean {
|
||||||
|
let parentId = parentByBackendId.get(nodeId);
|
||||||
|
let guard = 0;
|
||||||
|
while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) {
|
||||||
|
if (parentId === ancestorId) return true;
|
||||||
|
parentId = parentByBackendId.get(parentId);
|
||||||
|
guard += 1;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] {
|
||||||
|
const beforeKeys = new Set(before.map((item) => item.text.toLowerCase()));
|
||||||
|
const out: string[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const item of after) {
|
||||||
|
const text = item.text.replace(/\s+/g, " ").trim();
|
||||||
|
const key = text.toLowerCase();
|
||||||
|
if (!text || beforeKeys.has(key) || seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
out.push(text);
|
||||||
|
if (out.length >= MAX_HOVER_SUB_ITEMS) break;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function confidenceForHover(
|
||||||
|
candidate: HoverCandidate,
|
||||||
|
subItems: string[],
|
||||||
|
): "high" | "medium" | "low" {
|
||||||
|
if (subItems.length >= 2 && candidate.score >= 80) return "high";
|
||||||
|
if (subItems.length >= 2 || candidate.score >= 80) return "medium";
|
||||||
|
return "low";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function probeHoverSurfaces(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
nodes: CapturedNode[],
|
||||||
|
options: CaptureViewModelOptions,
|
||||||
|
): Promise<CapturedSurfaceProbe[]> {
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
try {
|
try {
|
||||||
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: hoverCssScanExpression(),
|
expression: hoverCssTriggerScanExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
});
|
});
|
||||||
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
|
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
|
||||||
|
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
|
||||||
|
if (candidates.length === 0) return [];
|
||||||
|
|
||||||
const results: CapturedSurfaceProbe[] = [];
|
const results: CapturedSurfaceProbe[] = [];
|
||||||
const seen = new Set<string>();
|
const seen = new Set<number>();
|
||||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
try {
|
||||||
if (!candidate.label || seen.has(candidate.label)) continue;
|
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||||
try {
|
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||||
type: "mouseMoved",
|
if (seen.has(candidate.backendNodeId)) continue;
|
||||||
x: candidate.x,
|
try {
|
||||||
y: candidate.y,
|
await clearHover(cdp, tabId);
|
||||||
});
|
await wait(HOVER_SETTLE_MS);
|
||||||
await wait(HOVER_SETTLE_MS);
|
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
expression: hoverStateExpression(),
|
||||||
expression: hoverCollectExpression(candidate),
|
returnByValue: true,
|
||||||
returnByValue: true,
|
});
|
||||||
});
|
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
|
||||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||||
.filter(Boolean);
|
type: "mouseMoved",
|
||||||
if (subItems.length === 0) continue;
|
x: candidate.x,
|
||||||
seen.add(candidate.label);
|
y: candidate.y,
|
||||||
results.push({
|
});
|
||||||
triggerLabel: candidate.label,
|
await wait(HOVER_SETTLE_MS);
|
||||||
triggerAction: "hover",
|
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
subItems,
|
expression: hoverStateExpression(),
|
||||||
});
|
returnByValue: true,
|
||||||
} catch {
|
});
|
||||||
continue;
|
const subItems = diffHoverItems(
|
||||||
} finally {
|
baselineItems,
|
||||||
await clearHover(cdp, tabId);
|
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||||
|
);
|
||||||
|
if (subItems.length === 0) continue;
|
||||||
|
seen.add(candidate.backendNodeId);
|
||||||
|
results.push({
|
||||||
|
triggerBackendNodeId: candidate.backendNodeId,
|
||||||
|
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||||
|
triggerAction: "hover",
|
||||||
|
subItems,
|
||||||
|
confidence: confidenceForHover(candidate, subItems),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
} finally {
|
||||||
|
await clearHover(cdp, tabId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -797,7 +959,9 @@ export async function captureViewModel(
|
|||||||
|
|
||||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||||
|
|
||||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
const surfaceProbes = options.conditionalSurfaceProbe
|
||||||
|
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||||
|
: [];
|
||||||
|
|
||||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -314,8 +314,21 @@ export interface SnapshotResult {
|
|||||||
dialogs?: JavaScriptDialogInfo[];
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ObserveParams = SnapshotParams;
|
export interface ObserveParams extends SnapshotParams {
|
||||||
export type ObserveResult = SnapshotResult;
|
debug_surfaces?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ObserveResult extends SnapshotResult {
|
||||||
|
debug?: {
|
||||||
|
surface_probes?: Array<{
|
||||||
|
trigger_backend_node_id: number;
|
||||||
|
trigger_point?: { x: number; y: number };
|
||||||
|
trigger_action: string;
|
||||||
|
sub_items: string[];
|
||||||
|
confidence?: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface GetHtmlParams {
|
export interface GetHtmlParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
@@ -410,6 +423,25 @@ export interface ClickResult {
|
|||||||
dialogs?: JavaScriptDialogInfo[];
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HoverParams {
|
||||||
|
session_id: string;
|
||||||
|
ref?: string;
|
||||||
|
selector?: string;
|
||||||
|
tab_id?: number;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
settle_ms?: number;
|
||||||
|
timeout_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverResult {
|
||||||
|
tab_id: number;
|
||||||
|
used_ref?: string;
|
||||||
|
used_selector?: string;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface FillParams {
|
export interface FillParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
value: string;
|
value: string;
|
||||||
@@ -668,6 +700,11 @@ export type DraftTraceStep =
|
|||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
}
|
}
|
||||||
|
| {
|
||||||
|
op: "hover";
|
||||||
|
target: TargetDescriptor;
|
||||||
|
page_url?: string;
|
||||||
|
}
|
||||||
| {
|
| {
|
||||||
op: "fill";
|
op: "fill";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
@@ -701,6 +738,7 @@ export type DraftTraceStep =
|
|||||||
export type Step =
|
export type Step =
|
||||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||||
|
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||||
| ({ op: "fill" } & StepCommon & {
|
| ({ op: "fill" } & StepCommon & {
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
value: string;
|
value: string;
|
||||||
|
|||||||
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
|||||||
|
|
||||||
```
|
```
|
||||||
bsk navigate <url> --session <id>
|
bsk navigate <url> --session <id>
|
||||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||||
bsk snapshot --session <id> → again after navigation / DOM change
|
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||||
|
bsk observe --session <id> → again after navigation / DOM change
|
||||||
```
|
```
|
||||||
|
|
||||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||||
|
|
||||||
|
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||||
|
|
||||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||||
|
|
||||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
|||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||||
|
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ use std::time::Duration;
|
|||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use bsk_protocol::Method;
|
use bsk_protocol::Method;
|
||||||
use bsk_protocol::tools::{
|
use bsk_protocol::tools::{
|
||||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
|
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||||
PressResult, SelectParams, SelectResult,
|
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||||
};
|
};
|
||||||
use clap::{Args, ValueEnum};
|
use clap::{Args, ValueEnum};
|
||||||
|
|
||||||
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// bsk hover
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Args)]
|
||||||
|
pub struct HoverArgs {
|
||||||
|
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||||
|
pub target: Option<String>,
|
||||||
|
|
||||||
|
/// Force-treat the value as a snapshot ref (overrides positional detection).
|
||||||
|
#[arg(long = "ref")]
|
||||||
|
pub ref_: Option<String>,
|
||||||
|
|
||||||
|
/// Force-treat the value as a CSS selector.
|
||||||
|
#[arg(long = "selector")]
|
||||||
|
pub selector: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
pub session: String,
|
||||||
|
|
||||||
|
#[arg(long = "tab-id")]
|
||||||
|
pub tab_id: Option<i64>,
|
||||||
|
|
||||||
|
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
|
||||||
|
#[arg(long, default_value = "")]
|
||||||
|
pub modifiers: String,
|
||||||
|
|
||||||
|
/// Wait after moving the mouse so hover-triggered UI can settle.
|
||||||
|
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
|
||||||
|
pub settle: u32,
|
||||||
|
|
||||||
|
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||||
|
pub timeout: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
|
||||||
|
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||||
|
let (ref_, selector) = split_target(
|
||||||
|
args.target.clone(),
|
||||||
|
args.ref_.clone(),
|
||||||
|
args.selector.clone(),
|
||||||
|
)?;
|
||||||
|
let modifiers = parse_modifiers(&args.modifiers)
|
||||||
|
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||||
|
let params = HoverParams {
|
||||||
|
session_id: args.session.clone(),
|
||||||
|
ref_,
|
||||||
|
selector,
|
||||||
|
tab_id: args.tab_id,
|
||||||
|
modifiers: if modifiers.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(modifiers)
|
||||||
|
},
|
||||||
|
settle_ms: Some(args.settle),
|
||||||
|
timeout_ms: Some(args.timeout),
|
||||||
|
};
|
||||||
|
let reply: HoverResult = call(
|
||||||
|
info.sock_path,
|
||||||
|
Method::ToolHover,
|
||||||
|
params,
|
||||||
|
"hover-1",
|
||||||
|
args.timeout,
|
||||||
|
)?;
|
||||||
|
match format {
|
||||||
|
Format::Json => {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(&reply)
|
||||||
|
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Format::Human => {
|
||||||
|
let target =
|
||||||
|
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||||
|
println!(
|
||||||
|
"hover ok tab={} target={target} at=({}, {})",
|
||||||
|
reply.tab_id, reply.x, reply.y
|
||||||
|
);
|
||||||
|
print_dialog_summaries(&reply.dialogs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// bsk fill
|
// bsk fill
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
|||||||
use crate::cli::get_html::GetHtmlArgs;
|
use crate::cli::get_html::GetHtmlArgs;
|
||||||
use crate::cli::human_loop::RequestHelpArgs;
|
use crate::cli::human_loop::RequestHelpArgs;
|
||||||
use crate::cli::install_skill::InstallSkillArgs;
|
use crate::cli::install_skill::InstallSkillArgs;
|
||||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||||
use crate::cli::network::NetworkArgs;
|
use crate::cli::network::NetworkArgs;
|
||||||
use crate::cli::observe::ObserveArgs;
|
use crate::cli::observe::ObserveArgs;
|
||||||
@@ -164,6 +164,9 @@ pub enum Command {
|
|||||||
/// Click a snapshot ref or CSS selector.
|
/// Click a snapshot ref or CSS selector.
|
||||||
Click(ClickArgs),
|
Click(ClickArgs),
|
||||||
|
|
||||||
|
/// Hover a snapshot ref or CSS selector.
|
||||||
|
Hover(HoverArgs),
|
||||||
|
|
||||||
/// Fill an input / textarea / contenteditable.
|
/// Fill an input / textarea / contenteditable.
|
||||||
Fill(FillArgs),
|
Fill(FillArgs),
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,10 @@ pub struct ObserveArgs {
|
|||||||
/// Soft cap on rendered tokens (~4 chars/token).
|
/// Soft cap on rendered tokens (~4 chars/token).
|
||||||
#[arg(long = "max-tokens")]
|
#[arg(long = "max-tokens")]
|
||||||
pub max_tokens: Option<u32>,
|
pub max_tokens: Option<u32>,
|
||||||
|
|
||||||
|
/// Include conditional surface probe diagnostics in JSON output.
|
||||||
|
#[arg(long = "debug-surfaces")]
|
||||||
|
pub debug_surfaces: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||||
@@ -42,6 +46,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
|||||||
tab_id: args.tab_id,
|
tab_id: args.tab_id,
|
||||||
max_depth: args.max_depth,
|
max_depth: args.max_depth,
|
||||||
max_tokens: args.max_tokens,
|
max_tokens: args.max_tokens,
|
||||||
|
debug_surfaces: args.debug_surfaces,
|
||||||
};
|
};
|
||||||
let reply: ObserveResult = call(sock, params)?;
|
let reply: ObserveResult = call(sock, params)?;
|
||||||
match format {
|
match format {
|
||||||
|
|||||||
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
|||||||
| Method::ToolNavigateForward
|
| Method::ToolNavigateForward
|
||||||
| Method::ToolReload
|
| Method::ToolReload
|
||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
|
| Method::ToolHover
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect
|
| Method::ToolSelect
|
||||||
|
|||||||
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
|||||||
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
|
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
|
||||||
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
||||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||||
|
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||||
|
|||||||
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
|
|||||||
assert_eq!(args.click_count, 2);
|
assert_eq!(args.click_count, 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_hover_with_settle() {
|
||||||
|
let cli = parse(&[
|
||||||
|
"bsk",
|
||||||
|
"hover",
|
||||||
|
"@e1",
|
||||||
|
"--session",
|
||||||
|
"s1",
|
||||||
|
"--settle",
|
||||||
|
"300ms",
|
||||||
|
]);
|
||||||
|
let Command::Hover(args) = cli.command else {
|
||||||
|
panic!("expected hover command");
|
||||||
|
};
|
||||||
|
assert_eq!(args.target.as_deref(), Some("@e1"));
|
||||||
|
assert_eq!(args.settle, 300);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn rejects_zero_click_count() {
|
fn rejects_zero_click_count() {
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
|||||||
tab_id: 13,
|
tab_id: 13,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
dialogs: vec![],
|
dialogs: vec![],
|
||||||
|
debug: None,
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
@@ -492,6 +493,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
|||||||
tab_id: None,
|
tab_id: None,
|
||||||
max_depth: None,
|
max_depth: None,
|
||||||
max_tokens: None,
|
max_tokens: None,
|
||||||
|
debug_surfaces: false,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "HoverParams",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"session_id"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"modifiers": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/KeyModifier"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ref": {
|
||||||
|
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"selector": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"session_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"settle_ms": {
|
||||||
|
"description": "Wait after the mouse move so hover-triggered UI can settle.",
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"description": "Target tab. Defaults to the Agent Window's active tab.",
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"timeout_ms": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"KeyModifier": {
|
||||||
|
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"alt",
|
||||||
|
"ctrl",
|
||||||
|
"meta",
|
||||||
|
"shift"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "HoverResult",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"tab_id",
|
||||||
|
"x",
|
||||||
|
"y"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"dialogs": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"used_ref": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"used_selector": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"x": {
|
||||||
|
"description": "Viewport-relative hover coordinates (CSS pixels).",
|
||||||
|
"type": "number",
|
||||||
|
"format": "double"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"type": "number",
|
||||||
|
"format": "double"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"JavaScriptDialogHandledAction": {
|
||||||
|
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"accepted",
|
||||||
|
"dismissed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"JavaScriptDialogInfo": {
|
||||||
|
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"handled",
|
||||||
|
"message",
|
||||||
|
"sequence",
|
||||||
|
"tab_id",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"default_prompt": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"handled": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||||
|
},
|
||||||
|
"has_browser_handler": {
|
||||||
|
"type": [
|
||||||
|
"boolean",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sequence": {
|
||||||
|
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint64",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogType"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"JavaScriptDialogType": {
|
||||||
|
"description": "Native JS dialog kind reported by CDP.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"alert",
|
||||||
|
"confirm",
|
||||||
|
"prompt",
|
||||||
|
"beforeunload"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,11 @@
|
|||||||
"session_id"
|
"session_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"debug_surfaces": {
|
||||||
|
"description": "Include implementation diagnostics for conditional surface probes.",
|
||||||
|
"default": false,
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"max_depth": {
|
"max_depth": {
|
||||||
"description": "Cap the depth of the rendered VOM tree.",
|
"description": "Cap the depth of the rendered VOM tree.",
|
||||||
"type": [
|
"type": [
|
||||||
|
|||||||
@@ -8,6 +8,16 @@
|
|||||||
"text"
|
"text"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"debug": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/ObserveDebug"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"dialogs": {
|
"dialogs": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
@@ -103,6 +113,73 @@
|
|||||||
"prompt",
|
"prompt",
|
||||||
"beforeunload"
|
"beforeunload"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"ObserveDebug": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"surface_probes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/SurfaceProbeDebug"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"SurfaceProbeDebug": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"sub_items",
|
||||||
|
"trigger_action",
|
||||||
|
"trigger_backend_node_id"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"confidence": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"sub_items": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"trigger_action": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"trigger_backend_node_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"trigger_point": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/SurfaceProbePoint"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"SurfaceProbePoint": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"x",
|
||||||
|
"y"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"x": {
|
||||||
|
"type": "number",
|
||||||
|
"format": "double"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"type": "number",
|
||||||
|
"format": "double"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -144,6 +144,46 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"page",
|
||||||
|
"target"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"effect": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/StepEffect"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"hover"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Reference into `pages[]`.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -144,6 +144,46 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"page",
|
||||||
|
"target"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"effect": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/StepEffect"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"hover"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Reference into `pages[]`.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -177,6 +177,46 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"page",
|
||||||
|
"target"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"effect": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/StepEffect"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"hover"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Reference into `pages[]`.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -83,6 +83,46 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"page",
|
||||||
|
"target"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"effect": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/StepEffect"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"hover"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Reference into `pages[]`.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -77,6 +77,8 @@ fn main() {
|
|||||||
|
|
||||||
dump!(ClickParams, "tool_click_params");
|
dump!(ClickParams, "tool_click_params");
|
||||||
dump!(ClickResult, "tool_click_result");
|
dump!(ClickResult, "tool_click_result");
|
||||||
|
dump!(HoverParams, "tool_hover_params");
|
||||||
|
dump!(HoverResult, "tool_hover_result");
|
||||||
dump!(FillParams, "tool_fill_params");
|
dump!(FillParams, "tool_fill_params");
|
||||||
dump!(FillResult, "tool_fill_result");
|
dump!(FillResult, "tool_fill_result");
|
||||||
dump!(PressParams, "tool_press_params");
|
dump!(PressParams, "tool_press_params");
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ pub enum Method {
|
|||||||
ToolReload,
|
ToolReload,
|
||||||
#[serde(rename = "tool.click")]
|
#[serde(rename = "tool.click")]
|
||||||
ToolClick,
|
ToolClick,
|
||||||
|
#[serde(rename = "tool.hover")]
|
||||||
|
ToolHover,
|
||||||
#[serde(rename = "tool.fill")]
|
#[serde(rename = "tool.fill")]
|
||||||
ToolFill,
|
ToolFill,
|
||||||
#[serde(rename = "tool.press")]
|
#[serde(rename = "tool.press")]
|
||||||
@@ -161,7 +163,7 @@ impl Method {
|
|||||||
|
|
||||||
// Transient input — no committed browser action, but still page
|
// Transient input — no committed browser action, but still page
|
||||||
// input. It must be stopped by pending user interrupts.
|
// input. It must be stopped by pending user interrupts.
|
||||||
Method::ToolObserve => MethodEffect::TransientInput,
|
Method::ToolHover | Method::ToolObserve => MethodEffect::TransientInput,
|
||||||
|
|
||||||
// Passive reads — transparent.
|
// Passive reads — transparent.
|
||||||
// `record_stop` / `record_await` observe / finish a recording
|
// `record_stop` / `record_await` observe / finish a recording
|
||||||
@@ -259,6 +261,7 @@ mod tests {
|
|||||||
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
|
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
|
||||||
assert!(!Method::ToolTabList.is_mutating());
|
assert!(!Method::ToolTabList.is_mutating());
|
||||||
assert!(!Method::ToolSnapshot.is_mutating());
|
assert!(!Method::ToolSnapshot.is_mutating());
|
||||||
|
assert!(!Method::ToolHover.is_mutating());
|
||||||
assert!(!Method::ToolObserve.is_mutating());
|
assert!(!Method::ToolObserve.is_mutating());
|
||||||
assert!(!Method::ToolGetHtml.is_mutating());
|
assert!(!Method::ToolGetHtml.is_mutating());
|
||||||
assert!(!Method::ToolScreenshot.is_mutating());
|
assert!(!Method::ToolScreenshot.is_mutating());
|
||||||
@@ -321,6 +324,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn effect_classifies_observe_as_transient_input() {
|
fn effect_classifies_observe_as_transient_input() {
|
||||||
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
|
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
|
||||||
|
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
|
||||||
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
|
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
|
||||||
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
|
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
|
||||||
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
|
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
|
||||||
@@ -329,6 +333,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn interrupt_gate_includes_transient_input() {
|
fn interrupt_gate_includes_transient_input() {
|
||||||
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
|
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
|
||||||
|
assert!(Method::ToolHover.requires_interrupt_gate());
|
||||||
assert!(Method::ToolObserve.requires_interrupt_gate());
|
assert!(Method::ToolObserve.requires_interrupt_gate());
|
||||||
assert!(Method::ToolClick.requires_interrupt_gate());
|
assert!(Method::ToolClick.requires_interrupt_gate());
|
||||||
assert!(!Method::Cancel.requires_interrupt_gate());
|
assert!(!Method::Cancel.requires_interrupt_gate());
|
||||||
|
|||||||
@@ -85,6 +85,52 @@ pub struct ClickResult {
|
|||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// hover
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HoverParams {
|
||||||
|
pub session_id: String,
|
||||||
|
/// Optional `@e<N>` ref allocated by the last observation.
|
||||||
|
/// Mutually exclusive with `selector`.
|
||||||
|
#[serde(
|
||||||
|
rename = "ref",
|
||||||
|
alias = "ref_",
|
||||||
|
default,
|
||||||
|
skip_serializing_if = "Option::is_none"
|
||||||
|
)]
|
||||||
|
pub ref_: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector: Option<String>,
|
||||||
|
/// Target tab. Defaults to the Agent Window's active tab.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub tab_id: Option<i64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub modifiers: Option<Vec<KeyModifier>>,
|
||||||
|
/// Wait after the mouse move so hover-triggered UI can settle.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[schemars(range(min = 0))]
|
||||||
|
pub settle_ms: Option<u32>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[schemars(range(min = 1))]
|
||||||
|
pub timeout_ms: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HoverResult {
|
||||||
|
pub tab_id: i64,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub used_ref: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub used_selector: Option<String>,
|
||||||
|
/// Viewport-relative hover coordinates (CSS pixels).
|
||||||
|
pub x: f64,
|
||||||
|
pub y: f64,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// fill
|
// fill
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -261,6 +307,24 @@ mod tests {
|
|||||||
assert_eq!(p.ref_.as_deref(), Some("e1"));
|
assert_eq!(p.ref_.as_deref(), Some("e1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hover_params_serialise_ref_field_name() {
|
||||||
|
let p = HoverParams {
|
||||||
|
session_id: "abcd".into(),
|
||||||
|
ref_: Some("@e3".into()),
|
||||||
|
selector: None,
|
||||||
|
tab_id: Some(42),
|
||||||
|
modifiers: Some(vec![KeyModifier::Shift]),
|
||||||
|
settle_ms: Some(200),
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
};
|
||||||
|
let v = serde_json::to_value(&p).unwrap();
|
||||||
|
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||||
|
assert!(v.get("ref_").is_none());
|
||||||
|
let round: HoverParams = serde_json::from_value(v).unwrap();
|
||||||
|
assert_eq!(round, p);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn modifiers_render_as_lowercase_strings() {
|
fn modifiers_render_as_lowercase_strings() {
|
||||||
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
|
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
|
||||||
|
|||||||
@@ -74,6 +74,32 @@ pub struct ObserveParams {
|
|||||||
/// best-effort heuristic based on character count).
|
/// best-effort heuristic based on character count).
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub max_tokens: Option<u32>,
|
pub max_tokens: Option<u32>,
|
||||||
|
/// Include implementation diagnostics for conditional surface probes.
|
||||||
|
#[serde(default)]
|
||||||
|
pub debug_surfaces: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct SurfaceProbePoint {
|
||||||
|
pub x: f64,
|
||||||
|
pub y: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct SurfaceProbeDebug {
|
||||||
|
pub trigger_backend_node_id: i64,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub trigger_point: Option<SurfaceProbePoint>,
|
||||||
|
pub trigger_action: String,
|
||||||
|
pub sub_items: Vec<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub confidence: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ObserveDebug {
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub surface_probes: Vec<SurfaceProbeDebug>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
@@ -92,6 +118,8 @@ pub struct ObserveResult {
|
|||||||
pub truncated: bool,
|
pub truncated: bool,
|
||||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub debug: Option<ObserveDebug>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -280,6 +308,7 @@ mod tests {
|
|||||||
tab_id: 42,
|
tab_id: 42,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
dialogs: Vec::new(),
|
dialogs: Vec::new(),
|
||||||
|
debug: None,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&r).unwrap();
|
let v = serde_json::to_value(&r).unwrap();
|
||||||
assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1));
|
assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1));
|
||||||
|
|||||||
@@ -94,6 +94,11 @@ pub enum Step {
|
|||||||
common: StepCommon,
|
common: StepCommon,
|
||||||
target: TargetDescriptor,
|
target: TargetDescriptor,
|
||||||
},
|
},
|
||||||
|
Hover {
|
||||||
|
#[serde(flatten)]
|
||||||
|
common: StepCommon,
|
||||||
|
target: TargetDescriptor,
|
||||||
|
},
|
||||||
Fill {
|
Fill {
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
common: StepCommon,
|
common: StepCommon,
|
||||||
|
|||||||
@@ -404,7 +404,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Products" [hover: Shoes | Bags | Accessories]');
|
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export { renderVom } from "./render";
|
export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render";
|
||||||
export type {
|
export type {
|
||||||
ActiveScopeBlock,
|
ActiveScopeBlock,
|
||||||
BlockingLayer,
|
BlockingLayer,
|
||||||
|
|||||||
+13
-11
@@ -197,7 +197,7 @@ function shouldRender(node: VomNode): boolean {
|
|||||||
return cleaned(node.name) !== undefined;
|
return cleaned(node.name) !== undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function shouldReference(node: VomNode): boolean {
|
export function isVomReferenceNode(node: VomNode): boolean {
|
||||||
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
|
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +342,7 @@ function isAncestorOf(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyRecovery(nodes: VomNode[]): VomNode[] {
|
export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] {
|
||||||
const parentMap = buildParentMap(nodes);
|
const parentMap = buildParentMap(nodes);
|
||||||
const children = buildChildren(nodes);
|
const children = buildChildren(nodes);
|
||||||
const recoveredIds = new Set<number>();
|
const recoveredIds = new Set<number>();
|
||||||
@@ -350,7 +350,7 @@ function applyRecovery(nodes: VomNode[]): VomNode[] {
|
|||||||
const recoveredNameById = new Map<number, string>();
|
const recoveredNameById = new Map<number, string>();
|
||||||
|
|
||||||
for (const node of nodes) {
|
for (const node of nodes) {
|
||||||
if (shouldReference(node)) continue;
|
if (isVomReferenceNode(node)) continue;
|
||||||
const role = recoveredRole(node, children);
|
const role = recoveredRole(node, children);
|
||||||
if (!role || !passesRecoveryGuards(node, children)) continue;
|
if (!role || !passesRecoveryGuards(node, children)) continue;
|
||||||
recoveredIds.add(node.id);
|
recoveredIds.add(node.id);
|
||||||
@@ -397,7 +397,7 @@ function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
|
|||||||
function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
|
function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
|
||||||
const counts = new Map<string, number>();
|
const counts = new Map<string, number>();
|
||||||
for (const node of nodes) {
|
for (const node of nodes) {
|
||||||
if (!shouldReference(node)) continue;
|
if (!isVomReferenceNode(node)) continue;
|
||||||
const name = cleaned(node.name);
|
const name = cleaned(node.name);
|
||||||
if (!name) continue;
|
if (!name) continue;
|
||||||
const key = normalizeContextKey(name);
|
const key = normalizeContextKey(name);
|
||||||
@@ -427,7 +427,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function weakLabelContext(node: VomNode, targetName: string): string | undefined {
|
function weakLabelContext(node: VomNode, targetName: string): string | undefined {
|
||||||
if (shouldReference(node)) return undefined;
|
if (isVomReferenceNode(node)) return undefined;
|
||||||
return weakContextText(
|
return weakContextText(
|
||||||
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
|
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
|
||||||
targetName,
|
targetName,
|
||||||
@@ -509,7 +509,7 @@ function collectWeakLabelsFromSubtree(
|
|||||||
state: RenderState,
|
state: RenderState,
|
||||||
labels: string[],
|
labels: string[],
|
||||||
): void {
|
): void {
|
||||||
if (shouldReference(node)) return;
|
if (isVomReferenceNode(node)) return;
|
||||||
const label = weakLabelContext(node, nodeName);
|
const label = weakLabelContext(node, nodeName);
|
||||||
if (label) {
|
if (label) {
|
||||||
labels.push(label);
|
labels.push(label);
|
||||||
@@ -637,7 +637,9 @@ function renderNodeLine(
|
|||||||
if (surface && surface.subItems.length > 0) {
|
if (surface && surface.subItems.length > 0) {
|
||||||
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
|
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
|
||||||
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
|
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
|
||||||
line += ` [${surface.triggerAction}: ${items}${suffix}]`;
|
const action =
|
||||||
|
surface.triggerAction === "hover" ? "hover first" : `${surface.triggerAction} first`;
|
||||||
|
line += ` [${action}: ${items}${suffix}]`;
|
||||||
}
|
}
|
||||||
|
|
||||||
return line;
|
return line;
|
||||||
@@ -664,7 +666,7 @@ function descendantTextCoveredByName(node: VomNode, state: RenderState): boolean
|
|||||||
const texts: string[] = [];
|
const texts: string[] = [];
|
||||||
while (stack.length > 0) {
|
while (stack.length > 0) {
|
||||||
const child = stack.pop() as VomNode;
|
const child = stack.pop() as VomNode;
|
||||||
if (shouldReference(child)) return false;
|
if (isVomReferenceNode(child)) return false;
|
||||||
const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value);
|
const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value);
|
||||||
if (text) texts.push(text);
|
if (text) texts.push(text);
|
||||||
stack.push(...(state.children.get(child.id) ?? []));
|
stack.push(...(state.children.get(child.id) ?? []));
|
||||||
@@ -748,7 +750,7 @@ function renderTree(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined;
|
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||||
const context = ref ? handleContext(node, state) : [];
|
const context = ref ? handleContext(node, state) : [];
|
||||||
const surface = state.surfaceMap.get(node.id);
|
const surface = state.surfaceMap.get(node.id);
|
||||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
const line = renderNodeLine(node, depth, ref, context, surface);
|
||||||
@@ -913,7 +915,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
|||||||
|
|
||||||
const blockedRoots = new Set<number>();
|
const blockedRoots = new Set<number>();
|
||||||
for (const target of nodes) {
|
for (const target of nodes) {
|
||||||
if (!shouldReference(target)) continue;
|
if (!isVomReferenceNode(target)) continue;
|
||||||
const blocked = candidates.some((candidate) =>
|
const blocked = candidates.some((candidate) =>
|
||||||
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
|
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
|
||||||
);
|
);
|
||||||
@@ -968,7 +970,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
|
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
|
||||||
const nodes = applyRecovery(scene.nodes);
|
const nodes = applyVomInteractionRecovery(scene.nodes);
|
||||||
const renderScene = { ...scene, nodes };
|
const renderScene = { ...scene, nodes };
|
||||||
const layer = detectBlockingLayer(nodes, scene.viewport);
|
const layer = detectBlockingLayer(nodes, scene.viewport);
|
||||||
if (layer) return renderDoubleLayer(renderScene, layer, options);
|
if (layer) return renderDoubleLayer(renderScene, layer, options);
|
||||||
|
|||||||
+11
-7
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
|||||||
|
|
||||||
```
|
```
|
||||||
bsk navigate <url> --session <id>
|
bsk navigate <url> --session <id>
|
||||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||||
bsk snapshot --session <id> → again after navigation / DOM change
|
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||||
|
bsk observe --session <id> → again after navigation / DOM change
|
||||||
```
|
```
|
||||||
|
|
||||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||||
|
|
||||||
|
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||||
|
|
||||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||||
|
|
||||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
|||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||||
|
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||||
|
|||||||
Reference in New Issue
Block a user