mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): add bsk hover and adjust vom to support
This commit is contained in:
@@ -1,4 +1,9 @@
|
||||
import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
|
||||
import {
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
resolveHoverElement,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
@@ -49,8 +54,16 @@ interface FillSession {
|
||||
lastValue: string;
|
||||
}
|
||||
|
||||
interface PendingHover {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
recordedAt: number;
|
||||
}
|
||||
|
||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||
|
||||
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||
|
||||
function eventTarget(event: Event): EventTarget | null {
|
||||
return event.composedPath()[0] ?? event.target;
|
||||
}
|
||||
@@ -122,6 +135,14 @@ function fillableValue(el: FillableElement): string {
|
||||
return el.textContent ?? "";
|
||||
}
|
||||
|
||||
function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const element = resolveHoverElement(target);
|
||||
if (!element) return null;
|
||||
const desc = describeTarget(element);
|
||||
return { element, target: desc, recordedAt: Date.now() };
|
||||
}
|
||||
|
||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||
if (!session || !(target instanceof Element)) return false;
|
||||
@@ -172,6 +193,7 @@ export function startRecordCapture(
|
||||
let composing = false;
|
||||
let lastUrl = location.href;
|
||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||
let pendingHover: PendingHover | null = null;
|
||||
let generatedControlClick: Element | null = null;
|
||||
let navigationActionPending = false;
|
||||
let navigationActionVersion = 0;
|
||||
@@ -249,6 +271,28 @@ export function startRecordCapture(
|
||||
});
|
||||
};
|
||||
|
||||
const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => {
|
||||
if (!pendingHover || !(clickTarget instanceof Element)) return;
|
||||
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
|
||||
pendingHover = null;
|
||||
return;
|
||||
}
|
||||
if (pendingHover.element.contains(clickTarget)) return;
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: pendingHover.target,
|
||||
});
|
||||
pendingHover = null;
|
||||
};
|
||||
|
||||
const onMouseOver = (event: MouseEvent) => {
|
||||
const target = eventTarget(event);
|
||||
if (isOverlayTarget(target)) return;
|
||||
const hover = hoverTargetFromEvent(target);
|
||||
if (!hover || hover.element === pendingHover?.element) return;
|
||||
pendingHover = hover;
|
||||
};
|
||||
|
||||
const onClick = (event: MouseEvent) => {
|
||||
if (event.button !== 0) return;
|
||||
const target = eventTarget(event);
|
||||
@@ -304,6 +348,7 @@ export function startRecordCapture(
|
||||
|
||||
commitFillSession();
|
||||
if (target instanceof Element && target.closest("select")) return;
|
||||
emitPendingHoverBeforeClick(target);
|
||||
emitClick(event);
|
||||
};
|
||||
|
||||
@@ -408,6 +453,7 @@ export function startRecordCapture(
|
||||
};
|
||||
|
||||
document.addEventListener("click", onClick, true);
|
||||
document.addEventListener("mouseover", onMouseOver, true);
|
||||
document.addEventListener("focusin", onFocusIn, true);
|
||||
document.addEventListener("focusout", onFocusOut, true);
|
||||
document.addEventListener("input", onInput, true);
|
||||
@@ -438,6 +484,7 @@ export function startRecordCapture(
|
||||
dispose() {
|
||||
commitFillSession();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("mouseover", onMouseOver, true);
|
||||
document.removeEventListener("focusin", onFocusIn, true);
|
||||
document.removeEventListener("focusout", onFocusOut, true);
|
||||
document.removeEventListener("input", onInput, true);
|
||||
|
||||
@@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "span", role: "button", name: "Account" },
|
||||
page_url: "https://example.com/app",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "a", role: "link", name: "Profile" },
|
||||
page_url: "https://example.com/app",
|
||||
},
|
||||
],
|
||||
"https://example.com/app",
|
||||
);
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
page: "p1",
|
||||
target: { name: "Account" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
|
||||
@@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveHoverElement(target: Element): Element | null {
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (clickable) return clickable;
|
||||
|
||||
let node: Element | null = target;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
|
||||
if (looksClickable(node)) return node;
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
|
||||
@@ -33,7 +33,7 @@ export interface RecordStartMessage {
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "fill" | "press" | "select" | "navigate";
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
|
||||
@@ -139,6 +139,13 @@ function toV2Step(
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
};
|
||||
case "fill":
|
||||
return {
|
||||
op: "fill",
|
||||
|
||||
@@ -311,6 +311,84 @@ describe("ToolDispatcher", () => {
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
const cdp = {
|
||||
send: vi.fn(async () => {
|
||||
order.push("hover");
|
||||
return {};
|
||||
}),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
|
||||
(dispatcher as unknown as { rememberHover: (result: unknown) => unknown }).rememberHover({
|
||||
tab_id: 7,
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
withHoverReassert: <T>(
|
||||
work: () => Promise<T>,
|
||||
options?: { releaseAfter?: boolean },
|
||||
) => Promise<T>;
|
||||
setHoverBypass: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
};
|
||||
|
||||
await helpers.withHoverReassert(async () => {
|
||||
order.push("observe");
|
||||
return {};
|
||||
});
|
||||
expect(order).toEqual(["hover", "observe"]);
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.setHoverBypass(7, true);
|
||||
await helpers.withHoverReassert(
|
||||
async () => {
|
||||
order.push("click");
|
||||
return {};
|
||||
},
|
||||
{ releaseAfter: true },
|
||||
);
|
||||
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
handleClick,
|
||||
handleFill,
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
modifiersBitfield,
|
||||
@@ -435,6 +436,50 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleHover", () => {
|
||||
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
|
||||
const res = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
|
||||
{
|
||||
cdp: fake.cdp,
|
||||
tabsApi: fake.tabsApi,
|
||||
bypassOverlay,
|
||||
keepOverlayBypassAfterHover: true,
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||
expect(bypassOverlay).toHaveBeenCalledTimes(1);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
|
||||
@@ -1480,7 +1480,7 @@ describe("buildVomScene", () => {
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
|
||||
@@ -8,6 +8,8 @@ import type {
|
||||
EvaluateParams,
|
||||
FillParams,
|
||||
GetHtmlParams,
|
||||
HoverParams,
|
||||
HoverResult,
|
||||
NavigateBackParams,
|
||||
NavigateForwardParams,
|
||||
NavigateParams,
|
||||
@@ -33,7 +35,7 @@ import { handleConsole } from "./console";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
handleNavigateBack,
|
||||
@@ -126,6 +128,8 @@ export class ToolDispatcher {
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
private readonly hoverBypassTabs = new Set<number>();
|
||||
private readonly hoverLatches = new Map<number, { x: number; y: number }>();
|
||||
/**
|
||||
* Per-rpc-id `AbortController` registry. Populated inside
|
||||
* [`dispatch`] before we await the tool handler and torn down in
|
||||
@@ -271,10 +275,12 @@ export class ToolDispatcher {
|
||||
switch (req.method) {
|
||||
case "tool.session_start":
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
||||
case "tool.session_stop":
|
||||
case "tool.session_stop": {
|
||||
await this.releaseHoverLatch();
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams);
|
||||
case "tool.tab_create":
|
||||
@@ -319,16 +325,26 @@ export class ToolDispatcher {
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
);
|
||||
case "tool.snapshot":
|
||||
return handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
return this.withHoverReassert(() =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.observe":
|
||||
return handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
return this.withHoverReassert(() =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatch(),
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
);
|
||||
case "tool.get_html":
|
||||
return handleGetHtml(
|
||||
@@ -361,27 +377,38 @@ export class ToolDispatcher {
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
);
|
||||
case "tool.click":
|
||||
return handleClick(
|
||||
return this.withHoverReassert(
|
||||
() =>
|
||||
handleClick(
|
||||
this.sessions,
|
||||
req.params as ClickParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
{ releaseAfter: true },
|
||||
);
|
||||
case "tool.hover": {
|
||||
const result = await handleHover(
|
||||
this.sessions,
|
||||
req.params as ClickParams,
|
||||
req.params as HoverParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
bypassOverlay: (tabId, enabled) => this.setHoverBypass(tabId, enabled),
|
||||
keepOverlayBypassAfterHover: true,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
return this.rememberHover(result);
|
||||
}
|
||||
case "tool.fill":
|
||||
return handleFill(
|
||||
this.sessions,
|
||||
@@ -471,6 +498,66 @@ export class ToolDispatcher {
|
||||
} satisfies RpcError;
|
||||
}
|
||||
}
|
||||
|
||||
private async setHoverBypass(tabId: number, enabled: boolean): Promise<void> {
|
||||
if (enabled) {
|
||||
if (this.hoverBypassTabs.has(tabId)) return;
|
||||
await bypassOverlay(tabId, true);
|
||||
this.hoverBypassTabs.add(tabId);
|
||||
} else {
|
||||
if (!this.hoverBypassTabs.has(tabId)) return;
|
||||
await bypassOverlay(tabId, false);
|
||||
this.hoverBypassTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
private rememberHover(result: HoverResult | RpcError): HoverResult | RpcError {
|
||||
if (!isRpcError(result)) {
|
||||
this.hoverLatches.set(result.tab_id, { x: result.x, y: result.y });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private hasHoverLatch(): boolean {
|
||||
return this.hoverLatches.size > 0;
|
||||
}
|
||||
|
||||
private async withHoverReassert<T>(
|
||||
work: () => Promise<T>,
|
||||
options: { releaseAfter?: boolean } = {},
|
||||
): Promise<T> {
|
||||
await this.reassertHover();
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
if (options.releaseAfter) {
|
||||
await this.releaseHoverLatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async reassertHover(): Promise<void> {
|
||||
if (!this.cdp) return;
|
||||
await Promise.all(
|
||||
[...this.hoverLatches.entries()].map(([tabId, point]) =>
|
||||
this.cdp!.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
}).catch((err) => {
|
||||
console.debug("[bsk dispatcher] hover reassert failed", err);
|
||||
this.hoverLatches.delete(tabId);
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private async releaseHoverLatch(): Promise<void> {
|
||||
const tabs = new Set([...this.hoverBypassTabs, ...this.hoverLatches.keys()]);
|
||||
this.hoverBypassTabs.clear();
|
||||
this.hoverLatches.clear();
|
||||
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
|
||||
}
|
||||
}
|
||||
|
||||
function isRpcError(v: unknown): v is RpcError {
|
||||
@@ -497,6 +584,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.navigate_forward":
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.hover":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
@@ -512,6 +600,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
|
||||
@@ -17,6 +17,8 @@ import type {
|
||||
ClickResult,
|
||||
FillParams,
|
||||
FillResult,
|
||||
HoverParams,
|
||||
HoverResult,
|
||||
KeyModifier,
|
||||
MouseButton,
|
||||
PressParams,
|
||||
@@ -53,9 +55,12 @@ export interface InteractionDeps {
|
||||
defaultTimeoutMs?: number;
|
||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||
|
||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||
@@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isAbortLikeError(err: unknown): boolean {
|
||||
return err instanceof DOMException && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => signal?.removeEventListener("abort", onAbort);
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
reject(new DOMException("aborted", "AbortError"));
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
@@ -286,7 +312,104 @@ export async function handleClick(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
if (automationBypassEnabled && deps.bypassOverlay) {
|
||||
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function handleHover(
|
||||
manager: SessionManager,
|
||||
params: HoverParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<HoverResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "hover");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const aborted = throwIfAborted(deps.signal);
|
||||
if (aborted) return aborted;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "hover");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
}
|
||||
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
let hoverCompleted = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
automationBypassEnabled = true;
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
modifiers,
|
||||
});
|
||||
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
|
||||
if (settleMs > 0) {
|
||||
await wait(settleMs, deps.signal);
|
||||
}
|
||||
hoverCompleted = true;
|
||||
} catch (err) {
|
||||
if (isAbortLikeError(err)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
}
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
if (
|
||||
automationBypassEnabled &&
|
||||
deps.bypassOverlay &&
|
||||
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
|
||||
) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
|
||||
@@ -1072,6 +1072,7 @@ export interface SnapshotDeps {
|
||||
get(tabId: number): Promise<chrome.tabs.Tab>;
|
||||
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
||||
};
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
}
|
||||
|
||||
let defaultDeps: SnapshotDeps | null = null;
|
||||
@@ -1258,7 +1259,9 @@ async function handleVomObservation(
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, effectiveConditionalSurfaceProbe);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
|
||||
@@ -410,6 +410,25 @@ export interface ClickResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface HoverParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
modifiers?: KeyModifier[];
|
||||
settle_ms?: number;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface HoverResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface FillParams {
|
||||
session_id: string;
|
||||
value: string;
|
||||
@@ -668,6 +687,11 @@ export type DraftTraceStep =
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "hover";
|
||||
target: TargetDescriptor;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
target: TargetDescriptor;
|
||||
@@ -701,6 +725,7 @@ export type DraftTraceStep =
|
||||
export type Step =
|
||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "fill" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
|
||||
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk snapshot --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
@@ -10,8 +10,8 @@ use std::time::Duration;
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
|
||||
PressResult, SelectParams, SelectResult,
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct HoverArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||
pub target: Option<String>,
|
||||
|
||||
/// Force-treat the value as a snapshot ref (overrides positional detection).
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
/// Force-treat the value as a CSS selector.
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
|
||||
#[arg(long, default_value = "")]
|
||||
pub modifiers: String,
|
||||
|
||||
/// Wait after moving the mouse so hover-triggered UI can settle.
|
||||
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
|
||||
pub settle: u32,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(
|
||||
args.target.clone(),
|
||||
args.ref_.clone(),
|
||||
args.selector.clone(),
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let params = HoverParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
modifiers: if modifiers.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(modifiers)
|
||||
},
|
||||
settle_ms: Some(args.settle),
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: HoverResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolHover,
|
||||
params,
|
||||
"hover-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target =
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||
println!(
|
||||
"hover ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -164,6 +164,9 @@ pub enum Command {
|
||||
/// Click a snapshot ref or CSS selector.
|
||||
Click(ClickArgs),
|
||||
|
||||
/// Hover a snapshot ref or CSS selector.
|
||||
Hover(HoverArgs),
|
||||
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
|
||||
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolNavigateForward
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
|
||||
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
|
||||
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
|
||||
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"hover",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--settle",
|
||||
"300ms",
|
||||
]);
|
||||
let Command::Hover(args) = cli.command else {
|
||||
panic!("expected hover command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("@e1"));
|
||||
assert_eq!(args.settle, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_click_count() {
|
||||
assert!(
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "HoverParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"ref": {
|
||||
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"settle_ms": {
|
||||
"description": "Wait after the mouse move so hover-triggered UI can settle.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"description": "Target tab. Defaults to the Agent Window's active tab.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "HoverResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tab_id",
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"x": {
|
||||
"description": "Viewport-relative hover coordinates (CSS pixels).",
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"y": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -144,6 +144,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -144,6 +144,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -177,6 +177,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -83,6 +83,46 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
|
||||
@@ -77,6 +77,8 @@ fn main() {
|
||||
|
||||
dump!(ClickParams, "tool_click_params");
|
||||
dump!(ClickResult, "tool_click_result");
|
||||
dump!(HoverParams, "tool_hover_params");
|
||||
dump!(HoverResult, "tool_hover_result");
|
||||
dump!(FillParams, "tool_fill_params");
|
||||
dump!(FillResult, "tool_fill_result");
|
||||
dump!(PressParams, "tool_press_params");
|
||||
|
||||
@@ -70,6 +70,8 @@ pub enum Method {
|
||||
ToolReload,
|
||||
#[serde(rename = "tool.click")]
|
||||
ToolClick,
|
||||
#[serde(rename = "tool.hover")]
|
||||
ToolHover,
|
||||
#[serde(rename = "tool.fill")]
|
||||
ToolFill,
|
||||
#[serde(rename = "tool.press")]
|
||||
@@ -161,7 +163,7 @@ impl Method {
|
||||
|
||||
// Transient input — no committed browser action, but still page
|
||||
// input. It must be stopped by pending user interrupts.
|
||||
Method::ToolObserve => MethodEffect::TransientInput,
|
||||
Method::ToolHover | Method::ToolObserve => MethodEffect::TransientInput,
|
||||
|
||||
// Passive reads — transparent.
|
||||
// `record_stop` / `record_await` observe / finish a recording
|
||||
@@ -259,6 +261,7 @@ mod tests {
|
||||
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
|
||||
assert!(!Method::ToolTabList.is_mutating());
|
||||
assert!(!Method::ToolSnapshot.is_mutating());
|
||||
assert!(!Method::ToolHover.is_mutating());
|
||||
assert!(!Method::ToolObserve.is_mutating());
|
||||
assert!(!Method::ToolGetHtml.is_mutating());
|
||||
assert!(!Method::ToolScreenshot.is_mutating());
|
||||
@@ -321,6 +324,7 @@ mod tests {
|
||||
#[test]
|
||||
fn effect_classifies_observe_as_transient_input() {
|
||||
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
|
||||
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
|
||||
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
|
||||
@@ -329,6 +333,7 @@ mod tests {
|
||||
#[test]
|
||||
fn interrupt_gate_includes_transient_input() {
|
||||
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
|
||||
assert!(Method::ToolHover.requires_interrupt_gate());
|
||||
assert!(Method::ToolObserve.requires_interrupt_gate());
|
||||
assert!(Method::ToolClick.requires_interrupt_gate());
|
||||
assert!(!Method::Cancel.requires_interrupt_gate());
|
||||
|
||||
@@ -85,6 +85,52 @@ pub struct ClickResult {
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverParams {
|
||||
pub session_id: String,
|
||||
/// Optional `@e<N>` ref allocated by the last observation.
|
||||
/// Mutually exclusive with `selector`.
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// Target tab. Defaults to the Agent Window's active tab.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub modifiers: Option<Vec<KeyModifier>>,
|
||||
/// Wait after the mouse move so hover-triggered UI can settle.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 0))]
|
||||
pub settle_ms: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
/// Viewport-relative hover coordinates (CSS pixels).
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// fill
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -261,6 +307,24 @@ mod tests {
|
||||
assert_eq!(p.ref_.as_deref(), Some("e1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hover_params_serialise_ref_field_name() {
|
||||
let p = HoverParams {
|
||||
session_id: "abcd".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
tab_id: Some(42),
|
||||
modifiers: Some(vec![KeyModifier::Shift]),
|
||||
settle_ms: Some(200),
|
||||
timeout_ms: Some(5_000),
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||
assert!(v.get("ref_").is_none());
|
||||
let round: HoverParams = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn modifiers_render_as_lowercase_strings() {
|
||||
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
|
||||
|
||||
@@ -94,6 +94,11 @@ pub enum Step {
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Hover {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
|
||||
@@ -404,7 +404,7 @@ describe("renderVom single-layer page", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [hover: Shoes | Bags | Accessories]');
|
||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
|
||||
@@ -637,7 +637,9 @@ function renderNodeLine(
|
||||
if (surface && surface.subItems.length > 0) {
|
||||
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
|
||||
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
|
||||
line += ` [${surface.triggerAction}: ${items}${suffix}]`;
|
||||
const action =
|
||||
surface.triggerAction === "hover" ? "hover first" : `${surface.triggerAction} first`;
|
||||
line += ` [${action}: ${items}${suffix}]`;
|
||||
}
|
||||
|
||||
return line;
|
||||
|
||||
+11
-7
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk snapshot --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
Reference in New Issue
Block a user