diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 12bf6af..8c9d5d0 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,4 +1,9 @@ -import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target"; +import { + describeEventTarget, + describeTarget, + resolveHoverElement, + type TargetDescriptor, +} from "@/lib/describe-target"; import { isRecordCancelMessage, isRecordStartMessage, @@ -49,8 +54,16 @@ interface FillSession { lastValue: string; } +interface PendingHover { + element: Element; + target: TargetDescriptor; + recordedAt: number; +} + type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; +const HOVER_BEFORE_CLICK_MAX_MS = 10_000; + function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; } @@ -122,6 +135,14 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } +function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { + if (!(target instanceof Element)) return null; + const element = resolveHoverElement(target); + if (!element) return null; + const desc = describeTarget(element); + return { element, target: desc, recordedAt: Date.now() }; +} + /** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */ function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean { if (!session || !(target instanceof Element)) return false; @@ -172,6 +193,7 @@ export function startRecordCapture( let composing = false; let lastUrl = location.href; let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined; + let pendingHover: PendingHover | null = null; let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -249,6 +271,28 @@ export function startRecordCapture( }); }; + const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => { + if (!pendingHover || !(clickTarget instanceof Element)) return; + if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { + pendingHover = null; + return; + } + if (pendingHover.element.contains(clickTarget)) return; + emitStep({ + op: "hover", + target: pendingHover.target, + }); + pendingHover = null; + }; + + const onMouseOver = (event: MouseEvent) => { + const target = eventTarget(event); + if (isOverlayTarget(target)) return; + const hover = hoverTargetFromEvent(target); + if (!hover || hover.element === pendingHover?.element) return; + pendingHover = hover; + }; + const onClick = (event: MouseEvent) => { if (event.button !== 0) return; const target = eventTarget(event); @@ -304,6 +348,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; + emitPendingHoverBeforeClick(target); emitClick(event); }; @@ -408,6 +453,7 @@ export function startRecordCapture( }; document.addEventListener("click", onClick, true); + document.addEventListener("mouseover", onMouseOver, true); document.addEventListener("focusin", onFocusIn, true); document.addEventListener("focusout", onFocusOut, true); document.addEventListener("input", onInput, true); @@ -438,6 +484,7 @@ export function startRecordCapture( dispose() { commitFillSession(); document.removeEventListener("click", onClick, true); + document.removeEventListener("mouseover", onMouseOver, true); document.removeEventListener("focusin", onFocusIn, true); document.removeEventListener("focusout", onFocusOut, true); document.removeEventListener("input", onInput, true); diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer.test.ts index 777a86c..8cc773c 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer.test.ts @@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => { }); }); + it("keeps hover steps before menu clicks", () => { + const { steps } = reduceTraceSteps( + [ + { + op: "hover", + target: { tag: "span", role: "button", name: "Account" }, + page_url: "https://example.com/app", + }, + { + op: "click", + target: { tag: "a", role: "link", name: "Profile" }, + page_url: "https://example.com/app", + }, + ], + "https://example.com/app", + ); + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + page: "p1", + target: { name: "Account" }, + }); + }); + it("resolveTraceStartUrl prefers explicit start URL", () => { expect( resolveTraceStartUrl( diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts index 7d3898a..0a1fcb2 100644 --- a/apps/extension/src/lib/describe-target.ts +++ b/apps/extension/src/lib/describe-target.ts @@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null { return null; } +export function resolveHoverElement(target: Element): Element | null { + const clickable = resolveClickableElement(target); + if (clickable) return clickable; + + let node: Element | null = target; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 8) { + if (looksClickable(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + /** * Teachable clicks: the LLM must get a label it can later find via snapshot * (visible name), or at least a form `name_attr` for checkbox/radio. diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts index cb5d446..cb7eb88 100644 --- a/apps/extension/src/lib/record-bridge.ts +++ b/apps/extension/src/lib/record-bridge.ts @@ -33,7 +33,7 @@ export interface RecordStartMessage { } export interface RecordStepPayload { - op: "click" | "fill" | "press" | "select" | "navigate"; + op: "click" | "hover" | "fill" | "press" | "select" | "navigate"; target?: TargetDescriptor; value?: string; key?: string; diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/trace-reducer.ts index e3572ee..6da0c3d 100644 --- a/apps/extension/src/lib/trace-reducer.ts +++ b/apps/extension/src/lib/trace-reducer.ts @@ -139,6 +139,13 @@ function toV2Step( }, effectForNavigation(step.navigated_to, urlToId), ); + case "hover": + return { + op: "hover", + id, + page, + target: step.target, + }; case "fill": return { op: "fill", diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 060b30a..1b99653 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -311,6 +311,84 @@ describe("ToolDispatcher", () => { expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); }); + it("reasserts remembered hover before follow-up work and releases only after actions", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const order: string[] = []; + const cdp = { + send: vi.fn(async () => { + order.push("hover"); + return {}; + }), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + + (dispatcher as unknown as { rememberHover: (result: unknown) => unknown }).rememberHover({ + tab_id: 7, + x: 10, + y: 20, + }); + const helpers = dispatcher as unknown as { + withHoverReassert: ( + work: () => Promise, + options?: { releaseAfter?: boolean }, + ) => Promise; + setHoverBypass: (tabId: number, enabled: boolean) => Promise; + }; + + await helpers.withHoverReassert(async () => { + order.push("observe"); + return {}; + }); + expect(order).toEqual(["hover", "observe"]); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.setHoverBypass(7, true); + await helpers.withHoverReassert( + async () => { + order.push("click"); + return {}; + }, + { releaseAfter: true }, + ); + + expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 10, + y: 20, + }); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + it("disconnects the transport when send() fails so keepalive can rebuild", async () => { const { transport, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index de56af4..6f8eeed 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared"; import { handleClick, handleFill, + handleHover, handlePress, handleSelect, modifiersBitfield, @@ -435,6 +436,50 @@ describe("handleClick", () => { }); }); +describe("handleHover", () => { + it("keeps overlay bypass enabled after a successful hover when requested", async () => { + const bypassOverlay = vi.fn().mockResolvedValue(undefined); + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + "Runtime.evaluate": (params: unknown) => { + const expr = String((params as { expression?: string })?.expression ?? ""); + if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { + return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; + } + if (expr.includes("hitIndex")) { + return { + result: { + value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, + }, + }; + } + throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); + }, + "Input.dispatchMouseEvent": () => ({}), + }); + + const res = await handleHover( + sm, + { session_id: "aa11", ref: "@e3", settle_ms: 0 }, + { + cdp: fake.cdp, + tabsApi: fake.tabsApi, + bypassOverlay, + keepOverlayBypassAfterHover: true, + }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); + expect(bypassOverlay).toHaveBeenCalledTimes(1); + expect(bypassOverlay).toHaveBeenCalledWith(4, true); + }); +}); + describe("handleFill", () => { it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index d6cd621..643ad0b 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1480,7 +1480,7 @@ describe("buildVomScene", () => { expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ]); - expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]'); + expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); it("enriches names and active scope signals from AX properties", () => { diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index dd419fb..308db59 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -8,6 +8,8 @@ import type { EvaluateParams, FillParams, GetHtmlParams, + HoverParams, + HoverResult, NavigateBackParams, NavigateForwardParams, NavigateParams, @@ -33,7 +35,7 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handlePress, handleSelect } from "./interaction"; +import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -126,6 +128,8 @@ export class ToolDispatcher { private readonly approveBorrow?: BorrowConfirmationApprover; private readonly helpNotificationCopy?: () => { title: string; body: string }; private subscription: { dispose(): void } | null = null; + private readonly hoverBypassTabs = new Set(); + private readonly hoverLatches = new Map(); /** * Per-rpc-id `AbortController` registry. Populated inside * [`dispatch`] before we await the tool handler and torn down in @@ -271,10 +275,12 @@ export class ToolDispatcher { switch (req.method) { case "tool.session_start": return handleSessionStart(this.sessions, req.params as SessionStartParams); - case "tool.session_stop": + case "tool.session_stop": { + await this.releaseHoverLatch(); return handleSessionStop(this.sessions, req.params as SessionStopParams, { cdp: this.cdp, }); + } case "tool.tab_list": return handleTabList(this.sessions, req.params as TabListParams); case "tool.tab_create": @@ -319,16 +325,26 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined, ); case "tool.snapshot": - return handleSnapshot( - this.sessions, - req.params as SnapshotParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + return this.withHoverReassert(() => + handleSnapshot( + this.sessions, + req.params as SnapshotParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + ), ); case "tool.observe": - return handleObserve( - this.sessions, - req.params as ObserveParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + return this.withHoverReassert(() => + handleObserve( + this.sessions, + req.params as ObserveParams, + this.cdp + ? { + cdp: this.cdp, + tabsApi: chromeTabsCaptureApi, + conditionalSurfaceProbe: !this.hasHoverLatch(), + } + : undefined, + ), ); case "tool.get_html": return handleGetHtml( @@ -361,27 +377,38 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, ); case "tool.click": - return handleClick( + return this.withHoverReassert( + () => + handleClick( + this.sessions, + req.params as ClickParams, + this.cdp + ? { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + bypassOverlay, + } + : undefined, + ), + { releaseAfter: true }, + ); + case "tool.hover": { + const result = await handleHover( this.sessions, - req.params as ClickParams, + req.params as HoverParams, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, - bypassOverlay: async (tabId, enabled) => { - try { - await chrome.tabs.sendMessage(tabId, { - type: OVERLAY_AUTOMATION_BYPASS, - enabled, - }); - } catch { - // Content script may be unavailable on restricted pages. - } - }, + bypassOverlay: (tabId, enabled) => this.setHoverBypass(tabId, enabled), + keepOverlayBypassAfterHover: true, } : undefined, ); + return this.rememberHover(result); + } case "tool.fill": return handleFill( this.sessions, @@ -471,6 +498,66 @@ export class ToolDispatcher { } satisfies RpcError; } } + + private async setHoverBypass(tabId: number, enabled: boolean): Promise { + if (enabled) { + if (this.hoverBypassTabs.has(tabId)) return; + await bypassOverlay(tabId, true); + this.hoverBypassTabs.add(tabId); + } else { + if (!this.hoverBypassTabs.has(tabId)) return; + await bypassOverlay(tabId, false); + this.hoverBypassTabs.delete(tabId); + } + } + + private rememberHover(result: HoverResult | RpcError): HoverResult | RpcError { + if (!isRpcError(result)) { + this.hoverLatches.set(result.tab_id, { x: result.x, y: result.y }); + } + return result; + } + + private hasHoverLatch(): boolean { + return this.hoverLatches.size > 0; + } + + private async withHoverReassert( + work: () => Promise, + options: { releaseAfter?: boolean } = {}, + ): Promise { + await this.reassertHover(); + try { + return await work(); + } finally { + if (options.releaseAfter) { + await this.releaseHoverLatch(); + } + } + } + + private async reassertHover(): Promise { + if (!this.cdp) return; + await Promise.all( + [...this.hoverLatches.entries()].map(([tabId, point]) => + this.cdp!.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + }).catch((err) => { + console.debug("[bsk dispatcher] hover reassert failed", err); + this.hoverLatches.delete(tabId); + }), + ), + ); + } + + private async releaseHoverLatch(): Promise { + const tabs = new Set([...this.hoverBypassTabs, ...this.hoverLatches.keys()]); + this.hoverBypassTabs.clear(); + this.hoverLatches.clear(); + await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false))); + } } function isRpcError(v: unknown): v is RpcError { @@ -497,6 +584,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.navigate_forward": case "tool.reload": case "tool.click": + case "tool.hover": case "tool.fill": case "tool.press": case "tool.select": @@ -512,6 +600,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { } } +async function bypassOverlay(tabId: number, enabled: boolean): Promise { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } +} + /** * Resolves never; rejects with `AbortLikeError` as soon as the signal * fires (or immediately if it is already aborted). Used by the diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 87decea..ee66dac 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -17,6 +17,8 @@ import type { ClickResult, FillParams, FillResult, + HoverParams, + HoverResult, KeyModifier, MouseButton, PressParams, @@ -53,9 +55,12 @@ export interface InteractionDeps { defaultTimeoutMs?: number; /** Temporarily disable overlay click blocker during CDP automation. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + /** Keep hover hit-testing active for the caller's next observation/action. */ + keepOverlayBypassAfterHover?: boolean; } const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_HOVER_SETTLE_MS = 200; let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { @@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null { return null; } +function isAbortLikeError(err: unknown): boolean { + return err instanceof DOMException && err.name === "AbortError"; +} + +async function wait(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) throw new DOMException("aborted", "AbortError"); + await new Promise((resolve, reject) => { + const cleanup = () => signal?.removeEventListener("abort", onAbort); + const timer = setTimeout(() => { + cleanup(); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + cleanup(); + reject(new DOMException("aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + /** * Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an * `RpcError` if the caller supplied neither (or both), or if neither @@ -286,7 +312,104 @@ export async function handleClick( message: err instanceof Error ? err.message : String(err), }; } finally { - if (automationBypassEnabled && deps.bypassOverlay) { + if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (err) { + console.debug("[bsk interaction] overlay bypass disable failed", err); + } + } + } + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + x: centre.x, + y: centre.y, + }); +} + +// --------------------------------------------------------------------------- +// tool.hover +// --------------------------------------------------------------------------- + +export async function handleHover( + manager: SessionManager, + params: HoverParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "hover"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "hover"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); + if (isRpcError(node)) return node; + + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + if (scrollErr) return scrollErr; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + + const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); + if (isRpcError(centre)) return centre; + + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "hover aborted" }; + } + + const modifiers = modifiersBitfield(params.modifiers); + const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); + let automationBypassEnabled = false; + let hoverCompleted = false; + if (overlayBlocking && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, true); + automationBypassEnabled = true; + } catch (err) { + console.debug("[bsk interaction] overlay bypass enable failed", err); + } + } + + try { + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: centre.x, + y: centre.y, + modifiers, + }); + const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS; + if (settleMs > 0) { + await wait(settleMs, deps.signal); + } + hoverCompleted = true; + } catch (err) { + if (isAbortLikeError(err)) { + return { code: "cancelled", message: "hover aborted" }; + } + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + if ( + automationBypassEnabled && + deps.bypassOverlay && + (!deps.keepOverlayBypassAfterHover || !hoverCompleted) + ) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 48fe799..11ad305 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1072,6 +1072,7 @@ export interface SnapshotDeps { get(tabId: number): Promise; query(q: chrome.tabs.QueryInfo): Promise; }; + conditionalSurfaceProbe?: boolean; } let defaultDeps: SnapshotDeps | null = null; @@ -1258,7 +1259,9 @@ async function handleVomObservation( {}, ); const axNodes = result.nodes ?? []; - const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe); + const effectiveConditionalSurfaceProbe = + deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; + const captured = await captureForVom(deps.cdp, target.tabId, effectiveConditionalSurfaceProbe); const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index a335747..c3ebdff 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -410,6 +410,25 @@ export interface ClickResult { dialogs?: JavaScriptDialogInfo[]; } +export interface HoverParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + modifiers?: KeyModifier[]; + settle_ms?: number; + timeout_ms?: number; +} + +export interface HoverResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; @@ -668,6 +687,11 @@ export type DraftTraceStep = navigated_to?: string; page_url?: string; } + | { + op: "hover"; + target: TargetDescriptor; + page_url?: string; + } | { op: "fill"; target: TargetDescriptor; @@ -701,6 +725,7 @@ export type DraftTraceStep = export type Step = | ({ op: "navigate" } & StepCommon & { to: string }) | ({ op: "click" } & StepCommon & { target: TargetDescriptor }) + | ({ op: "hover" } & StepCommon & { target: TargetDescriptor }) | ({ op: "fill" } & StepCommon & { target: TargetDescriptor; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 3c36961..9a1b6a8 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe ``` bsk navigate --session -bsk snapshot --session → aria tree with @e1, @e2, … refs -bsk observe --session → semantic VOM view; may reveal hover/focus surfaces -bsk click @e3 --session → or bsk fill, bsk select, bsk press -bsk snapshot --session → again after navigation / DOM change +bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces +bsk snapshot --session → static aria tree fallback when VOM is insufficient +bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking +bsk click @e4 --session → or bsk fill, bsk select, bsk press +bsk observe --session → again after navigation / DOM change ``` **Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. + ## Observation priority -Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: -1. `bsk snapshot` — strict static page understanding and interaction planning -2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery +1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery +2. `bsk snapshot` — strict static accessibility tree fallback 3. `bsk get-html` — when hidden DOM, metadata, or markup details are required 4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. @@ -186,6 +189,7 @@ bsk emulate --session --off | Command | Summary | |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | +| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |