docs(skill): give probe-hover a decidable trigger condition

The previous wording asked the agent to notice when a page "looks like"
it hides content behind hover. That is not something an agent reading a
text tree can judge: hover-only content is invisible by definition, and
the marker that used to point at it came from the probing that is now off
by default.

Replaces it with a condition the agent can actually evaluate — an
expected control is missing and no [has-submenu] marker points at a
trigger — and places it on the escalation ladder ahead of get-html and
screenshot, which is where that dead end is currently reached.

Also documents [has-submenu]/[expanded] as the primary hover signal. It
comes from aria-haspopup in the accessibility tree, costs nothing, and is
unaffected by probing being off, so it covers menus that are marked up
correctly.

apps/extension/src/session-manager/__tests__/ref-store.test.ts

@@ -13,8 +13,6 @@ describe("RefStore", () => {
       backendNodeId: 42,
       tabId: 7,
       generation: 0,
-      kind: "dom",
-      capabilities: ["interact", "screenshot"],
     });
     expect(s.size()).toBe(1);
     expect(s.isEmpty()).toBe(false);
@@ -50,31 +48,4 @@ describe("RefStore", () => {
       cdpSessionId: "child-session",
     });
   });
-
-  it("preserves explicit capabilities for visual surface refs", () => {
-    const s = new RefStore();
-    s.set("e1", 42, {
-      tabId: 7,
-      kind: "surface",
-      capabilities: ["screenshot"],
-    });
-
-    expect(s.resolveEntry("e1")).toMatchObject({
-      kind: "surface",
-      capabilities: ["screenshot"],
-    });
-  });
-
-  it("never grants ordinary interaction to a surface through defaults or caller input", () => {
-    const s = new RefStore();
-    s.set("e1", 42, { tabId: 7, kind: "surface" });
-    s.set("e2", 43, {
-      tabId: 7,
-      kind: "surface",
-      capabilities: ["interact", "screenshot"],
-    });
-
-    expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
-    expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
-  });
 });

apps/extension/src/session-manager/__tests__/surface-capture-store.test.ts

@@ -1,67 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
-
-function input(): SurfaceCaptureInput {
-  return {
-    sessionId: "aa11",
-    tabId: 7,
-    navigationIdentity: "navigation",
-    surface: {
-      ref: "e3",
-      frameId: "main",
-      backendNodeId: 99,
-      observationGeneration: 2,
-    },
-    topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
-    imageWidth: 200,
-    imageHeight: 100,
-    viewportSignature: "viewport",
-    frameProjectionSignature: "frame",
-  };
-}
-
-describe("SurfaceCaptureStore", () => {
-  it("creates metadata-only, short-lived captures and consumes them once", () => {
-    let now = 1_000;
-    const store = new SurfaceCaptureStore({
-      now: () => now,
-      ttlMs: 500,
-      createId: () => "sc_test",
-    });
-    const capture = store.create(input());
-
-    expect(capture).toMatchObject({
-      id: "sc_test",
-      createdAt: 1_000,
-      expiresAt: 1_500,
-      consumed: false,
-    });
-    expect(capture).not.toHaveProperty("imageBase64");
-    expect(store.consume("sc_test")).toMatchObject({ ok: true });
-    expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
-
-    now = 1_600;
-    expect(store.size()).toBe(0);
-  });
-
-  it("rejects expired and unknown captures", () => {
-    let now = 1_000;
-    const store = new SurfaceCaptureStore({
-      now: () => now,
-      ttlMs: 10,
-      createId: () => "sc_expired",
-    });
-    store.create(input());
-    now = 1_010;
-
-    expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
-    expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
-  });
-
-  it("clears every capture on session cleanup", () => {
-    const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
-    store.create(input());
-    store.clear();
-    expect(store.size()).toBe(0);
-  });
-});

apps/extension/src/session-manager/manager.ts

@@ -1,12 +1,10 @@
 import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
 import { RefStore } from "./ref-store";
-import { SurfaceCaptureStore } from "./surface-capture-store";
 
 export interface SessionContext {
   sessionId: string;
   agentWindowId: number;
   refStore: RefStore;
-  surfaceCaptures: SurfaceCaptureStore;
   borrowedTabs: Map<number, BorrowedTab>;
   createdAtMs: number;
 }
@@ -188,7 +186,6 @@ export class SessionManager {
         sessionId,
         agentWindowId: windowId,
         refStore: new RefStore(),
-        surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
         borrowedTabs: new Map(),
         createdAtMs: this.now(),
       };

apps/extension/src/session-manager/ref-store.ts

@@ -1,5 +1,3 @@
-import type { Rect } from "@browser-skill/vom";
-
 /**
  * Per-session map from `@e<N>` snapshot refs to a CDP node address.
  *
@@ -14,17 +12,12 @@ import type { Rect } from "@browser-skill/vom";
  * inside the `SessionContext`.
  */
 export type BackendNodeId = number;
-export type RefCapability = "interact" | "screenshot";
-export type RefTargetKind = "dom" | "surface";
 
 export interface RefEntry {
   backendNodeId: BackendNodeId;
   tabId: number | null;
   frameId?: string;
   cdpSessionId?: string;
-  visibleRect?: Rect;
-  kind: RefTargetKind;
-  capabilities: RefCapability[];
   generation: number;
 }
 
@@ -35,19 +28,8 @@ export type RefInput =
       tabId: number;
       frameId?: string;
       cdpSessionId?: string;
-      visibleRect?: Rect;
-      kind?: RefTargetKind;
-      capabilities?: RefCapability[];
     };
 
-function capabilitiesFor(
-  kind: RefTargetKind,
-  capabilities: RefCapability[] | undefined,
-): RefCapability[] {
-  if (kind === "surface") return ["screenshot"];
-  return capabilities ?? ["interact", "screenshot"];
-}
-
 export class RefStore {
   private readonly map = new Map<string, RefEntry>();
   private generation = 0;
@@ -88,20 +70,13 @@ export class RefStore {
       tabId?: number;
       frameId?: string;
       cdpSessionId?: string;
-      visibleRect?: Rect;
-      kind?: RefTargetKind;
-      capabilities?: RefCapability[];
     } = {},
   ): void {
-    const kind = opts.kind ?? "dom";
     this.map.set(normaliseRef(ref), {
       backendNodeId: id,
       tabId: opts.tabId ?? null,
       ...(opts.frameId ? { frameId: opts.frameId } : {}),
       ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
-      ...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
-      kind,
-      capabilities: capabilitiesFor(kind, opts.capabilities),
       generation: this.generation,
     });
   }
@@ -119,20 +94,14 @@ export class RefStore {
       return {
         backendNodeId: input,
         tabId: null,
-        kind: "dom",
-        capabilities: ["interact", "screenshot"],
         generation: this.generation,
       };
     }
-    const kind = input.kind ?? "dom";
     return {
       backendNodeId: input.backendNodeId,
       tabId: input.tabId,
       ...(input.frameId ? { frameId: input.frameId } : {}),
       ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
-      ...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
-      kind,
-      capabilities: capabilitiesFor(kind, input.capabilities),
       generation: this.generation,
     };
   }

apps/extension/src/session-manager/surface-capture-store.ts

@@ -1,100 +0,0 @@
-import type { Rect } from "@browser-skill/vom";
-
-export const SURFACE_CAPTURE_TTL_MS = 30_000;
-
-export interface SurfaceCapture {
-  id: string;
-  sessionId: string;
-  tabId: number;
-  navigationIdentity: string;
-  surface: {
-    ref: string;
-    frameId?: string;
-    backendNodeId: number;
-    observationGeneration: number;
-  };
-  topViewportRect: Rect;
-  imageWidth: number;
-  imageHeight: number;
-  viewportSignature: string;
-  frameProjectionSignature: string;
-  createdAt: number;
-  expiresAt: number;
-  consumed: boolean;
-}
-
-export type SurfaceCaptureInput = Omit<
-  SurfaceCapture,
-  "id" | "createdAt" | "expiresAt" | "consumed"
->;
-
-export type SurfaceCaptureConsumeResult =
-  | { ok: true; capture: SurfaceCapture }
-  | { ok: false; reason: "not_found" | "expired" | "consumed" };
-
-export interface SurfaceCaptureStoreOptions {
-  now?: () => number;
-  ttlMs?: number;
-  createId?: () => string;
-}
-
-function randomCaptureId(): string {
-  const bytes = crypto.getRandomValues(new Uint8Array(12));
-  return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
-}
-
-/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
-export class SurfaceCaptureStore {
-  private readonly captures = new Map<string, SurfaceCapture>();
-  private readonly now: () => number;
-  private readonly ttlMs: number;
-  private readonly createId: () => string;
-
-  constructor(options: SurfaceCaptureStoreOptions = {}) {
-    this.now = options.now ?? Date.now;
-    this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
-    this.createId = options.createId ?? randomCaptureId;
-  }
-
-  create(input: SurfaceCaptureInput): SurfaceCapture {
-    this.purgeExpired();
-    const createdAt = this.now();
-    const capture: SurfaceCapture = {
-      ...input,
-      id: this.createId(),
-      createdAt,
-      expiresAt: createdAt + this.ttlMs,
-      consumed: false,
-    };
-    this.captures.set(capture.id, capture);
-    return capture;
-  }
-
-  consume(id: string): SurfaceCaptureConsumeResult {
-    const capture = this.captures.get(id);
-    if (!capture) return { ok: false, reason: "not_found" };
-    if (capture.expiresAt <= this.now()) {
-      this.captures.delete(id);
-      return { ok: false, reason: "expired" };
-    }
-    if (capture.consumed) return { ok: false, reason: "consumed" };
-    capture.consumed = true;
-    return { ok: true, capture };
-  }
-
-  clear(): void {
-    this.captures.clear();
-  }
-
-  size(): number {
-    this.purgeExpired();
-    return this.captures.size;
-  }
-
-  private purgeExpired(): void {
-    const now = this.now();
-    for (const [id, capture] of this.captures) {
-      if (capture.expiresAt <= now) this.captures.delete(id);
-    }
-  }
-}

apps/extension/src/tools/__tests__/editable-input.test.ts

@@ -1,104 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import { fillEditable } from "../editable-input";
-import type { CdpRunner } from "../shared";
-
-interface FakeOptions {
-  initialValue?: string;
-  finalValue: string;
-  remount?: boolean;
-}
-
-function editableState(value: string, connected = true, focused = true) {
-  return {
-    result: {
-      value: { supported: true, connected, focused, value, kind: "input" },
-    },
-  };
-}
-
-function makeEditableCdp(options: FakeOptions) {
-  const sent: Array<{ method: string; params?: object }> = [];
-  let inserted = false;
-  const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
-    sent.push({ method, params });
-    if (method === "Input.insertText") {
-      inserted = true;
-      return {} as T;
-    }
-    if (method === "Input.dispatchKeyEvent") {
-      if ((params as { type?: string }).type === "rawKeyDown") inserted = true;
-      return {} as T;
-    }
-    if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`);
-
-    const call = params as { objectId?: string; functionDeclaration?: string };
-    const fn = call.functionDeclaration ?? "";
-    if (fn.includes("bsk-editable-selection")) {
-      return { result: { value: { selected: true } } } as T;
-    }
-    if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T;
-    if (fn.includes("bsk-editable-live")) {
-      return {
-        result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" },
-      } as T;
-    }
-    if (fn.includes("bsk-editable-state")) {
-      if (call.objectId === "replacement") return editableState(options.finalValue) as T;
-      if (options.remount && inserted) {
-        return editableState(options.finalValue, false, false) as T;
-      }
-      return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T;
-    }
-    throw new Error("unexpected Runtime.callFunctionOn function");
-  });
-  return { cdp: { send } as CdpRunner, sent };
-}
-
-describe("fillEditable", () => {
-  it("replaces text through the browser input path without synthetic events or value setters", async () => {
-    const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" });
-
-    const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true);
-
-    expect(result).toMatchObject({ value: "新值", replacedNode: false });
-    expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } });
-    const declarations = fake.sent
-      .map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "")
-      .join("\n");
-    expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/);
-  });
-
-  it("reacquires the focused editable after a framework remount", async () => {
-    const fake = makeEditableCdp({ finalValue: "persisted", remount: true });
-
-    const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true);
-
-    expect(result).toMatchObject({ value: "persisted", replacedNode: true });
-  });
-
-  it("reports a page rollback after the bounded settlement", async () => {
-    const fake = makeEditableCdp({ finalValue: "" });
-
-    const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true);
-
-    expect(result).toMatchObject({
-      code: "cdp_failed",
-      data: {
-        reason: "input_not_applied",
-        expected_value_length: 8,
-        actual_value_length: 0,
-        phase: "readback",
-      },
-    });
-  });
-
-  it("clears to an empty value with a browser Backspace edit", async () => {
-    const fake = makeEditableCdp({ initialValue: "old", finalValue: "" });
-
-    const result = await fillEditable(fake.cdp, 4, "anchor", "", true);
-
-    expect(result).toMatchObject({ value: "" });
-    expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
-    expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2);
-  });
-});

apps/extension/src/tools/__tests__/interaction.test.ts

@@ -7,12 +7,10 @@ import {
   handleHover,
   handlePress,
   handleSelect,
-  handleType,
   modifiersBitfield,
   parseKeySpec,
   resolveKeyDescriptor,
 } from "../interaction";
-import { captureSurfaceEnvironment } from "../surface-target";
 
 function fakeAgentWindow(ids: number[]) {
   let i = 0;
@@ -35,15 +33,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
     if (!h && method === "Page.getLayoutMetrics") {
       return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
     }
-    if (
-      !h &&
-      method === "Runtime.evaluate" &&
-      String((params as { expression?: string })?.expression ?? "").includes(
-        "bsk-focus-target-probe",
-      )
-    ) {
-      return { result: { value: { hasFocus: true, activeTag: "input" } } };
-    }
     if (!h) throw new Error(`unexpected CDP call ${method}`);
     return h(params);
   };
@@ -55,10 +44,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
       params?: object,
     ) => Promise<T>,
     trackSessionTab: vi.fn(),
-    getFrameGraph: vi.fn(async (tabId) => ({
-      rootFrameId: "main",
-      frames: [{ frameId: "main", target: { tabId } }],
-    })),
   };
   const tabsApi = {
     get: vi.fn(
@@ -69,84 +54,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
   return { cdp, tabsApi, sent };
 }
 
-function fillRuntimeReadback(value: string) {
-  return (params: unknown) => {
-    const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
-    if (fn.includes("bsk-editable-state")) {
-      return {
-        result: {
-          value: {
-            supported: true,
-            connected: true,
-            focused: true,
-            value,
-            kind: "input",
-          },
-        },
-      };
-    }
-    if (fn.includes("bsk-editable-selection")) {
-      return { result: { value: { selected: true } } };
-    }
-    return { result: { type: "undefined" } };
-  };
-}
-
-async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
-  const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-  const context = await manager.start("aa11");
-  context.refStore.set("e3", 99, {
-    tabId: 4,
-    kind: "surface",
-    visibleRect: { x: 10, y: 20, w: 100, h: 40 },
-  });
-  const fake = makeFakeCdp({
-    "Page.getFrameTree": () => ({
-      frameTree: {
-        frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
-      },
-    }),
-    "Page.getLayoutMetrics": () => ({
-      cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
-      cssVisualViewport: {
-        clientWidth: 800,
-        clientHeight: 600,
-        pageX: 0,
-        pageY: 0,
-        scale: 1,
-      },
-    }),
-    "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
-    "Runtime.evaluate": () => ({
-      result: { value: { overlayHostPresent: false, overlayHostConnected: false } },
-    }),
-    "Input.dispatchMouseEvent": (params) => {
-      onMouse?.(params as Record<string, unknown>);
-      return {};
-    },
-  });
-  const environment = await captureSurfaceEnvironment(fake.cdp, 4, undefined);
-  if ("code" in environment) throw new Error(environment.message);
-  const entry = context.refStore.resolveEntry("e3");
-  if (!entry) throw new Error("missing Surface ref");
-  const capture = context.surfaceCaptures.create({
-    sessionId: "aa11",
-    tabId: 4,
-    navigationIdentity: environment.navigationIdentity,
-    surface: {
-      ref: "e3",
-      backendNodeId: 99,
-      observationGeneration: entry.generation,
-    },
-    topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
-    imageWidth: 200,
-    imageHeight: 80,
-    viewportSignature: environment.viewportSignature,
-    frameProjectionSignature: environment.frameProjectionSignature,
-  });
-  return { manager, context, fake, capture };
-}
-
 describe("modifiersBitfield", () => {
   it("matches CDP's expected bit layout", () => {
     expect(modifiersBitfield([])).toBe(0);
@@ -216,29 +123,6 @@ describe("handleClick", () => {
     expect(fake.cdp.send).not.toHaveBeenCalled();
   });
 
-  it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e3", 1234, {
-      tabId: 4,
-      kind: "surface",
-      capabilities: ["screenshot"],
-    });
-    const fake = makeFakeCdp({});
-
-    const res = await handleClick(
-      sm,
-      { session_id: "aa11", ref: "@e3" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expect(res).toMatchObject({
-      code: "permission_denied",
-      data: { reason: "ref_capability_denied", required_capability: "interact" },
-    });
-    expect(fake.cdp.send).not.toHaveBeenCalled();
-  });
-
   it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
@@ -283,74 +167,6 @@ describe("handleClick", () => {
     });
   });
 
-  it.each([
-    "left",
-    "middle",
-    "right",
-  ] as const)("uses the common click executor for a %s-button Surface click", async (button) => {
-    const { manager, fake, capture } = await makeSurfaceClickFixture();
-    const result = await handleClick(
-      manager,
-      {
-        session_id: "aa11",
-        ref: "@e3",
-        capture_id: capture.id,
-        image_x: 50,
-        image_y: 20,
-        button,
-        click_count: 2,
-        modifiers: ["ctrl", "shift"],
-      },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    if ("code" in result) throw new Error(JSON.stringify(result));
-    expect(result).toMatchObject({
-      used_ref: "e3",
-      capture_id: capture.id,
-      image_x: 50,
-      image_y: 20,
-      x: 35,
-      y: 30,
-    });
-    const mouse = fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent");
-    expect(mouse).toHaveLength(3);
-    expect(mouse[1].params).toMatchObject({
-      type: "mousePressed",
-      button,
-      clickCount: 2,
-      modifiers: 2 | 8,
-    });
-    expect(mouse[2].params).toMatchObject({
-      type: "mouseReleased",
-      button,
-      clickCount: 2,
-    });
-  });
-
-  it("consumes a Surface capture when the common click executor fails", async () => {
-    let mouseCalls = 0;
-    const { manager, fake, capture } = await makeSurfaceClickFixture((params) => {
-      mouseCalls += 1;
-      if (params.type === "mousePressed") throw new Error("input failed");
-    });
-    const params = {
-      session_id: "aa11",
-      ref: "@e3",
-      capture_id: capture.id,
-      image_x: 50,
-      image_y: 20,
-    };
-
-    expect(
-      await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
-    ).toMatchObject({ code: "cdp_failed" });
-    expect(mouseCalls).toBe(2);
-    expect(
-      await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
-    ).toMatchObject({ data: { reason: "surface_capture_consumed" } });
-  });
-
   it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
@@ -770,7 +586,7 @@ describe("handleFill", () => {
       "DOM.scrollIntoViewIfNeeded": () => ({}),
       "DOM.focus": () => ({}),
       "DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
-      "Runtime.callFunctionOn": fillRuntimeReadback("hello"),
+      "Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
       "Input.insertText": () => ({}),
     });
     const res = await handleFill(
@@ -784,13 +600,13 @@ describe("handleFill", () => {
     expect(res.used_ref).toBe("e1");
     const insert = fake.sent.find((c) => c.method === "Input.insertText");
     expect(insert?.params).toEqual({ text: "hello" });
-    // clear_before defaults to true, so the edit transaction prepares a
+    // clear_before defaults to true → callFunctionOn invoked once to
-    // replacement selection and performs a settled live-value readback.
+    // clear, once to fire input/change after typing.
     const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
     expect(callFns.length).toBeGreaterThanOrEqual(2);
   });
 
-  it("clear_before=false skips replacement selection", async () => {
+  it("clear_before=false skips the wipe call", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
     ctx.refStore.set("e1", 1, { tabId: 4 });
@@ -803,20 +619,9 @@ describe("handleFill", () => {
       "DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
       "Runtime.callFunctionOn": (p) => {
         const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
-        if (fn.includes("bsk-editable-state")) {
+        // No "this.value = ''" clearing on a no-clear path; only the
-          return {
+        // post-input dispatchEvent.
-            result: {
+        expect(fn).not.toMatch(/this\.value\s*=\s*''/);
-              value: {
-                supported: true,
-                connected: true,
-                focused: true,
-                value: "existingx",
-                kind: "textarea",
-              },
-            },
-          };
-        }
-        expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/);
         return { result: { type: "undefined" } };
       },
       "Input.insertText": () => ({}),
@@ -827,7 +632,7 @@ describe("handleFill", () => {
       { cdp: fake.cdp, tabsApi: fake.tabsApi },
     );
     if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
-    expect(res.value_length).toBe(9);
+    expect(res.value_length).toBe(1);
   });
 
   it("treats contenteditable=true as fillable", async () => {
@@ -845,7 +650,7 @@ describe("handleFill", () => {
       "DOM.scrollIntoViewIfNeeded": () => ({}),
       "DOM.focus": () => ({}),
       "DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
-      "Runtime.callFunctionOn": fillRuntimeReadback("rich"),
+      "Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
       "Input.insertText": () => ({}),
     });
     const res = await handleFill(
@@ -856,103 +661,7 @@ describe("handleFill", () => {
     expect("code" in res).toBe(false);
   });
 
-  it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
+  it("stops before Input.insertText when abort fires after clearing", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e1", 77, {
-      tabId: 4,
-      frameId: "child-frame",
-      cdpSessionId: "child-session",
-    });
-    const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
-    fake.cdp.getFrameGraph = vi.fn(async () => ({
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        {
-          frameId: "child-frame",
-          parentFrameId: "main",
-          ownerBackendNodeId: 12,
-          target: { tabId: 4, sessionId: "child-session" },
-        },
-      ],
-    }));
-    const childCalls: string[] = [];
-    fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
-      expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
-      childCalls.push(method);
-      if (method === "DOM.describeNode") {
-        return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
-      }
-      if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
-      if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
-      if (method === "Input.insertText") {
-        expect(params).toEqual({ text: "中文输入" });
-        return {};
-      }
-      if (method === "Runtime.callFunctionOn") {
-        const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
-        if (fn.includes("bsk-editable-state")) {
-          return {
-            result: {
-              value: {
-                supported: true,
-                connected: true,
-                focused: true,
-                value: "中文输入",
-                kind: "input",
-              },
-            },
-          };
-        }
-        if (fn.includes("bsk-editable-selection")) {
-          return { result: { value: { selected: true } } };
-        }
-        return { result: { type: "undefined" } };
-      }
-      throw new Error(`unexpected child call ${method}`);
-    }) as CdpRunner["sendToTarget"];
-
-    const res = await handleFill(
-      sm,
-      { session_id: "aa11", ref: "@e1", value: "中文输入" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    if ("code" in res) throw new Error(JSON.stringify(res));
-    expect(res.value_length).toBe(4);
-    expect(childCalls).toContain("DOM.focus");
-    expect(childCalls).toContain("Input.insertText");
-    expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
-    expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
-  });
-
-  it("returns input_not_applied when a controlled input resets the value", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e1", 8, { tabId: 4 });
-    const fake = makeFakeCdp({
-      "DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
-      "DOM.scrollIntoViewIfNeeded": () => ({}),
-      "DOM.focus": () => ({}),
-      "DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
-      "Input.insertText": () => ({}),
-      "Runtime.callFunctionOn": fillRuntimeReadback(""),
-    });
-
-    const res = await handleFill(
-      sm,
-      { session_id: "aa11", ref: "@e1", value: "rejected" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expect(res).toMatchObject({
-      code: "cdp_failed",
-      data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
-    });
-  });
-
-  it("stops before Input.insertText when abort fires after preparing replacement", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
     ctx.refStore.set("e1", 555, { tabId: 4 });
@@ -964,25 +673,8 @@ describe("handleFill", () => {
       "DOM.scrollIntoViewIfNeeded": () => ({}),
       "DOM.focus": () => ({}),
       "DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
-      "Runtime.callFunctionOn": (params) => {
+      "Runtime.callFunctionOn": () => {
-        const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
+        abort.abort();
-        if (fn.includes("bsk-editable-state")) {
-          return {
-            result: {
-              value: {
-                supported: true,
-                connected: true,
-                focused: true,
-                value: "",
-                kind: "input",
-              },
-            },
-          };
-        }
-        if (fn.includes("bsk-editable-selection")) {
-          abort.abort();
-          return { result: { value: { selected: true } } };
-        }
         return { result: { type: "undefined" } };
       },
       "Input.insertText": () => ({}),
@@ -1025,204 +717,6 @@ describe("handleFill", () => {
   });
 });
 
-describe("handleType", () => {
-  it("requires an explicit target mode instead of silently using focus", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    await sm.start("aa11");
-    const fake = makeFakeCdp({});
-
-    const missing = await handleType(
-      sm,
-      { session_id: "aa11", text: "x" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-    expect(missing).toMatchObject({
-      code: "invalid_params",
-      message: expect.stringContaining("focused=true"),
-    });
-
-    const mixed = await handleType(
-      sm,
-      { session_id: "aa11", ref: "@e1", focused: true, text: "x" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-    expect(mixed).toMatchObject({
-      code: "invalid_params",
-      message: expect.stringContaining("not both"),
-    });
-    expect(fake.sent).toHaveLength(0);
-  });
-
-  it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e1", 77, {
-      tabId: 4,
-      frameId: "child-frame",
-      cdpSessionId: "child-session",
-    });
-    const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
-    fake.cdp.getFrameGraph = vi.fn(async () => ({
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        {
-          frameId: "child-frame",
-          parentFrameId: "main",
-          ownerBackendNodeId: 12,
-          target: { tabId: 4, sessionId: "child-session" },
-        },
-      ],
-    }));
-    const childCalls: Array<{ method: string; params?: object }> = [];
-    fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => {
-      childCalls.push({ method, params });
-      if (method === "DOM.describeNode") {
-        return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } };
-      }
-      return {};
-    }) as CdpRunner["sendToTarget"];
-
-    const result = await handleType(
-      sm,
-      { session_id: "aa11", ref: "@e1", text: "测试" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    if ("code" in result) throw new Error(JSON.stringify(result));
-    expect(result).toMatchObject({
-      tab_id: 4,
-      used_ref: "e1",
-      text_length: 2,
-      verification: "dispatched",
-    });
-    expect(childCalls.map((call) => call.method)).toEqual([
-      "DOM.describeNode",
-      "DOM.scrollIntoViewIfNeeded",
-      "DOM.focus",
-      "Input.dispatchKeyEvent",
-      "Input.insertText",
-      "Input.dispatchKeyEvent",
-    ]);
-    expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
-    expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
-  });
-
-  it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    await sm.start("aa11");
-    const fake = makeFakeCdp({
-      "Runtime.evaluate": () => ({
-        result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } },
-      }),
-    });
-    fake.cdp.getFrameGraph = vi.fn(async () => ({
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        {
-          frameId: "child-frame",
-          parentFrameId: "main",
-          target: { tabId: 4, sessionId: "child-session" },
-        },
-      ],
-    }));
-    const childCalls: string[] = [];
-    fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
-      childCalls.push(method);
-      if (method === "Runtime.evaluate") {
-        return {
-          result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } },
-        };
-      }
-      return {};
-    }) as CdpRunner["sendToTarget"];
-
-    const result = await handleType(
-      sm,
-      { session_id: "aa11", focused: true, text: "A" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    if ("code" in result) throw new Error(JSON.stringify(result));
-    expect(result).toMatchObject({ text_length: 1, verification: "dispatched" });
-    expect(childCalls).toEqual([
-      "Runtime.evaluate",
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-    ]);
-    expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
-  });
-
-  it("refuses focused mode when focus is not owned by a text-entry target", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    await sm.start("aa11");
-    const fake = makeFakeCdp({
-      "Runtime.evaluate": () => ({
-        result: { value: { hasFocus: true, activeTag: "button", textEntry: false } },
-      }),
-    });
-
-    const result = await handleType(
-      sm,
-      { session_id: "aa11", focused: true, text: "x" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expect(result).toMatchObject({
-      code: "cdp_failed",
-      data: { reason: "focus_target_unresolved", candidate_count: 0 },
-    });
-    expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
-  });
-
-  it("rejects explicit non-text input controls", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e1", 77, { tabId: 4 });
-    const fake = makeFakeCdp({
-      "DOM.describeNode": () => ({
-        node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] },
-      }),
-    });
-
-    const result = await handleType(
-      sm,
-      { session_id: "aa11", ref: "@e1", text: "x" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expect(result).toMatchObject({
-      code: "invalid_params",
-      data: { reason: "target_not_fillable" },
-    });
-    expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
-  });
-
-  it("rejects a Surface ref instead of treating it as a text target", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e1", 77, {
-      tabId: 4,
-      kind: "surface",
-      capabilities: ["screenshot"],
-    });
-    const fake = makeFakeCdp({});
-
-    const result = await handleType(
-      sm,
-      { session_id: "aa11", ref: "@e1", text: "x" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expect(result).toMatchObject({
-      code: "permission_denied",
-      data: { reason: "ref_capability_denied", required_capability: "interact" },
-    });
-  });
-});
-
 describe("parseKeySpec", () => {
   it("splits compound expressions into modifiers + base key", () => {
     expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
@@ -1361,125 +855,6 @@ describe("handlePress", () => {
     expect(res.key).toBe("Enter");
   });
 
-  it("routes ref-targeted key events through the OOPIF session", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e1", 555, {
-      tabId: 4,
-      frameId: "child-frame",
-      cdpSessionId: "child-session",
-    });
-    const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
-    fake.cdp.getFrameGraph = vi.fn(async () => ({
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        {
-          frameId: "child-frame",
-          parentFrameId: "main",
-          ownerBackendNodeId: 12,
-          target: { tabId: 4, sessionId: "child-session" },
-        },
-      ],
-    }));
-    const childCalls: string[] = [];
-    fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
-      childCalls.push(method);
-      return {};
-    }) as CdpRunner["sendToTarget"];
-
-    const res = await handlePress(
-      sm,
-      { session_id: "aa11", key: "Enter", ref: "@e1" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expectPressOk(res);
-    expect(childCalls).toEqual([
-      "DOM.scrollIntoViewIfNeeded",
-      "DOM.focus",
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-    ]);
-    expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
-  });
-
-  it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    await sm.start("aa11");
-    const fake = makeFakeCdp({
-      "Runtime.evaluate": () => ({
-        result: { value: { hasFocus: true, activeTag: "iframe" } },
-      }),
-    });
-    fake.cdp.getFrameGraph = vi.fn(async () => ({
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        {
-          frameId: "child-frame",
-          parentFrameId: "main",
-          target: { tabId: 4, sessionId: "child-session" },
-        },
-      ],
-    }));
-    const childCalls: string[] = [];
-    fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
-      childCalls.push(method);
-      if (method === "Runtime.evaluate") {
-        return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
-      }
-      return {};
-    }) as CdpRunner["sendToTarget"];
-
-    const res = await handlePress(
-      sm,
-      { session_id: "aa11", key: "a" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expectPressOk(res);
-    expect(childCalls).toEqual([
-      "Runtime.evaluate",
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-    ]);
-  });
-
-  it("refuses an untargeted press when focus ownership is ambiguous", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    await sm.start("aa11");
-    const fake = makeFakeCdp({
-      "Runtime.evaluate": () => ({
-        result: { value: { hasFocus: true, activeTag: "input" } },
-      }),
-    });
-    fake.cdp.getFrameGraph = vi.fn(async () => ({
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        { frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
-      ],
-    }));
-    fake.cdp.sendToTarget = vi.fn(async () => ({
-      result: { value: { hasFocus: true, activeTag: "textarea" } },
-    })) as CdpRunner["sendToTarget"];
-
-    const res = await handlePress(
-      sm,
-      { session_id: "aa11", key: "Enter" },
-      { cdp: fake.cdp, tabsApi: fake.tabsApi },
-    );
-
-    expect(res).toMatchObject({
-      code: "cdp_failed",
-      data: { reason: "focus_target_unresolved", candidate_count: 2 },
-    });
-    expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
-  });
-
   it("stops before focus when abort fires after press scroll", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");

apps/extension/src/tools/__tests__/mouse-input.test.ts

@@ -1,119 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import { dispatchMouseClick } from "../mouse-input";
-import type { CdpRunner } from "../shared";
-
-function fakeCdp(onSend?: (method: string, params: Record<string, unknown>) => void): {
-  cdp: CdpRunner;
-  events: Record<string, unknown>[];
-} {
-  const events: Record<string, unknown>[] = [];
-  return {
-    events,
-    cdp: {
-      send: vi.fn(async (_tabId, method, params) => {
-        const event = params as Record<string, unknown>;
-        events.push(event);
-        onSend?.(method, event);
-        return {} as never;
-      }),
-    },
-  };
-}
-
-describe("dispatchMouseClick", () => {
-  it("emits one move, press, and release sequence", async () => {
-    const { cdp, events } = fakeCdp();
-
-    expect(
-      await dispatchMouseClick(
-        cdp,
-        4,
-        { x: 12, y: 34 },
-        {
-          button: "left",
-          clickCount: 1,
-          modifiers: 0,
-          moveSettleMs: 0,
-        },
-      ),
-    ).toBe("completed");
-
-    expect(events).toEqual([
-      {
-        type: "mouseMoved",
-        x: 12,
-        y: 34,
-        modifiers: 0,
-      },
-      {
-        type: "mousePressed",
-        x: 12,
-        y: 34,
-        button: "left",
-        clickCount: 1,
-        modifiers: 0,
-      },
-      {
-        type: "mouseReleased",
-        x: 12,
-        y: 34,
-        button: "left",
-        clickCount: 1,
-        modifiers: 0,
-      },
-    ]);
-  });
-
-  it("releases a pressed button when cancellation arrives", async () => {
-    const controller = new AbortController();
-    const { cdp, events } = fakeCdp((_method, event) => {
-      if (event.type === "mousePressed") controller.abort();
-    });
-
-    expect(
-      await dispatchMouseClick(
-        cdp,
-        4,
-        { x: 1, y: 2 },
-        {
-          button: "left",
-          clickCount: 1,
-          modifiers: 0,
-          signal: controller.signal,
-          moveSettleMs: 0,
-        },
-      ),
-    ).toBe("cancelled");
-    expect(events.at(-1)).toMatchObject({ type: "mouseReleased" });
-  });
-
-  it("settles pointer-move hit testing before pressing", async () => {
-    vi.useFakeTimers();
-    try {
-      const { cdp, events } = fakeCdp();
-      const click = dispatchMouseClick(
-        cdp,
-        4,
-        { x: 1, y: 2 },
-        {
-          button: "left",
-          clickCount: 1,
-          modifiers: 0,
-          moveSettleMs: 32,
-        },
-      );
-
-      await vi.advanceTimersByTimeAsync(31);
-      expect(events.map((event) => event.type)).toEqual(["mouseMoved"]);
-      await vi.advanceTimersByTimeAsync(1);
-      await click;
-      expect(events.map((event) => event.type)).toEqual([
-        "mouseMoved",
-        "mousePressed",
-        "mouseReleased",
-      ]);
-    } finally {
-      vi.useRealTimers();
-    }
-  });
-});

apps/extension/src/tools/__tests__/observation.test.ts

@@ -67,9 +67,6 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
     if (!handler && method === "Page.getLayoutMetrics") {
       return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
     }
-    if (!handler && method === "Page.getFrameTree") {
-      return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } };
-    }
     if (!handler) throw new Error(`unexpected CDP call ${method}`);
     return handler(params);
   });
@@ -303,82 +300,6 @@ describe("handleScreenshot", () => {
     expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
   });
 
-  it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e5", 999, {
-      tabId: 7,
-      kind: "surface",
-      capabilities: ["screenshot"],
-    });
-    const { cdp, sent } = makeFakeCdp({
-      "Page.getLayoutMetrics": () => ({
-        cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
-      }),
-      "DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
-      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
-    });
-
-    const res = await handleScreenshot(
-      sm,
-      { session_id: "aa11", ref: "@e5", tab_id: 7 },
-      makeScreenshotDeps({
-        cdp,
-        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
-        query: vi.fn(),
-        captureVisibleTab: vi.fn(),
-      }),
-    );
-
-    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
-    expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
-    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
-      clip?: { width: number; height: number; scale: number };
-    };
-    expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
-    expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
-    expect(res.capture).toMatchObject({
-      surface_ref: "@e5",
-      coordinate_space: "capture-image-pixel",
-    });
-    expect(ctx.surfaceCaptures.size()).toBe(1);
-  });
-
-  it("crops a visual surface screenshot to its observation-time visible region", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e5", 999, {
-      tabId: 7,
-      kind: "surface",
-      visibleRect: { x: 25, y: 30, w: 50, h: 40 },
-    });
-    const { cdp, sent } = makeFakeCdp({
-      "Page.getLayoutMetrics": () => ({
-        cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
-      }),
-      "DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
-      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
-    });
-
-    const res = await handleScreenshot(
-      sm,
-      { session_id: "aa11", ref: "@e5", tab_id: 7 },
-      makeScreenshotDeps({
-        cdp,
-        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
-        query: vi.fn(),
-        captureVisibleTab: vi.fn(),
-      }),
-    );
-
-    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
-    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
-      clip?: { x: number; y: number; width: number; height: number };
-    };
-    expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
-    expect(res.capture?.id).toMatch(/^sc_/);
-  });
-
   it("returns not_found for unknown ref", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");
@@ -1983,36 +1904,41 @@ describe("buildVomScene", () => {
         backendDOMNodeId: 20,
       },
     ];
-    const scene = buildVomScene(axNodes, {
+    const scene = buildVomScene(
-      viewport: { width: 1000, height: 800 },
+      axNodes,
-      iframeNodes: new Map(),
+      {
-      excludedBackendNodeIds: new Set(),
+        viewport: { width: 1000, height: 800 },
-      surfaceProbes: [
+        iframeNodes: new Map(),
-        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
+        excludedBackendNodeIds: new Set(),
-      ],
+        nodes: [
-      nodes: [
+          {
-        {
+            backendNodeId: 10,
-          backendNodeId: 10,
+            parentBackendNodeId: null,
-          parentBackendNodeId: null,
+            tag: "body",
-          tag: "body",
+            attrs: {},
-          attrs: {},
+            rect: { x: 0, y: 0, w: 1000, h: 800 },
-          rect: { x: 0, y: 0, w: 1000, h: 800 },
+            paintOrder: 0,
-          paintOrder: 0,
+            position: "static",
-          position: "static",
+            pointerEvents: "auto",
-          pointerEvents: "auto",
+          },
-        },
+          {
-        {
+            backendNodeId: 20,
-          backendNodeId: 20,
+            parentBackendNodeId: 10,
-          parentBackendNodeId: 10,
+            tag: "button",
-          tag: "button",
+            attrs: {},
-          attrs: {},
+            rect: { x: 20, y: 20, w: 120, h: 40 },
-          rect: { x: 20, y: 20, w: 120, h: 40 },
+            paintOrder: 1,
-          paintOrder: 1,
+            position: "static",
-          position: "static",
+            pointerEvents: "auto",
-          pointerEvents: "auto",
+          },
-        },
+        ],
-      ],
+      },
-    });
+      {
+        surfaceProbes: [
+          { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
+        ],
+      },
+    );
 
     expect(scene.surfaces).toEqual([
       { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
@@ -2036,46 +1962,51 @@ describe("buildVomScene", () => {
         backendDOMNodeId: 21,
       },
     ];
-    const scene = buildVomScene(axNodes, {
+    const scene = buildVomScene(
-      viewport: { width: 1000, height: 800 },
+      axNodes,
-      iframeNodes: new Map(),
+      {
-      excludedBackendNodeIds: new Set(),
+        viewport: { width: 1000, height: 800 },
-      surfaceProbes: [
+        iframeNodes: new Map(),
-        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
+        excludedBackendNodeIds: new Set(),
-      ],
+        nodes: [
-      nodes: [
+          {
-        {
+            backendNodeId: 10,
-          backendNodeId: 10,
+            parentBackendNodeId: null,
-          parentBackendNodeId: null,
+            tag: "body",
-          tag: "body",
+            attrs: {},
-          attrs: {},
+            rect: { x: 0, y: 0, w: 1000, h: 800 },
-          rect: { x: 0, y: 0, w: 1000, h: 800 },
+            paintOrder: 0,
-          paintOrder: 0,
+            position: "static",
-          position: "static",
+            pointerEvents: "auto",
-          pointerEvents: "auto",
+          },
-        },
+          {
-        {
+            backendNodeId: 20,
-          backendNodeId: 20,
+            parentBackendNodeId: 10,
-          parentBackendNodeId: 10,
+            tag: "div",
-          tag: "div",
+            attrs: { class: "tg-avatar" },
-          attrs: { class: "tg-avatar" },
+            rect: { x: 900, y: 10, w: 30, h: 30 },
-          rect: { x: 900, y: 10, w: 30, h: 30 },
+            paintOrder: 1,
-          paintOrder: 1,
+            position: "static",
-          position: "static",
+            pointerEvents: "auto",
-          pointerEvents: "auto",
+          },
-        },
+          {
-        {
+            backendNodeId: 21,
-          backendNodeId: 21,
+            parentBackendNodeId: 20,
-          parentBackendNodeId: 20,
+            tag: "div",
-          tag: "div",
+            attrs: { class: "tg-avatar__inner" },
-          attrs: { class: "tg-avatar__inner" },
+            rect: { x: 902, y: 12, w: 26, h: 26 },
-          rect: { x: 902, y: 12, w: 26, h: 26 },
+            paintOrder: 2,
-          paintOrder: 2,
+            position: "static",
-          position: "static",
+            pointerEvents: "auto",
-          pointerEvents: "auto",
+          },
-        },
+        ],
-      ],
+      },
-    });
+      {
+        surfaceProbes: [
+          { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
+        ],
+      },
+    );
 
     expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
   });
@@ -2096,48 +2027,53 @@ describe("buildVomScene", () => {
         backendDOMNodeId: 21,
       },
     ];
-    const scene = buildVomScene(axNodes, {
+    const scene = buildVomScene(
-      viewport: { width: 1000, height: 800 },
+      axNodes,
-      iframeNodes: new Map(),
+      {
-      excludedBackendNodeIds: new Set(),
+        viewport: { width: 1000, height: 800 },
-      surfaceProbes: [
+        iframeNodes: new Map(),
-        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
+        excludedBackendNodeIds: new Set(),
-        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
+        nodes: [
-        { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
+          {
-      ],
+            backendNodeId: 10,
-      nodes: [
+            parentBackendNodeId: null,
-        {
+            tag: "body",
-          backendNodeId: 10,
+            attrs: {},
-          parentBackendNodeId: null,
+            rect: { x: 0, y: 0, w: 1000, h: 800 },
-          tag: "body",
+            paintOrder: 0,
-          attrs: {},
+            position: "static",
-          rect: { x: 0, y: 0, w: 1000, h: 800 },
+            pointerEvents: "auto",
-          paintOrder: 0,
+          },
-          position: "static",
+          {
-          pointerEvents: "auto",
+            backendNodeId: 20,
-        },
+            parentBackendNodeId: 10,
-        {
+            tag: "div",
-          backendNodeId: 20,
+            attrs: { class: "tg-avatar" },
-          parentBackendNodeId: 10,
+            rect: { x: 900, y: 10, w: 30, h: 30 },
-          tag: "div",
+            paintOrder: 1,
-          attrs: { class: "tg-avatar" },
+            position: "static",
-          rect: { x: 900, y: 10, w: 30, h: 30 },
+            pointerEvents: "auto",
-          paintOrder: 1,
+          },
-          position: "static",
+          {
-          pointerEvents: "auto",
+            backendNodeId: 21,
-        },
+            parentBackendNodeId: 20,
-        {
+            tag: "div",
-          backendNodeId: 21,
+            attrs: { class: "tg-avatar__inner" },
-          parentBackendNodeId: 20,
+            rect: { x: 902, y: 12, w: 26, h: 26 },
-          tag: "div",
+            paintOrder: 2,
-          attrs: { class: "tg-avatar__inner" },
+            position: "static",
-          rect: { x: 902, y: 12, w: 26, h: 26 },
+            pointerEvents: "auto",
-          paintOrder: 2,
+          },
-          position: "static",
+        ],
-          pointerEvents: "auto",
+      },
-        },
+      {
-      ],
+        surfaceProbes: [
-    });
+          { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
+          { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
+          { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
+        ],
+      },
+    );
 
     expect(scene.surfaces).toEqual([
       { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
@@ -2160,54 +2096,59 @@ describe("buildVomScene", () => {
         backendDOMNodeId: 21,
       },
     ];
-    const scene = buildVomScene(axNodes, {
+    const scene = buildVomScene(
-      viewport: { width: 1000, height: 800 },
+      axNodes,
-      iframeNodes: new Map(),
+      {
-      excludedBackendNodeIds: new Set(),
+        viewport: { width: 1000, height: 800 },
-      surfaceProbes: [
+        iframeNodes: new Map(),
-        {
+        excludedBackendNodeIds: new Set(),
-          triggerBackendNodeId: 20,
+        nodes: [
-          triggerPoint: { x: 915, y: 25 },
+          {
-          triggerAction: "hover",
+            backendNodeId: 10,
-          subItems: ["My profile", "Sign out"],
+            parentBackendNodeId: null,
-        },
+            tag: "body",
-      ],
+            attrs: {},
-      nodes: [
+            rect: { x: 0, y: 0, w: 1000, h: 800 },
-        {
+            paintOrder: 0,
-          backendNodeId: 10,
+            position: "static",
-          parentBackendNodeId: null,
+            pointerEvents: "auto",
-          tag: "body",
+            cursor: "auto",
-          attrs: {},
+          },
-          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          {
-          paintOrder: 0,
+            backendNodeId: 20,
-          position: "static",
+            parentBackendNodeId: 10,
-          pointerEvents: "auto",
+            tag: "div",
-          cursor: "auto",
+            attrs: { class: "tg-avatar" },
-        },
+            rect: { x: 900, y: 10, w: 30, h: 30 },
-        {
+            paintOrder: 1,
-          backendNodeId: 20,
+            position: "static",
-          parentBackendNodeId: 10,
+            pointerEvents: "auto",
-          tag: "div",
+            cursor: "pointer",
-          attrs: { class: "tg-avatar" },
+          },
-          rect: { x: 900, y: 10, w: 30, h: 30 },
+          {
-          paintOrder: 1,
+            backendNodeId: 21,
-          position: "static",
+            parentBackendNodeId: 20,
-          pointerEvents: "auto",
+            tag: "div",
-          cursor: "pointer",
+            attrs: { class: "tg-avatar__inner" },
-        },
+            rect: { x: 902, y: 12, w: 26, h: 26 },
-        {
+            paintOrder: 2,
-          backendNodeId: 21,
+            position: "static",
-          parentBackendNodeId: 20,
+            pointerEvents: "auto",
-          tag: "div",
+            cursor: "pointer",
-          attrs: { class: "tg-avatar__inner" },
+          },
-          rect: { x: 902, y: 12, w: 26, h: 26 },
+        ],
-          paintOrder: 2,
+      },
-          position: "static",
+      {
-          pointerEvents: "auto",
+        surfaceProbes: [
-          cursor: "pointer",
+          {
-        },
+            triggerBackendNodeId: 20,
-      ],
+            triggerPoint: { x: 915, y: 25 },
-    });
+            triggerAction: "hover",
+            subItems: ["My profile", "Sign out"],
+          },
+        ],
+      },
+    );
 
     expect(scene.surfaces).toEqual([
       { triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
@@ -2233,41 +2174,46 @@ describe("buildVomScene", () => {
         backendDOMNodeId: 42,
       },
     ];
-    const scene = buildVomScene(axNodes, {
+    const scene = buildVomScene(
-      viewport: { width: 1000, height: 800 },
+      axNodes,
-      iframeNodes: new Map(),
+      {
-      excludedBackendNodeIds: new Set(),
+        viewport: { width: 1000, height: 800 },
-      surfaceProbes: [
+        iframeNodes: new Map(),
-        {
+        excludedBackendNodeIds: new Set(),
-          triggerBackendNodeId: 999,
+        nodes: [
-          triggerPoint: { x: 900, y: 20 },
+          {
-          triggerAction: "hover",
+            backendNodeId: 10,
-          subItems: ["My profile"],
+            parentBackendNodeId: null,
-        },
+            tag: "body",
-      ],
+            attrs: {},
-      nodes: [
+            rect: { x: 0, y: 0, w: 1000, h: 800 },
-        {
+            paintOrder: 0,
-          backendNodeId: 10,
+            position: "static",
-          parentBackendNodeId: null,
+            pointerEvents: "auto",
-          tag: "body",
+          },
-          attrs: {},
+          {
-          rect: { x: 0, y: 0, w: 1000, h: 800 },
+            backendNodeId: 42,
-          paintOrder: 0,
+            parentBackendNodeId: 10,
-          position: "static",
+            tag: "a",
-          pointerEvents: "auto",
+            attrs: {},
-        },
+            rect: { x: 860, y: 120, w: 40, h: 30 },
-        {
+            paintOrder: 1,
-          backendNodeId: 42,
+            position: "static",
-          parentBackendNodeId: 10,
+            pointerEvents: "auto",
-          tag: "a",
+          },
-          attrs: {},
+        ],
-          rect: { x: 860, y: 120, w: 40, h: 30 },
+      },
-          paintOrder: 1,
+      {
-          position: "static",
+        surfaceProbes: [
-          pointerEvents: "auto",
+          {
-        },
+            triggerBackendNodeId: 999,
-      ],
+            triggerPoint: { x: 900, y: 20 },
-    });
+            triggerAction: "hover",
+            subItems: ["My profile"],
+          },
+        ],
+      },
+    );
 
     expect(scene.surfaces).toBeUndefined();
     expect(renderVom(scene).text).not.toContain("[hover first:");
@@ -2685,7 +2631,106 @@ describe("handleSnapshot", () => {
     );
   });
 
-  it("enables conditional surface probing for semantic observe", async () => {
+  it("hovers the page only after the accessibility tree has been captured", async () => {
+    // Hovering can open menus and reflow the page. Probing between the DOM and
+    // AX captures would leave the two halves of one observation describing the
+    // page on either side of that change.
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Example" },
+      backendDOMNodeId: 100,
+      childIds: ["2"],
+    };
+    const button: CdpAxNode = {
+      nodeId: "2",
+      parentId: "1",
+      role: { type: "role", value: "button" },
+      name: { type: "computedString", value: "Products" },
+      backendDOMNodeId: 200,
+    };
+    const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
+    const i = (s: string) => strings.indexOf(s);
+    const methodOrder: string[] = [];
+    const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
+      methodOrder.push(method);
+      if (method === "Accessibility.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
+      }
+      if (method === "DOMSnapshot.enable") return {};
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0],
+                nodeName: [i("body"), i("button")],
+                backendNodeId: [100, 200],
+                attributes: [[], []],
+              },
+              layout: {
+                nodeIndex: [0, 1],
+                styles: [
+                  [i("static"), i("auto"), i("auto")],
+                  [i("static"), i("auto"), i("auto")],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [20, 20, 120, 40],
+                ],
+                paintOrders: [0, 1],
+              },
+            },
+          ],
+        };
+      }
+      if (method === "Input.dispatchMouseEvent") return {};
+      if (method === "Runtime.evaluate") {
+        const expression = (params as { expression?: string } | undefined)?.expression ?? "";
+        // Report the button as a `:hover` rule target so a real hover fires.
+        if (expression.includes("document.styleSheets")) {
+          return { result: { value: [{ x: 80, y: 40 }] } };
+        }
+        return { result: { value: [] } };
+      }
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+
+    const res = await handleObserve(
+      sm,
+      { session_id: "aa11", probe_hover: true },
+      {
+        cdp: {
+          send: send as unknown as <T = unknown>(
+            tabId: number,
+            method: string,
+            params?: object,
+          ) => Promise<T>,
+          trackSessionTab: vi.fn(),
+        },
+        tabsApi: {
+          get: vi.fn(
+            async (tabId: number) =>
+              ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+          ),
+          query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+        },
+      },
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
+    expect(firstHover).toBeGreaterThan(-1);
+    expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
+    expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
+  });
+
+  it("runs conditional surface probing only when observe opts in", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");
     const root: CdpAxNode = {
@@ -2741,83 +2786,6 @@ describe("handleSnapshot", () => {
       if (method === "Runtime.evaluate") return { result: { value: [] } };
       throw new Error(`unexpected CDP method ${method}`);
     });
-    const res = await handleObserve(
-      sm,
-      { session_id: "aa11", debug_surfaces: true },
-      {
-        cdp: {
-          send: send as unknown as <T = unknown>(
-            tabId: number,
-            method: string,
-            params?: object,
-          ) => Promise<T>,
-          trackSessionTab: vi.fn(),
-        },
-        tabsApi: {
-          get: vi.fn(
-            async (tabId: number) =>
-              ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
-          ),
-          query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
-        },
-      },
-    );
-
-    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
-    expect(res.debug).toEqual({ surface_probes: [] });
-    expect(send).toHaveBeenCalledWith(
-      4,
-      "Runtime.evaluate",
-      expect.objectContaining({ returnByValue: true }),
-    );
-  });
-
-  it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    const root: CdpAxNode = {
-      nodeId: "1",
-      role: { type: "role", value: "RootWebArea" },
-      name: { type: "computedString", value: "Canvas app" },
-      backendDOMNodeId: 100,
-    };
-    const strings = ["body", "canvas", "static", "auto"];
-    const i = (value: string) => strings.indexOf(value);
-    const send = vi.fn(async (_tabId: number, method: string) => {
-      if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
-      if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
-      if (method === "Page.getLayoutMetrics") {
-        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
-      }
-      if (method === "DOMSnapshot.captureSnapshot") {
-        return {
-          strings,
-          documents: [
-            {
-              nodes: {
-                parentIndex: [-1, 0],
-                nodeName: [i("body"), i("canvas")],
-                backendNodeId: [100, 200],
-                attributes: [[], []],
-              },
-              layout: {
-                nodeIndex: [0, 1],
-                styles: [
-                  [i("static"), i("auto"), i("auto")],
-                  [i("static"), i("auto"), i("auto")],
-                ],
-                bounds: [
-                  [0, 0, 1000, 800],
-                  [20, 30, 800, 500],
-                ],
-                paintOrders: [0, 1],
-              },
-            },
-          ],
-        };
-      }
-      throw new Error(`unexpected CDP method ${method}`);
-    });
     const deps = {
       cdp: {
         send: send as unknown as <T = unknown>(
@@ -2833,22 +2801,36 @@ describe("handleSnapshot", () => {
         ),
         query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
       },
-      conditionalSurfaceProbe: false,
     };
 
-    const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
+    const withoutProbe = await handleObserve(
-    if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
+      sm,
-    expect(observed.text).toContain('@e1 surface "canvas visual surface"');
+      { session_id: "aa11", debug_surfaces: true },
-    expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
+      deps,
-      backendNodeId: 200,
+    );
-      kind: "surface",
+    if ("code" in withoutProbe) {
-      capabilities: ["screenshot"],
+      throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
-    });
+    }
+    expect(withoutProbe.hover_probe).toBeUndefined();
+    expect(send).not.toHaveBeenCalledWith(
+      4,
+      "Runtime.evaluate",
+      expect.objectContaining({ returnByValue: true }),
+    );
 
-    const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
+    const withProbe = await handleObserve(
-    if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
+      sm,
-    expect(snapshot.text).not.toContain("visual surface");
+      { session_id: "aa11", debug_surfaces: true, probe_hover: true },
-    expect(snapshot.ref_count).toBe(0);
+      deps,
+    );
+    if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
+    expect(withProbe.debug).toEqual({ surface_probes: [] });
+    expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
+    expect(send).toHaveBeenCalledWith(
+      4,
+      "Runtime.evaluate",
+      expect.objectContaining({ returnByValue: true }),
+    );
   });
 
   it("allows passive snapshots of explicit user-window tabs", async () => {

apps/extension/src/tools/__tests__/session.test.ts

@@ -252,23 +252,12 @@ describe("handleSessionStop with auto-return", () => {
     expect(aw.remove).not.toHaveBeenCalled();
   });
 
-  it("clears refs and Surface captures before window teardown", async () => {
+  it("clears the RefStore before window teardown", async () => {
     const aw = fakeAgentWindow([100]);
     const sm = new SessionManager({ agentWindow: aw });
     const ctx = await sm.start("aa11");
     // Insert a fake ref so we can verify clear() ran.
     ctx.refStore.set("e1", 123, { tabId: 7 });
-    ctx.surfaceCaptures.create({
-      sessionId: "aa11",
-      tabId: 7,
-      navigationIdentity: "navigation",
-      surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 },
-      topViewportRect: { x: 0, y: 0, w: 100, h: 50 },
-      imageWidth: 100,
-      imageHeight: 50,
-      viewportSignature: "viewport",
-      frameProjectionSignature: "frame",
-    });
     const state: FakeState = {
       tabs: new Map(),
       windowsClosed: new Set(),
@@ -277,6 +266,5 @@ describe("handleSessionStop with auto-return", () => {
     const { tabs, windows } = makeApis(state);
     await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
     expect(ctx.refStore.isEmpty()).toBe(true);
-    expect(ctx.surfaceCaptures.size()).toBe(0);
   });
 });

apps/extension/src/tools/__tests__/snapshot-ref.test.ts

@@ -24,16 +24,10 @@ describe("lookupSnapshotRef", () => {
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
-      kind: "dom",
-      capabilities: ["interact", "screenshot"],
-      generation: 0,
     });
     expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
-      kind: "dom",
-      capabilities: ["interact", "screenshot"],
-      generation: 0,
     });
   });
 
@@ -59,8 +53,6 @@ describe("resolveSnapshotRef", () => {
       tabId: 4,
       frameId: "child-frame",
       cdpSessionId: "child-session",
-      kind: "dom",
-      capabilities: ["interact", "screenshot"],
     });
 
     const expected = {
@@ -68,9 +60,6 @@ describe("resolveSnapshotRef", () => {
       refKey: "e3",
       frameId: "child-frame",
       cdpSessionId: "child-session",
-      kind: "dom" as const,
-      capabilities: ["interact", "screenshot"] as const,
-      generation: 0,
     };
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -116,29 +105,6 @@ describe("resolveSnapshotRef", () => {
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
-      kind: "dom",
-      capabilities: ["interact", "screenshot"],
-      generation: 0,
-    });
-  });
-
-  it("rejects an operation outside the ref's declared capabilities", async () => {
-    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
-    const ctx = await sm.start("aa11");
-    ctx.refStore.set("e3", 1234, {
-      tabId: 4,
-      kind: "surface",
-      capabilities: ["screenshot"],
-    });
-
-    expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
-      code: "permission_denied",
-      message: "ref @e3 does not support interact",
-      data: {
-        reason: "ref_capability_denied",
-        required_capability: "interact",
-        kind: "surface",
-      },
     });
   });
 });

apps/extension/src/tools/__tests__/surface-coordinate.test.ts

@@ -1,52 +0,0 @@
-import { describe, expect, it } from "vitest";
-import {
-  mapImagePointToViewport,
-  pointInRegion,
-  sameRect,
-  surfaceVisibleRect,
-} from "../surface-coordinate";
-
-describe("surface coordinate mapping", () => {
-  it("maps capture image pixels through the actual screenshot dimensions", () => {
-    expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({
-      x: 200,
-      y: 100,
-    });
-  });
-
-  it("rejects invalid image coordinates and dimensions", () => {
-    const rect = { x: 0, y: 0, w: 100, h: 50 };
-    expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull();
-    expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull();
-    expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull();
-    expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull();
-    expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull();
-    expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull();
-  });
-
-  it("intersects live and observed visible regions and compares them strictly", () => {
-    const clipped = surfaceVisibleRect(
-      { x: 0, y: 0, width: 200, height: 100 },
-      { x: 25, y: 30, w: 50, h: 40 },
-    );
-    expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 });
-    expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 25, y: 30, w: 50, h: 40 })).toBe(
-      true,
-    );
-    expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 26, y: 30, w: 50, h: 40 })).toBe(
-      false,
-    );
-  });
-
-  it("checks the projected visible region rather than only its bounding box", () => {
-    const triangle = [
-      [
-        { x: 0, y: 0 },
-        { x: 100, y: 0 },
-        { x: 0, y: 100 },
-      ],
-    ];
-    expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true);
-    expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false);
-  });
-});

apps/extension/src/tools/__tests__/surface-target.test.ts

@@ -1,306 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import { SessionManager } from "@/session-manager/manager";
-import type { CdpRunner } from "../shared";
-import {
-  captureSurfaceEnvironment,
-  resolveSurfacePointerTarget,
-  SURFACE_POINTER_MOVE_SETTLE_MS,
-} from "../surface-target";
-
-function fakeAgentWindow() {
-  return {
-    create: vi.fn(async () => 100),
-    remove: vi.fn(async () => {}),
-    ensureActiveTab: vi.fn(async () => {}),
-  };
-}
-
-function fakeBrowser() {
-  let loaderId = "loader-1";
-  let pageY = 0;
-  let quad = [10, 20, 110, 20, 110, 60, 10, 60];
-  const sent: Array<{ method: string; params?: object }> = [];
-  const cdp: CdpRunner = {
-    send: vi.fn(async (_tabId: number, method: string, params?: object) => {
-      sent.push({ method, params });
-      if (method === "Page.getFrameTree") {
-        return {
-          frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } },
-        } as never;
-      }
-      if (method === "Page.getLayoutMetrics") {
-        return {
-          cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY },
-          cssVisualViewport: {
-            clientWidth: 800,
-            clientHeight: 600,
-            pageX: 0,
-            pageY,
-            scale: 1,
-          },
-        } as never;
-      }
-      if (method === "DOM.getContentQuads") return { quads: [quad] } as never;
-      throw new Error(`unexpected CDP call ${method}`);
-    }) as CdpRunner["send"],
-    trackSessionTab: vi.fn(),
-  };
-  return {
-    cdp,
-    sent,
-    setLoaderId: (value: string) => {
-      loaderId = value;
-    },
-    setPageY: (value: number) => {
-      pageY = value;
-    },
-    setQuad: (value: number[]) => {
-      quad = value;
-    },
-  };
-}
-
-async function setupCapture() {
-  const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
-  const context = await manager.start("aa11");
-  context.refStore.set("e3", 99, {
-    tabId: 4,
-    kind: "surface",
-    visibleRect: { x: 10, y: 20, w: 100, h: 40 },
-  });
-  const browser = fakeBrowser();
-  const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined);
-  if ("code" in environment) throw new Error(environment.message);
-  const entry = context.refStore.resolveEntry("e3");
-  if (!entry) throw new Error("missing ref");
-  const capture = context.surfaceCaptures.create({
-    sessionId: "aa11",
-    tabId: 4,
-    navigationIdentity: environment.navigationIdentity,
-    surface: {
-      ref: "e3",
-      backendNodeId: 99,
-      observationGeneration: entry.generation,
-    },
-    topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
-    imageWidth: 200,
-    imageHeight: 80,
-    viewportSignature: environment.viewportSignature,
-    frameProjectionSignature: environment.frameProjectionSignature,
-  });
-  return { manager, context, browser, capture };
-}
-
-function pointParams(captureId: string) {
-  return {
-    ref: "@e3",
-    captureId,
-    imageX: 50,
-    imageY: 20,
-  };
-}
-
-describe("Surface screenshot-bound pointer target", () => {
-  it("maps one fresh capture coordinate without executing an action", async () => {
-    const { context, browser, capture } = await setupCapture();
-    const result = await resolveSurfacePointerTarget(
-      context,
-      { tabId: 4 },
-      pointParams(capture.id),
-      { cdp: browser.cdp },
-    );
-
-    if ("code" in result) throw new Error(JSON.stringify(result));
-    expect(result).toMatchObject({
-      usedRef: "e3",
-      point: { x: 35, y: 30 },
-      moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
-      capture: { id: capture.id, imageX: 50, imageY: 20 },
-    });
-    expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
-
-    const repeated = await resolveSurfacePointerTarget(
-      context,
-      { tabId: 4 },
-      pointParams(capture.id),
-      { cdp: browser.cdp },
-    );
-    expect(repeated).toMatchObject({
-      code: "permission_denied",
-      data: { reason: "surface_capture_consumed" },
-    });
-  });
-
-  it("rejects a new observation generation before sending input", async () => {
-    const { context, browser, capture } = await setupCapture();
-    context.refStore.replace([
-      [
-        "e3",
-        {
-          backendNodeId: 99,
-          tabId: 4,
-          kind: "surface" as const,
-          visibleRect: { x: 10, y: 20, w: 100, h: 40 },
-        },
-      ],
-    ]);
-
-    const result = await resolveSurfacePointerTarget(
-      context,
-      { tabId: 4 },
-      pointParams(capture.id),
-      { cdp: browser.cdp },
-    );
-    expect(result).toMatchObject({
-      code: "permission_denied",
-      data: { reason: "surface_capture_stale" },
-    });
-    expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
-  });
-
-  it("rejects navigation, scroll, and visible geometry changes", async () => {
-    for (const mutate of [
-      (browser: ReturnType<typeof fakeBrowser>) => browser.setLoaderId("loader-2"),
-      (browser: ReturnType<typeof fakeBrowser>) => browser.setPageY(20),
-      (browser: ReturnType<typeof fakeBrowser>) =>
-        browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]),
-    ]) {
-      const { context, browser, capture } = await setupCapture();
-      mutate(browser);
-      const result = await resolveSurfacePointerTarget(
-        context,
-        { tabId: 4 },
-        pointParams(capture.id),
-        { cdp: browser.cdp },
-      );
-      expect(result).toMatchObject({
-        code: "permission_denied",
-        data: { reason: "surface_capture_stale" },
-      });
-      expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
-    }
-  });
-
-  it("consumes the capture when coordinate validation fails", async () => {
-    const invalid = await setupCapture();
-    const outside = await resolveSurfacePointerTarget(
-      invalid.context,
-      { tabId: 4 },
-      { ...pointParams(invalid.capture.id), imageX: 200 },
-      { cdp: invalid.browser.cdp },
-    );
-    expect(outside).toMatchObject({
-      code: "invalid_params",
-      data: { reason: "surface_coordinate_invalid" },
-    });
-    expect(
-      await resolveSurfacePointerTarget(
-        invalid.context,
-        { tabId: 4 },
-        pointParams(invalid.capture.id),
-        { cdp: invalid.browser.cdp },
-      ),
-    ).toMatchObject({ data: { reason: "surface_capture_consumed" } });
-  });
-
-  it("maps image coordinates through an OOPIF frame projection", async () => {
-    const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
-    const context = await manager.start("aa11");
-    context.refStore.set("e3", 99, {
-      tabId: 4,
-      frameId: "child",
-      cdpSessionId: "child-session",
-      kind: "surface",
-      visibleRect: { x: 120, y: 110, w: 100, h: 40 },
-    });
-    const graph = {
-      rootFrameId: "main",
-      frames: [
-        { frameId: "main", target: { tabId: 4 } },
-        {
-          frameId: "child",
-          parentFrameId: "main",
-          ownerBackendNodeId: 77,
-          target: { tabId: 4, sessionId: "child-session" },
-        },
-      ],
-    };
-    const sent: Array<{ method: string; params?: object }> = [];
-    const cdp: CdpRunner = {
-      send: vi.fn(async (_tabId: number, method: string, params?: object) => {
-        sent.push({ method, params });
-        if (method === "Page.getFrameTree") {
-          return {
-            frameTree: {
-              frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
-            },
-          } as never;
-        }
-        if (method === "Page.getLayoutMetrics") {
-          return {
-            cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
-            cssVisualViewport: {
-              clientWidth: 800,
-              clientHeight: 600,
-              pageX: 0,
-              pageY: 0,
-              scale: 1,
-            },
-          } as never;
-        }
-        if (method === "DOM.getBoxModel") {
-          return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never;
-        }
-        if (method === "Input.dispatchMouseEvent") return {} as never;
-        throw new Error(`unexpected root CDP call ${method}`);
-      }) as CdpRunner["send"],
-      sendToTarget: vi.fn(async (_target, method) => {
-        if (method === "DOM.getContentQuads") {
-          return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never;
-        }
-        if (method === "Page.getLayoutMetrics") {
-          return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never;
-        }
-        throw new Error(`unexpected child CDP call ${method}`);
-      }) as CdpRunner["sendToTarget"],
-      getFrameGraph: vi.fn(async () => graph),
-      trackSessionTab: vi.fn(),
-    };
-    const environment = await captureSurfaceEnvironment(cdp, 4, "child");
-    if ("code" in environment) throw new Error(environment.message);
-    const entry = context.refStore.resolveEntry("e3");
-    if (!entry) throw new Error("missing ref");
-    const capture = context.surfaceCaptures.create({
-      sessionId: "aa11",
-      tabId: 4,
-      navigationIdentity: environment.navigationIdentity,
-      surface: {
-        ref: "e3",
-        frameId: "child",
-        backendNodeId: 99,
-        observationGeneration: entry.generation,
-      },
-      topViewportRect: { x: 120, y: 110, w: 100, h: 40 },
-      imageWidth: 200,
-      imageHeight: 80,
-      viewportSignature: environment.viewportSignature,
-      frameProjectionSignature: environment.frameProjectionSignature,
-    });
-
-    const result = await resolveSurfacePointerTarget(
-      context,
-      { tabId: 4 },
-      {
-        ref: "@e3",
-        captureId: capture.id,
-        imageX: 100,
-        imageY: 40,
-      },
-      { cdp },
-    );
-
-    if ("code" in result) throw new Error(JSON.stringify(result));
-    expect(result).toMatchObject({ point: { x: 170, y: 130 } });
-    expect(sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
-  });
-});

apps/extension/src/tools/__tests__/text-input.test.ts

@@ -1,128 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import type { CdpRunner } from "../shared";
-import { dispatchTextInput } from "../text-input";
-
-function makeCdp(onSend?: (method: string, params?: object) => void) {
-  const sent: Array<{ method: string; params?: object }> = [];
-  const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
-    sent.push({ method, params });
-    onSend?.(method, params);
-    return {} as T;
-  });
-  return { cdp: { send } as CdpRunner, sent };
-}
-
-describe("dispatchTextInput", () => {
-  it("types ASCII through complete keydown, char, and keyup sequences", async () => {
-    const fake = makeCdp();
-
-    const result = await dispatchTextInput(fake.cdp, 4, "Ab");
-
-    expect(result).toBeNull();
-    expect(
-      fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`),
-    ).toEqual([
-      "Input.dispatchKeyEvent:rawKeyDown",
-      "Input.dispatchKeyEvent:char",
-      "Input.dispatchKeyEvent:keyUp",
-      "Input.dispatchKeyEvent:rawKeyDown",
-      "Input.dispatchKeyEvent:char",
-      "Input.dispatchKeyEvent:keyUp",
-    ]);
-    expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 });
-    expect(fake.sent[1].params).toMatchObject({ text: "A" });
-  });
-
-  it("types non-ASCII through the committed IME insertion path", async () => {
-    const fake = makeCdp();
-
-    const result = await dispatchTextInput(fake.cdp, 4, "测试");
-
-    expect(result).toBeNull();
-    expect(fake.sent).toEqual([
-      {
-        method: "Input.dispatchKeyEvent",
-        params: {
-          type: "rawKeyDown",
-          key: "Process",
-          code: "",
-          windowsVirtualKeyCode: 229,
-          nativeVirtualKeyCode: 229,
-          modifiers: 0,
-        },
-      },
-      { method: "Input.insertText", params: { text: "测试" } },
-      {
-        method: "Input.dispatchKeyEvent",
-        params: {
-          type: "keyUp",
-          key: "Process",
-          code: "",
-          windowsVirtualKeyCode: 229,
-          nativeVirtualKeyCode: 229,
-          modifiers: 0,
-        },
-      },
-    ]);
-  });
-
-  it("preserves text order across keyboard and IME segments", async () => {
-    const fake = makeCdp();
-
-    const result = await dispatchTextInput(fake.cdp, 4, "A测试1");
-
-    expect(result).toBeNull();
-    const meaningful = fake.sent.flatMap((call) => {
-      if (call.method === "Input.insertText") {
-        return [`ime:${(call.params as { text: string }).text}`];
-      }
-      if (
-        call.method === "Input.dispatchKeyEvent" &&
-        (call.params as { type?: string }).type === "char"
-      ) {
-        return [`key:${(call.params as { text: string }).text}`];
-      }
-      return [];
-    });
-    expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]);
-  });
-
-  it("reports cancellation observed after an accepted IME insertion", async () => {
-    const abort = new AbortController();
-    const fake = makeCdp((method) => {
-      if (method === "Input.insertText") abort.abort();
-    });
-
-    const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
-
-    expect(result).toMatchObject({ code: "cancelled" });
-    expect(fake.sent.map((call) => call.method)).toEqual([
-      "Input.dispatchKeyEvent",
-      "Input.insertText",
-      "Input.dispatchKeyEvent",
-    ]);
-    expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" });
-  });
-
-  it("releases the IME process key without committing when cancellation follows keydown", async () => {
-    const abort = new AbortController();
-    const fake = makeCdp((method, params) => {
-      if (
-        method === "Input.dispatchKeyEvent" &&
-        (params as { type?: string }).type === "rawKeyDown"
-      ) {
-        abort.abort();
-      }
-    });
-
-    const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
-
-    expect(result).toMatchObject({ code: "cancelled" });
-    expect(fake.sent.map((call) => call.method)).toEqual([
-      "Input.dispatchKeyEvent",
-      "Input.dispatchKeyEvent",
-    ]);
-    expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" });
-    expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" });
-  });
-});

apps/extension/src/tools/dispatcher.ts

@@ -28,7 +28,6 @@ import type {
   ScreenshotParams,
   SelectParams,
   SnapshotParams,
-  TypeParams,
   WaitForNavigationParams,
 } from "@/transport/types";
 import { isRequestFrame } from "@/transport/types";
@@ -36,14 +35,7 @@ import { handleConsole } from "./console";
 import { type EmulateCdpRunner, handleEmulate } from "./emulate";
 import { handleEvaluate } from "./evaluate";
 import { handleRequestHelp } from "./human-loop";
-import {
+import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
-  handleClick,
-  handleFill,
-  handleHover,
-  handlePress,
-  handleSelect,
-  handleType,
-} from "./interaction";
 import {
   handleNavigate,
   handleNavigateBack,
@@ -391,7 +383,11 @@ export class ToolDispatcher {
                 ? {
                     cdp: this.cdp,
                     tabsApi: chromeTabsCaptureApi,
-                    conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
+                    // Active hover probing is opt-in. A held hover latch still
+                    // suppresses it, because probing would move the cursor off
+                    // the element the caller is deliberately holding.
+                    conditionalSurfaceProbe:
+                      params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
                     hoverProbeBypassOverlay: bypassOverlay,
                   }
                 : undefined,
@@ -499,17 +495,6 @@ export class ToolDispatcher {
             ),
           signal,
         );
-      case "tool.type":
-        return this.withHoverReleaseForRequest(
-          req.params as TypeParams,
-          () =>
-            handleType(
-              this.sessions,
-              req.params as TypeParams,
-              this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
-            ),
-          signal,
-        );
       case "tool.press":
         return this.withHoverReleaseForRequest(
           req.params as PressParams,
@@ -736,7 +721,6 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
     case "tool.click":
     case "tool.hover":
     case "tool.fill":
-    case "tool.type":
     case "tool.press":
     case "tool.select":
     case "tool.evaluate":

apps/extension/src/tools/editable-input.ts

@@ -1,282 +0,0 @@
-// Browser-native editing transaction for input, textarea, and contenteditable
-// targets. The CDP target/session is selected by interaction.ts; this module
-// owns the shorter-lived editable identity inside that target.
-
-import type { RpcError } from "@/transport/types";
-import { rpcError } from "./errors";
-import type { CdpRunner } from "./shared";
-
-interface EditableState {
-  supported: boolean;
-  connected: boolean;
-  focused: boolean;
-  value: string;
-  kind: "input" | "textarea" | "contenteditable" | "unsupported";
-}
-
-export interface EditableFillResult {
-  value: string;
-  replacedNode: boolean;
-}
-
-function cancelled(signal: AbortSignal | undefined): RpcError | null {
-  return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null;
-}
-
-async function inspectEditable(
-  cdp: CdpRunner,
-  tabId: number,
-  objectId: string,
-): Promise<EditableState> {
-  const reply = await cdp.send<{
-    result?: { value?: EditableState };
-  }>(tabId, "Runtime.callFunctionOn", {
-    objectId,
-    functionDeclaration: `function() {
-      /* bsk-editable-state */
-      const doc = this.ownerDocument;
-      const focused = !!doc && doc.activeElement === this;
-      if (this instanceof HTMLInputElement) {
-        return { supported: true, connected: this.isConnected, focused,
-          value: this.value, kind: 'input' };
-      }
-      if (this instanceof HTMLTextAreaElement) {
-        return { supported: true, connected: this.isConnected, focused,
-          value: this.value, kind: 'textarea' };
-      }
-      if (this.isContentEditable) {
-        return { supported: true, connected: this.isConnected, focused,
-          value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' };
-      }
-      return { supported: false, connected: this.isConnected === true, focused,
-        value: '', kind: 'unsupported' };
-    }`,
-    returnByValue: true,
-  });
-  const state = reply.result?.value;
-  if (!state || typeof state.value !== "string") {
-    throw new Error("editable state probe returned no value");
-  }
-  return state;
-}
-
-/**
- * Return the active editable from the anchor's own document. Using the
- * anchor's execution context also works for same-process nested frames, where
- * a target-level Runtime.evaluate would otherwise see the top document.
- */
-async function focusedEditableFromAnchor(
-  cdp: CdpRunner,
-  tabId: number,
-  anchorObjectId: string,
-): Promise<string | null> {
-  const reply = await cdp.send<{
-    result?: { objectId?: string; subtype?: string };
-  }>(tabId, "Runtime.callFunctionOn", {
-    objectId: anchorObjectId,
-    functionDeclaration: `function() {
-      /* bsk-editable-live */
-      const active = this.ownerDocument?.activeElement;
-      if (!active) return null;
-      const editable = active instanceof HTMLInputElement ||
-        active instanceof HTMLTextAreaElement || active.isContentEditable;
-      return editable ? active : null;
-    }`,
-    returnByValue: false,
-  });
-  if (reply.result?.subtype === "null") return null;
-  return typeof reply.result?.objectId === "string" ? reply.result.objectId : null;
-}
-
-async function resolveLiveEditable(
-  cdp: CdpRunner,
-  tabId: number,
-  anchorObjectId: string,
-): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> {
-  const anchor = await inspectEditable(cdp, tabId, anchorObjectId);
-  if (anchor.supported && anchor.connected && anchor.focused) {
-    return { objectId: anchorObjectId, state: anchor, replaced: false };
-  }
-
-  const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId);
-  if (!focusedObjectId) {
-    return rpcError(
-      "cdp_failed",
-      "input_not_applied",
-      "the target editable lost focus or detached before input",
-      { phase: "resolve_live_editable" },
-    );
-  }
-  const focused = await inspectEditable(cdp, tabId, focusedObjectId);
-  if (!focused.supported || !focused.connected || !focused.focused) {
-    return rpcError(
-      "cdp_failed",
-      "input_not_applied",
-      "the replacement editable was not live and focused",
-      { phase: "resolve_live_editable" },
-    );
-  }
-  return { objectId: focusedObjectId, state: focused, replaced: true };
-}
-
-async function selectEditableContents(
-  cdp: CdpRunner,
-  tabId: number,
-  objectId: string,
-): Promise<void> {
-  const reply = await cdp.send<{
-    result?: { value?: { selected?: boolean } };
-  }>(tabId, "Runtime.callFunctionOn", {
-    objectId,
-    functionDeclaration: `function() {
-      /* bsk-editable-selection */
-      if (!this.isConnected || this.ownerDocument?.activeElement !== this) {
-        return { selected: false };
-      }
-      if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
-        try {
-          this.setSelectionRange(0, this.value.length);
-        } catch {
-          if (typeof this.select !== 'function') return { selected: false };
-          this.select();
-        }
-        return { selected: true };
-      }
-      if (this.isContentEditable) {
-        const selection = this.ownerDocument.getSelection();
-        if (!selection) return { selected: false };
-        const range = this.ownerDocument.createRange();
-        range.selectNodeContents(this);
-        selection.removeAllRanges();
-        selection.addRange(range);
-        return { selected: true };
-      }
-      return { selected: false };
-    }`,
-    returnByValue: true,
-  });
-  if (reply.result?.value?.selected !== true) {
-    throw new Error("could not select the live editable contents");
-  }
-}
-
-async function waitForEditableSettlement(
-  cdp: CdpRunner,
-  tabId: number,
-  anchorObjectId: string,
-): Promise<void> {
-  await cdp.send(tabId, "Runtime.callFunctionOn", {
-    objectId: anchorObjectId,
-    functionDeclaration: `function() {
-      /* bsk-editable-sync */
-      return new Promise((resolve) => {
-        let settled = false;
-        const finish = () => {
-          if (settled) return;
-          settled = true;
-          clearTimeout(timer);
-          resolve();
-        };
-        const timer = setTimeout(finish, 100);
-        const view = this.ownerDocument?.defaultView;
-        if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; }
-        view.requestAnimationFrame(() => view.requestAnimationFrame(finish));
-      });
-    }`,
-    awaitPromise: true,
-    returnByValue: true,
-  });
-}
-
-async function deleteSelection(cdp: CdpRunner, tabId: number): Promise<void> {
-  await cdp.send(tabId, "Input.dispatchKeyEvent", {
-    type: "rawKeyDown",
-    key: "Backspace",
-    code: "Backspace",
-    windowsVirtualKeyCode: 8,
-  });
-  await cdp.send(tabId, "Input.dispatchKeyEvent", {
-    type: "keyUp",
-    key: "Backspace",
-    code: "Backspace",
-    windowsVirtualKeyCode: 8,
-  });
-}
-
-/**
- * Replace or append text through Chromium's editing pipeline. No value setter
- * or synthetic DOM events are used: controlled components observe the same
- * beforeinput/input path as a real browser edit. The live editable is resolved
- * again after settlement so a framework may remount it without producing a
- * false failure or a stale-object success.
- */
-export async function fillEditable(
-  cdp: CdpRunner,
-  tabId: number,
-  anchorObjectId: string,
-  value: string,
-  clearBefore: boolean,
-  signal?: AbortSignal,
-): Promise<EditableFillResult | RpcError> {
-  try {
-    const abortedBefore = cancelled(signal);
-    if (abortedBefore) return abortedBefore;
-
-    const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId);
-    if ("code" in initial) return initial;
-    let replacedNode = initial.replaced;
-
-    if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId);
-    const abortedAfterSelection = cancelled(signal);
-    if (abortedAfterSelection) return abortedAfterSelection;
-
-    if (value.length > 0) {
-      await cdp.send(tabId, "Input.insertText", { text: value });
-    } else if (clearBefore) {
-      await deleteSelection(cdp, tabId);
-    }
-
-    const abortedAfterInput = cancelled(signal);
-    if (abortedAfterInput) return abortedAfterInput;
-    await waitForEditableSettlement(cdp, tabId, initial.objectId);
-
-    const abortedAfterSettlement = cancelled(signal);
-    if (abortedAfterSettlement) return abortedAfterSettlement;
-
-    let finalObjectId = initial.objectId;
-    let finalState = await inspectEditable(cdp, tabId, finalObjectId);
-    if (!finalState.connected || !finalState.focused) {
-      const live = await resolveLiveEditable(cdp, tabId, initial.objectId);
-      if ("code" in live) return live;
-      finalObjectId = live.objectId;
-      finalState = live.state;
-      replacedNode ||= live.replaced || finalObjectId !== initial.objectId;
-    }
-
-    const applied = clearBefore
-      ? finalState.value === value
-      : value.length === 0 || finalState.value.includes(value);
-    if (!finalState.supported || !applied) {
-      return rpcError(
-        "cdp_failed",
-        "input_not_applied",
-        "the page did not retain the requested input value",
-        {
-          expected_value_length: value.length,
-          actual_value_length: finalState.value.length,
-          phase: "readback",
-          editable_replaced: replacedNode,
-        },
-      );
-    }
-
-    return { value: finalState.value, replacedNode };
-  } catch (err) {
-    return rpcError(
-      "cdp_failed",
-      "input_not_applied",
-      err instanceof Error ? err.message : String(err),
-      { phase: "edit_transaction" },
-    );
-  }
-}

apps/extension/src/tools/interaction.ts

@@ -1,5 +1,5 @@
-// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`,
+// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
-// `tool.press`, and `tool.select`.
+// `tool.select`.
 //
 // All interaction tools:
 // 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -11,7 +11,7 @@
 //    commands.
 
 import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
-import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
+import type { CdpTarget } from "@/browser-driver/frame-graph";
 import type { SessionContext, SessionManager } from "@/session-manager/manager";
 import type {
   ClickParams,
@@ -27,15 +27,11 @@ import type {
   RpcError,
   SelectParams,
   SelectResult,
-  TypeParams,
-  TypeResult,
 } from "@/transport/types";
 import { attachDialogs, markDialogCursor } from "./dialogs";
-import { fillEditable } from "./editable-input";
 import { backendNodeToObject } from "./element-geometry";
 import { rpcError } from "./errors";
 import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
-import { dispatchMouseClick } from "./mouse-input";
 import {
   type CdpRunner,
   type ChromeTabsApi,
@@ -47,8 +43,6 @@ import {
   resolveTargetTab,
 } from "./shared";
 import { resolveSnapshotRef } from "./snapshot-ref";
-import { resolveSurfacePointerTarget } from "./surface-target";
-import { dispatchTextInput } from "./text-input";
 
 export interface InteractionDeps {
   cdp: CdpRunner;
@@ -165,7 +159,7 @@ async function resolveBackendNode(
     };
   }
   if (hasRef) {
-    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
+    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
     if (isRpcError(resolved)) return resolved;
     return {
       backendNodeId: resolved.backendNodeId,
@@ -221,146 +215,15 @@ async function resolveBackendNode(
   }
 }
 
-interface InputDispatchContext {
-  target: CdpTarget;
-  cdp: CdpRunner;
-  backendNodeId?: number;
-  frameId?: string;
-  usedRef?: string;
-  usedSelector?: string;
-}
-
-function inputContextForNode(
-  cdp: CdpRunner,
-  node: Awaited<ReturnType<typeof resolveBackendNode>>,
-): (InputDispatchContext & { backendNodeId: number }) | RpcError {
-  if (isRpcError(node)) return node;
-  return {
-    target: node.cdpTarget,
-    cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
-    backendNodeId: node.backendNodeId,
-    frameId: node.frameId,
-    usedRef: node.usedRef,
-    usedSelector: node.usedSelector,
-  };
-}
-
-interface FocusProbe {
-  hasFocus: boolean;
-  activeTag: string;
-  textEntry: boolean;
-}
-
-/** Resolve keyboard input to the CDP target that currently owns focus. */
-async function resolveFocusedInputContext(
-  cdp: CdpRunner,
-  tabId: number,
-  mode: "press" | "type",
-): Promise<InputDispatchContext | RpcError> {
-  if (!cdp.getFrameGraph) {
-    return rpcError(
-      "cdp_failed",
-      "focus_target_unresolved",
-      `${mode} cannot determine the focused frame target`,
-    );
-  }
-
-  let targets: CdpTarget[];
-  try {
-    const graph = await cdp.getFrameGraph(tabId);
-    const unique = new Map<string, CdpTarget>();
-    unique.set(cdpTargetKey({ tabId }), { tabId });
-    for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
-    targets = [...unique.values()];
-  } catch (err) {
-    return rpcError(
-      "cdp_failed",
-      "focus_target_unresolved",
-      err instanceof Error ? err.message : String(err),
-    );
-  }
-
-  const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
-  await Promise.all(
-    targets.map(async (candidate) => {
-      try {
-        const reply = await cdpRunnerForTarget(cdp, candidate).send<{
-          result?: { value?: FocusProbe };
-        }>(tabId, "Runtime.evaluate", {
-          expression: `(() => {
-            /* bsk-focus-target-probe */
-            const active = document.activeElement;
-            const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : '';
-            const nonTextInputTypes = new Set([
-              'button', 'checkbox', 'color', 'date', 'datetime-local', 'file',
-              'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit',
-              'time', 'week'
-            ]);
-            const role = active instanceof Element ? active.getAttribute('role') : null;
-            return {
-              hasFocus: document.hasFocus(),
-              activeTag: active instanceof Element ? active.tagName.toLowerCase() : '',
-              textEntry:
-                (active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) ||
-                active instanceof HTMLTextAreaElement ||
-                (active instanceof HTMLElement && active.isContentEditable) ||
-                role === 'textbox' || role === 'searchbox'
-            };
-          })()`,
-          returnByValue: true,
-        });
-        const probe = reply.result?.value;
-        if (probe?.hasFocus) focused.push({ target: candidate, probe });
-      } catch {
-        // A target may navigate or detach while the graph is being probed.
-      }
-    }),
-  );
-
-  // A top document whose active element is an iframe delegates focus to
-  // that frame. Removing those delegating targets leaves the deepest OOPIF
-  // target while same-process frames continue to use the shared root target.
-  const owners = focused.filter(({ probe }) => {
-    if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false;
-    return mode !== "type" || probe.textEntry;
-  });
-  if (owners.length !== 1) {
-    return rpcError(
-      "cdp_failed",
-      "focus_target_unresolved",
-      owners.length === 0
-        ? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target`
-        : `${mode} found multiple focused frame targets`,
-      { candidate_count: owners.length },
-    );
-  }
-  const target = owners[0].target;
-  return { target, cdp: cdpRunnerForTarget(cdp, target) };
-}
-
 // ---------------------------------------------------------------------------
 // tool.click
 // ---------------------------------------------------------------------------
 
-interface ResolvedClickPointerTarget {
-  point: { x: number; y: number };
-  usedRef?: string;
-  usedSelector?: string;
-  moveSettleMs: number;
-  capture?: {
-    id: string;
-    imageX: number;
-    imageY: number;
-  };
-}
-
 export async function handleClick(
   manager: SessionManager,
   params: ClickParams,
   deps: InteractionDeps = getDefaultDeps(),
 ): Promise<ClickResult | RpcError> {
-  const hasSurfacePointParams =
-    params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined;
   const ctxOrErr = lookupSession(manager, params, "click");
   if (isRpcError(ctxOrErr)) return ctxOrErr;
   const ctx = ctxOrErr;
@@ -372,67 +235,39 @@ export async function handleClick(
   if (denied) return denied;
   const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
 
-  const button: MouseButton = params.button ?? "left";
+  const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
-  const clickCount = params.click_count ?? 1;
+  if (isRpcError(node)) return node;
-  if (!Number.isSafeInteger(clickCount) || clickCount < 1) {
-    return { code: "invalid_params", message: "click_count must be a positive integer" };
-  }
-  const modifiers = modifiersBitfield(params.modifiers);
 
   if (throwIfAborted(deps.signal)) {
     return { code: "cancelled", message: "click aborted" };
   }
 
   deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-  const pointerTarget: ResolvedClickPointerTarget | RpcError = hasSurfacePointParams
+  const geometry = await resolveNodeGeometry(
-    ? await resolveSurfacePointerTarget(
+    deps.cdp,
-        ctx,
+    target.tabId,
-        target,
+    {
-        {
+      target: node.cdpTarget,
-          ref: params.ref,
+      backendNodeId: node.backendNodeId,
-          selector: params.selector,
+      ...(node.frameId ? { frameId: node.frameId } : {}),
-          captureId: params.capture_id,
+    },
-          imageX: params.image_x,
+    { scrollIntoView: true },
-          imageY: params.image_y,
+  );
-        },
+  if (isRpcError(geometry)) return geometry;
-        {
+  const centre = geometry.actionPoint;
-          cdp: deps.cdp,
-          signal: deps.signal,
-        },
-      )
-    : await (async () => {
-        const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
-        if (isRpcError(node)) return node;
-        const geometry = await resolveNodeGeometry(
-          deps.cdp,
-          target.tabId,
-          {
-            target: node.cdpTarget,
-            backendNodeId: node.backendNodeId,
-            ...(node.frameId ? { frameId: node.frameId } : {}),
-          },
-          { scrollIntoView: true },
-        );
-        if (isRpcError(geometry)) return geometry;
-        return {
-          point: geometry.actionPoint,
-          usedRef: node.usedRef,
-          usedSelector: node.usedSelector,
-          moveSettleMs: 0,
-        };
-      })();
-  if (isRpcError(pointerTarget)) return pointerTarget;
 
   if (throwIfAborted(deps.signal)) {
     return { code: "cancelled", message: "click aborted" };
   }
 
-  const overlayBlocking = await checkOverlayAtPoint(
+  const button: MouseButton = params.button ?? "left";
-    deps.cdp,
+  const clickCount = params.click_count ?? 1;
-    target.tabId,
+  if (clickCount < 1) {
-    pointerTarget.point.x,
+    return { code: "invalid_params", message: "click_count must be greater than zero" };
-    pointerTarget.point.y,
+  }
-  );
+  const modifiers = modifiersBitfield(params.modifiers);
+
+  const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
   let automationBypassEnabled = false;
   if (overlayBlocking && deps.bypassOverlay) {
     try {
@@ -444,16 +279,47 @@ export async function handleClick(
   }
 
   try {
-    const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, {
+    // Move first so hover state activates, then press → release.
+    await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
+      type: "mouseMoved",
+      x: centre.x,
+      y: centre.y,
+      modifiers,
+    });
+    if (throwIfAborted(deps.signal)) {
+      return { code: "cancelled", message: "click aborted" };
+    }
+    await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
+      type: "mousePressed",
+      x: centre.x,
+      y: centre.y,
       button,
       clickCount,
       modifiers,
-      signal: deps.signal,
-      moveSettleMs: pointerTarget.moveSettleMs,
     });
-    if (dispatch === "cancelled") {
+    if (throwIfAborted(deps.signal)) {
+      try {
+        await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
+          type: "mouseReleased",
+          x: centre.x,
+          y: centre.y,
+          button,
+          clickCount,
+          modifiers,
+        });
+      } catch (err) {
+        console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err);
+      }
       return { code: "cancelled", message: "click aborted" };
     }
+    await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
+      type: "mouseReleased",
+      x: centre.x,
+      y: centre.y,
+      button,
+      clickCount,
+      modifiers,
+    });
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -471,17 +337,10 @@ export async function handleClick(
 
   return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
     tab_id: target.tabId,
-    used_ref: pointerTarget.usedRef,
+    used_ref: node.usedRef,
-    used_selector: pointerTarget.usedSelector,
+    used_selector: node.usedSelector,
-    x: pointerTarget.point.x,
+    x: centre.x,
-    y: pointerTarget.point.y,
+    y: centre.y,
-    ...(pointerTarget.capture
-      ? {
-          capture_id: pointerTarget.capture.id,
-          image_x: pointerTarget.capture.imageX,
-          image_y: pointerTarget.capture.imageY,
-        }
-      : {}),
   });
 }
 
@@ -701,47 +560,6 @@ function isFillable(node: DescribedNode): boolean {
   return false;
 }
 
-/** `type` also supports custom text-entry widgets that expose an ARIA role. */
-function isTypeTarget(node: DescribedNode): boolean {
-  const tag = (node.nodeName ?? "").toUpperCase();
-  const attrs = node.attributes ?? [];
-  const attributes = new Map<string, string>();
-  for (let i = 0; i + 1 < attrs.length; i += 2) {
-    attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase());
-  }
-  if (tag === "TEXTAREA") return true;
-  if (tag === "INPUT") {
-    const type = attributes.get("type") ?? "text";
-    return !new Set([
-      "button",
-      "checkbox",
-      "color",
-      "date",
-      "datetime-local",
-      "file",
-      "hidden",
-      "image",
-      "month",
-      "radio",
-      "range",
-      "reset",
-      "submit",
-      "time",
-      "week",
-    ]).has(type);
-  }
-  const contentEditable = attributes.get("contenteditable");
-  if (
-    contentEditable === "" ||
-    contentEditable === "true" ||
-    contentEditable === "plaintext-only"
-  ) {
-    return true;
-  }
-  const role = attributes.get("role");
-  return role === "textbox" || role === "searchbox";
-}
-
 type SelectMutationResult =
   | {
       ok: true;
@@ -780,9 +598,8 @@ export async function handleFill(
   const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
 
   const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
-  const input = inputContextForNode(deps.cdp, node);
+  if (isRpcError(node)) return node;
-  if (isRpcError(input)) return input;
+  const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
-  const nodeCdp = input.cdp;
 
   try {
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -790,7 +607,7 @@ export async function handleFill(
       target.tabId,
       "DOM.describeNode",
       {
-        backendNodeId: input.backendNodeId,
+        backendNodeId: node.backendNodeId,
       },
     );
     if (!described.node || !isFillable(described.node)) {
@@ -803,15 +620,15 @@ export async function handleFill(
     const scrollErr = await scrollElementAndFramesIntoView(
       deps.cdp,
       target.tabId,
-      input.target,
+      node.cdpTarget,
-      input.backendNodeId,
+      node.backendNodeId,
-      input.frameId,
+      node.frameId,
     );
     if (scrollErr) return scrollErr;
     if (throwIfAborted(deps.signal)) {
       return { code: "cancelled", message: "fill aborted" };
     }
-    await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
+    await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -823,118 +640,62 @@ export async function handleFill(
     return { code: "cancelled", message: "fill aborted" };
   }
 
-  const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
+  const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
   if (isRpcError(objectIdOrErr)) return objectIdOrErr;
   const objectId = objectIdOrErr;
   const clearBefore = params.clear_before ?? true;
-  const fillResult = await fillEditable(
-    nodeCdp,
-    target.tabId,
-    objectId,
-    params.value,
-    clearBefore,
-    deps.signal,
-  );
-  if (isRpcError(fillResult)) return fillResult;
 
-  return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
+  try {
-    tab_id: target.tabId,
+    if (clearBefore) {
-    used_ref: input.usedRef,
+      // Clear input/textarea value or wipe contenteditable innerText,
-    used_selector: input.usedSelector,
+      // then fire `input` so frameworks observe the empty state.
-    value_length: fillResult.value.length,
+      await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
-  });
+        objectId,
-}
+        functionDeclaration: `function() {
-
+          if (this.isContentEditable) { this.textContent = ''; }
-// ---------------------------------------------------------------------------
+          else {
-// tool.type
+            const proto = this instanceof HTMLTextAreaElement
-// ---------------------------------------------------------------------------
+              ? HTMLTextAreaElement.prototype
-
+              : HTMLInputElement.prototype;
-export async function handleType(
+            const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
-  manager: SessionManager,
+            if (descriptor && descriptor.set) descriptor.set.call(this, '');
-  params: TypeParams,
+            else this.value = '';
-  deps: InteractionDeps = getDefaultDeps(),
+          }
-): Promise<TypeResult | RpcError> {
+          this.dispatchEvent(new Event('input', { bubbles: true }));
-  if (!params || typeof params.text !== "string") {
+        }`,
-    return { code: "invalid_params", message: "type requires a text string" };
+        returnByValue: true,
-  }
-  if (params.focused !== undefined && typeof params.focused !== "boolean") {
-    return { code: "invalid_params", message: "type focused must be a boolean" };
-  }
-  const hasExplicitTarget =
-    (typeof params.ref === "string" && params.ref.length > 0) ||
-    (typeof params.selector === "string" && params.selector.length > 0);
-  if (params.focused === true && hasExplicitTarget) {
-    return {
-      code: "invalid_params",
-      message: "type: pass a ref/selector or focused=true, not both",
-    };
-  }
-  if (params.focused !== true && !hasExplicitTarget) {
-    return {
-      code: "invalid_params",
-      message: "type requires a ref/selector or focused=true",
-    };
-  }
-  const ctxOrErr = lookupSession(manager, params, "type");
-  if (isRpcError(ctxOrErr)) return ctxOrErr;
-  const ctx = ctxOrErr;
-  if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
-  const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
-  if (isRpcError(target)) return target;
-  const denied = enforceAgentWindow(ctx, target, "type");
-  if (denied) return denied;
-  const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
-
-  let input: InputDispatchContext | RpcError;
-  if (hasExplicitTarget) {
-    const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type");
-    const explicitInput = inputContextForNode(deps.cdp, node);
-    if (isRpcError(explicitInput)) return explicitInput;
-    input = explicitInput;
-    try {
-      const described = await explicitInput.cdp.send<{ node?: DescribedNode }>(
-        target.tabId,
-        "DOM.describeNode",
-        { backendNodeId: explicitInput.backendNodeId },
-      );
-      if (!described.node || !isTypeTarget(described.node)) {
-        return rpcError(
-          "invalid_params",
-          "target_not_fillable",
-          `element ${described.node?.nodeName ?? "?"} not text-editable`,
-        );
-      }
-      deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-      const scrollErr = await scrollElementAndFramesIntoView(
-        deps.cdp,
-        target.tabId,
-        explicitInput.target,
-        explicitInput.backendNodeId,
-        explicitInput.frameId,
-      );
-      if (scrollErr) return scrollErr;
-      if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
-      await explicitInput.cdp.send(target.tabId, "DOM.focus", {
-        backendNodeId: explicitInput.backendNodeId,
       });
-    } catch (err) {
-      return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
     }
-  } else {
+    if (throwIfAborted(deps.signal)) {
-    input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type");
+      return { code: "cancelled", message: "fill aborted" };
-    if (isRpcError(input)) return input;
+    }
+    // CDP `Input.insertText` handles IME / multi-byte input out of the
+    // box, much more reliably than per-key `dispatchKeyEvent`.
+    await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
+    if (throwIfAborted(deps.signal)) {
+      return { code: "cancelled", message: "fill aborted" };
+    }
+    // Fire `input` + `change` so React / Vue controlled inputs commit.
+    await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
+      objectId,
+      functionDeclaration: `function() {
+        this.dispatchEvent(new Event('input', { bubbles: true }));
+        this.dispatchEvent(new Event('change', { bubbles: true }));
+      }`,
+      returnByValue: true,
+    });
+  } catch (err) {
+    return {
+      code: "cdp_failed",
+      message: err instanceof Error ? err.message : String(err),
+    };
   }
 
-  deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-  const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal);
-  if (dispatchError) return dispatchError;
-
   return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
     tab_id: target.tabId,
-    used_ref: input.usedRef,
+    used_ref: node.usedRef,
-    used_selector: input.usedSelector,
+    used_selector: node.usedSelector,
-    text_length: params.text.length,
+    value_length: params.value.length,
-    verification: "dispatched",
   });
 }
 
@@ -1109,38 +870,31 @@ export async function handlePress(
     };
   }
 
-  let input: InputDispatchContext | RpcError;
   // Optional focus before key dispatch.
   if (params.ref || params.selector) {
     const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
-    const explicitInput = inputContextForNode(deps.cdp, node);
+    if (isRpcError(node)) return node;
-    if (isRpcError(explicitInput)) return explicitInput;
+    const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
-    input = explicitInput;
     try {
       deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
       const scrollErr = await scrollElementAndFramesIntoView(
         deps.cdp,
         target.tabId,
-        explicitInput.target,
+        node.cdpTarget,
-        explicitInput.backendNodeId,
+        node.backendNodeId,
-        explicitInput.frameId,
+        node.frameId,
       );
       if (scrollErr) return scrollErr;
       if (throwIfAborted(deps.signal)) {
         return { code: "cancelled", message: "press aborted" };
       }
-      await explicitInput.cdp.send(target.tabId, "DOM.focus", {
+      await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
-        backendNodeId: explicitInput.backendNodeId,
-      });
     } catch (err) {
       return {
         code: "cdp_failed",
         message: err instanceof Error ? err.message : String(err),
       };
     }
-  } else {
-    input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press");
-    if (isRpcError(input)) return input;
   }
 
   if (throwIfAborted(deps.signal)) {
@@ -1154,7 +908,7 @@ export async function handlePress(
   try {
     let cancelled = false;
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-    await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
+    await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
       type: "rawKeyDown",
       key: descriptor.key,
       code: descriptor.code,
@@ -1168,7 +922,7 @@ export async function handlePress(
       typeof descriptor.text === "string" &&
       descriptor.text.length > 0
     ) {
-      await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
+      await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
         type: "char",
         key: descriptor.key,
         code: descriptor.code,
@@ -1185,7 +939,7 @@ export async function handlePress(
         cancelled = true;
       }
     }
-    await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
+    await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
       type: "keyUp",
       key: descriptor.key,
       code: descriptor.code,

apps/extension/src/tools/mouse-input.ts

@@ -1,96 +0,0 @@
-import type { MouseButton } from "@/transport/types";
-import type { Point } from "./geometry";
-import type { CdpRunner } from "./shared";
-
-interface MouseClickDispatchOptions {
-  button: MouseButton;
-  clickCount: number;
-  modifiers: number;
-  signal?: AbortSignal;
-  moveSettleMs?: number;
-}
-
-type MouseClickDispatchResult = "completed" | "cancelled";
-
-function waitForSettle(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
-  if (ms <= 0) return Promise.resolve(signal?.aborted !== true);
-  if (signal?.aborted) return Promise.resolve(false);
-  return new Promise((resolve) => {
-    let settled = false;
-    const finish = (completed: boolean) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      signal?.removeEventListener("abort", onAbort);
-      resolve(completed);
-    };
-    const timer = setTimeout(() => finish(true), ms);
-    const onAbort = () => finish(false);
-    signal?.addEventListener("abort", onAbort, { once: true });
-  });
-}
-
-/**
- * Dispatch one coherent CDP mouse click in top-viewport CSS coordinates.
- *
- * Keeping the sequence here prevents DOM and Surface interaction paths from
- * duplicating cancellation and best-effort release behavior. Callers that
- * target asynchronously hit-tested content may request a settle interval.
- */
-export async function dispatchMouseClick(
-  cdp: CdpRunner,
-  tabId: number,
-  point: Point,
-  options: MouseClickDispatchOptions,
-): Promise<MouseClickDispatchResult> {
-  const { button, clickCount, modifiers, signal, moveSettleMs = 0 } = options;
-  let pressed = false;
-  const release = () =>
-    cdp.send(tabId, "Input.dispatchMouseEvent", {
-      type: "mouseReleased",
-      x: point.x,
-      y: point.y,
-      button,
-      clickCount,
-      modifiers,
-    });
-
-  try {
-    await cdp.send(tabId, "Input.dispatchMouseEvent", {
-      type: "mouseMoved",
-      x: point.x,
-      y: point.y,
-      modifiers,
-    });
-    if (!(await waitForSettle(moveSettleMs, signal))) return "cancelled";
-
-    await cdp.send(tabId, "Input.dispatchMouseEvent", {
-      type: "mousePressed",
-      x: point.x,
-      y: point.y,
-      button,
-      clickCount,
-      modifiers,
-    });
-    pressed = true;
-
-    if (signal?.aborted) {
-      await release();
-      pressed = false;
-      return "cancelled";
-    }
-
-    await release();
-    pressed = false;
-    return "completed";
-  } catch (error) {
-    if (pressed) {
-      try {
-        await release();
-      } catch (releaseError) {
-        console.debug("[bsk mouse-input] best-effort mouse release failed", releaseError);
-      }
-    }
-    throw error;
-  }
-}

apps/extension/src/tools/observation.ts

@@ -8,7 +8,6 @@ import {
   applyVomInteractionRecovery,
   type CondSurface,
   isVomReferenceNode,
-  type Rect,
   renderVom,
   type VomNode,
   type VomOptions,
@@ -48,25 +47,21 @@ import {
   type ToolEffect,
 } from "./shared";
 import { resolveSnapshotRef } from "./snapshot-ref";
-import { surfaceVisibleRect } from "./surface-coordinate";
-import { captureSurfaceEnvironment } from "./surface-target";
 import {
   type CapturedNode,
+  type CapturedSurfaceProbe,
   type CapturedViewModel,
   captureViewModel,
   collectOverlayExcludedBackendIds,
+  probeHoverSurfaces,
 } from "./vom/capture";
 import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
+import { withOverlayBypass } from "./vom/hover-perception";
 import { probeTooltipNames } from "./vom/name-enrichment";
 import {
   type CaptureVomObservationResult,
   projectRecordSafeObservation,
 } from "./vom/record-safe-observation";
-import {
-  clusterRenderedSurfaces,
-  discoverRenderedSurfaces,
-  projectRenderedSurfaces,
-} from "./vom/rendered-surfaces";
 import {
   buildSemanticGraph,
   buildSemanticVomScene,
@@ -98,9 +93,6 @@ export const normaliseRef = sharedNormaliseRef;
 // screenshot — `tool.screenshot`
 // ---------------------------------------------------------------------------
 
-const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
-const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
-
 /**
  * Strip the `data:image/...;base64,` prefix from a Chrome
  * `captureVisibleTab` dataURL and return the raw base64 payload.
@@ -193,42 +185,18 @@ async function captureElementScreenshot(
   target: CdpTarget,
   backendNodeId: number,
   frameId?: string,
-  visualSurface = false,
-  observedVisibleRect?: Rect,
   signal?: AbortSignal,
-): Promise<
+): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
-  { image_base64: string; width: number; height: number; topViewportRect: Rect } | RpcError
-> {
   if (signal?.aborted) return cancelled("screenshot");
   const geometry = await resolveNodeGeometry(
     cdp,
     tabId,
     { target, backendNodeId, ...(frameId ? { frameId } : {}) },
-    { scrollIntoView: !visualSurface },
+    { scrollIntoView: true },
   );
   if (isRpcError(geometry)) return geometry;
   if (signal?.aborted) return cancelled("screenshot");
-  const topViewportRect = surfaceVisibleRect(
+  const rect = geometry.topBounds;
-    geometry.topBounds,
-    visualSurface ? observedVisibleRect : undefined,
-  );
-  if (!topViewportRect) {
-    return { code: "permission_denied", message: "surface is no longer visible" };
-  }
-  const rect = {
-    x: topViewportRect.x,
-    y: topViewportRect.y,
-    width: topViewportRect.w,
-    height: topViewportRect.h,
-  };
-  const scale = visualSurface
-    ? Math.min(
-        1,
-        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
-        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
-        Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
-      )
-    : 1;
 
   try {
     const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
@@ -238,7 +206,7 @@ async function captureElementScreenshot(
         y: rect.y,
         width: rect.width,
         height: rect.height,
-        scale,
+        scale: 1,
       },
     });
     if (signal?.aborted) return cancelled("screenshot");
@@ -250,7 +218,7 @@ async function captureElementScreenshot(
       width: Math.round(rect.width),
       height: Math.round(rect.height),
     };
-    return { image_base64, width: dims.width, height: dims.height, topViewportRect };
+    return { image_base64, width: dims.width, height: dims.height };
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -345,7 +313,7 @@ export async function handleScreenshot(
     if (!deps.cdp) {
       return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
     }
-    const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
+    const node = resolveSnapshotRef(ctx, ref, target.tabId);
     if (isRpcError(node)) return node;
     if (signal?.aborted) return cancelled("screenshot");
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -365,54 +333,18 @@ export async function handleScreenshot(
           nodeTarget,
           node.backendNodeId,
           node.frameId,
-          node.kind === "surface",
-          node.visibleRect,
           signal,
         ),
       deps.sendToTab,
     );
     if (isRpcError(captured)) return captured;
     if (signal?.aborted) return cancelled("screenshot");
-    const capture =
-      node.kind === "surface"
-        ? await (async () => {
-            const environment = await captureSurfaceEnvironment(cdp, target.tabId, node.frameId);
-            if (isRpcError(environment)) return environment;
-            return ctx.surfaceCaptures.create({
-              sessionId: ctx.sessionId,
-              tabId: target.tabId,
-              navigationIdentity: environment.navigationIdentity,
-              surface: {
-                ref: node.refKey,
-                ...(node.frameId ? { frameId: node.frameId } : {}),
-                backendNodeId: node.backendNodeId,
-                observationGeneration: node.generation,
-              },
-              topViewportRect: captured.topViewportRect,
-              imageWidth: captured.width,
-              imageHeight: captured.height,
-              viewportSignature: environment.viewportSignature,
-              frameProjectionSignature: environment.frameProjectionSignature,
-            });
-          })()
-        : null;
-    if (capture && isRpcError(capture)) return capture;
     return withShotDialogs({
       image_base64: captured.image_base64,
       width: captured.width,
       height: captured.height,
       format: "png",
       tab_id: target.tabId,
-      ...(capture
-        ? {
-            capture: {
-              id: capture.id,
-              surface_ref: `@${capture.surface.ref}`,
-              coordinate_space: "capture-image-pixel" as const,
-              expires_at: capture.expiresAt,
-            },
-          }
-        : {}),
     });
   }
 
@@ -596,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
   return blocks;
 }
 
-function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
+function buildConditionalSurfaces(
-  const probes = captured.surfaceProbes ?? [];
+  nodes: VomNode[],
+  captured: CapturedViewModel,
+  probes: CapturedSurfaceProbe[],
+): CondSurface[] {
   if (probes.length === 0) return [];
 
   const surfaces: CondSurface[] = [];
@@ -720,6 +655,7 @@ function findSurfaceNodeByPoint(
 export interface BuildVomSceneOptions {
   pageUrl?: string;
   supplementalNames?: ReadonlyMap<string, string>;
+  surfaceProbes?: CapturedSurfaceProbe[];
 }
 
 export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
@@ -888,18 +824,19 @@ export function buildFrameVomScene(
     excludedBackendNodeIds: captured.excludedBackendNodeIds,
     supplementalNames: options.supplementalNames,
   });
-  return attachCapturedSceneAnnotations(scene, documents, captured);
+  return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
 }
 
 function attachCapturedSceneAnnotations(
   scene: VomScene,
   documents: VomFrameDocument[],
   captured: CapturedViewModel,
+  surfaceProbes: CapturedSurfaceProbe[],
 ): VomScene {
   const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
   const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
   const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
-  const surfaces = buildConditionalSurfaces(scene.nodes, captured);
+  const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
   return {
     ...scene,
     ...(surfaces.length > 0 ? { surfaces } : {}),
@@ -1081,20 +1018,71 @@ async function captureForVom(
   options: CaptureVomObservationOptions,
 ): Promise<CapturedViewModel> {
   try {
-    return await captureViewModel(cdp, tabId, {
+    return await captureViewModel(cdp, tabId, { signal: options.signal });
-      conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
-      hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
-      signal: options.signal,
-    });
   } catch (error) {
     if (isAbortError(error)) throw error;
     return fallbackCapturedViewModel(cdp, tabId, options.signal);
   }
 }
 
+export interface HoverProbeOutcome {
+  /** Whether any active hover was dispatched during this observation. */
+  performed: boolean;
+  /**
+   * Whether hovering surfaced content absent from the static snapshot. Cached
+   * tooltip names count, so this may over-report on repeat observations — it
+   * errs towards telling the caller the snapshot is less fresh than it looks.
+   */
+  revealedContent: boolean;
+  surfaceProbes: CapturedSurfaceProbe[];
+  tooltipNames: Map<string, string>;
+}
+
+const NO_HOVER_PROBES: HoverProbeOutcome = {
+  performed: false,
+  revealedContent: false,
+  surfaceProbes: [],
+  tooltipNames: new Map(),
+};
+
+/**
+ * Runs both active-hover chains back to back.
+ *
+ * Ordering matters: this happens after DOM *and* accessibility capture so a
+ * hover that opens a menu cannot leave one half of the observation describing
+ * the page before the change and the other half after it. Sharing one overlay
+ * bypass span also means the agent overlay toggles once per observation
+ * instead of once per chain.
+ */
+async function runHoverProbes(
+  cdp: CdpRunner,
+  tabId: number,
+  captured: CapturedViewModel,
+  documents: VomFrameDocument[],
+  staticSemantics: ReturnType<typeof resolveSemanticGraph>,
+  options: CaptureVomObservationOptions,
+): Promise<HoverProbeOutcome> {
+  if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
+
+  return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
+    const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
+      signal: options.signal,
+    });
+    throwIfAborted(options.signal, "observation");
+    const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
+      signal: options.signal,
+    });
+    return {
+      performed: true,
+      revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
+      surfaceProbes,
+      tooltipNames,
+    };
+  });
+}
+
 export interface CaptureVomObservationOptions extends VomOptions {
   conditionalSurfaceProbe?: boolean;
-  renderedSurfaceDiscovery?: boolean;
   hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
   signal?: AbortSignal;
 }
@@ -1123,33 +1111,27 @@ export async function captureVomObservation(
     excludedBackendNodeIds: captured.excludedBackendNodeIds,
   });
   const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
-  const tooltipNames = options.conditionalSurfaceProbe
+  const hoverProbes = await runHoverProbes(
-    ? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
+    cdp,
-        signal: options.signal,
+    tabId,
-        hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
+    captured,
-      })
+    normalizedDocuments,
-    : new Map<string, string>();
+    staticSemantics,
+    options,
+  );
   throwIfAborted(options.signal, "observation");
   const scene = projectSemanticGraph(
     normalizeSemanticStructure(
-      resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
+      resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
     ),
   );
-  const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
+  const decoratedScene = attachCapturedSceneAnnotations(
-  const observedScene = options.renderedSurfaceDiscovery
+    scene,
-    ? projectRenderedSurfaces(
+    normalizedDocuments,
-        decoratedScene,
+    captured,
-        normalizedDocuments,
+    hoverProbes.surfaceProbes,
-        clusterRenderedSurfaces(
+  );
-          discoverRenderedSurfaces(
+  const rendered = renderVom(decoratedScene, {
-            normalizedDocuments,
-            captured.viewport,
-            captured.excludedBackendNodeIds,
-          ),
-        ),
-      )
-    : decoratedScene;
-  const rendered = renderVom(observedScene, {
     maxDepth: options.maxDepth,
     maxTokens: options.maxTokens,
     redactValues: options.redactValues,
@@ -1160,7 +1142,11 @@ export async function captureVomObservation(
     rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
     frameDocuments: normalizedDocuments,
     rendered,
-    surfaceProbes: captured.surfaceProbes,
+    surfaceProbes: hoverProbes.surfaceProbes,
+    hoverProbe: {
+      performed: hoverProbes.performed,
+      revealedContent: hoverProbes.revealedContent,
+    },
   });
 }
 
@@ -1200,7 +1186,6 @@ async function handleVomObservation(
       maxTokens: params.max_tokens,
       activeRegionPolicy: true,
       conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
-      renderedSurfaceDiscovery: toolName === "observe",
       hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
       signal,
     });
@@ -1218,9 +1203,6 @@ async function handleVomObservation(
             tabId: target.tabId,
             ...(ref.frameId ? { frameId: ref.frameId } : {}),
             ...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
-            ...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
-            ...(ref.kind ? { kind: ref.kind } : {}),
-            ...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
           },
         ] as const;
       }),
@@ -1230,6 +1212,14 @@ async function handleVomObservation(
       ref_count: observation.refs.length,
       tab_id: target.tabId,
       truncated: observation.truncated,
+      ...(toolName === "observe" && observation.hoverProbe?.performed
+        ? {
+            hover_probe: {
+              performed: true,
+              revealed_content: observation.hoverProbe.revealedContent,
+            },
+          }
+        : {}),
       ...(toolName === "observe" && (params as ObserveParams).debug_surfaces
         ? {
             debug: {
@@ -1268,5 +1258,13 @@ export async function handleObserve(
   deps: SnapshotDeps = getDefaultDeps(),
   signal?: AbortSignal,
 ): Promise<ObserveResult | RpcError> {
-  return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
+  return handleVomObservation(
+    manager,
+    params,
+    "observe",
+    "transient_input",
+    params.probe_hover === true,
+    deps,
+    signal,
+  );
 }

apps/extension/src/tools/session.ts

@@ -243,7 +243,6 @@ export async function handleSessionStop(
 
   // Step 2: clear the per-session RefStore (review M6/M7 parity).
   ctx.refStore.clear();
-  ctx.surfaceCaptures.clear();
 
   clearRecordingForSession(params.session_id);
 

apps/extension/src/tools/snapshot-ref.ts

@@ -2,9 +2,8 @@
 // compound CDP node identity via the session RefStore, and build stable
 // `ref_not_found` errors for hard-failure tool paths.
 
-import type { Rect } from "@browser-skill/vom";
 import type { SessionContext } from "@/session-manager/manager";
-import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
+import { normaliseRef } from "@/session-manager/ref-store";
 import type { RpcError } from "@/transport/types";
 import { rpcError } from "./errors";
 
@@ -13,10 +12,6 @@ export interface SnapshotRefLookup {
   refKey: string;
   frameId?: string;
   cdpSessionId?: string;
-  visibleRect?: Rect;
-  kind: RefTargetKind;
-  capabilities: RefCapability[];
-  generation: number;
 }
 
 function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
@@ -42,10 +37,6 @@ export function lookupSnapshotRef(
     refKey,
     ...(entry.frameId ? { frameId: entry.frameId } : {}),
     ...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
-    ...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
-    kind: entry.kind,
-    capabilities: entry.capabilities,
-    generation: entry.generation,
   };
 }
 
@@ -58,7 +49,6 @@ export function resolveSnapshotRef(
   ctx: SessionContext,
   ref: string,
   tabId: number,
-  requiredCapability?: RefCapability,
 ): SnapshotRefLookup | RpcError {
   const looked = lookupSnapshotRef(ctx, ref, tabId);
   if (looked === null) {
@@ -68,13 +58,5 @@ export function resolveSnapshotRef(
       `ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
     );
   }
-  if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
-    return rpcError(
-      "permission_denied",
-      "ref_capability_denied",
-      `ref ${ref} does not support ${requiredCapability}`,
-      { ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
-    );
-  }
   return looked;
 }

apps/extension/src/tools/surface-coordinate.ts

@@ -1,88 +0,0 @@
-import type { Rect } from "@browser-skill/vom";
-import type { Point, Region, ViewportRect } from "./geometry";
-
-const RECT_EPSILON = 0.01;
-
-export function surfaceVisibleRect(
-  liveRect: ViewportRect,
-  observedRect: Rect | undefined,
-): Rect | null {
-  const observed = observedRect ?? {
-    x: liveRect.x,
-    y: liveRect.y,
-    w: liveRect.width,
-    h: liveRect.height,
-  };
-  const x = Math.max(liveRect.x, observed.x);
-  const y = Math.max(liveRect.y, observed.y);
-  const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w);
-  const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h);
-  return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
-}
-
-export function sameRect(a: Rect, b: Rect): boolean {
-  return (
-    Math.abs(a.x - b.x) <= RECT_EPSILON &&
-    Math.abs(a.y - b.y) <= RECT_EPSILON &&
-    Math.abs(a.w - b.w) <= RECT_EPSILON &&
-    Math.abs(a.h - b.h) <= RECT_EPSILON
-  );
-}
-
-export function mapImagePointToViewport(
-  rect: Rect,
-  imageWidth: number,
-  imageHeight: number,
-  imageX: number,
-  imageY: number,
-): Point | null {
-  if (
-    !Number.isFinite(imageX) ||
-    !Number.isFinite(imageY) ||
-    !Number.isSafeInteger(imageWidth) ||
-    !Number.isSafeInteger(imageHeight) ||
-    imageWidth <= 0 ||
-    imageHeight <= 0 ||
-    imageX < 0 ||
-    imageY < 0 ||
-    imageX >= imageWidth ||
-    imageY >= imageHeight ||
-    !Number.isFinite(rect.x) ||
-    !Number.isFinite(rect.y) ||
-    !Number.isFinite(rect.w) ||
-    !Number.isFinite(rect.h) ||
-    rect.w <= 0 ||
-    rect.h <= 0
-  ) {
-    return null;
-  }
-  return {
-    x: rect.x + (imageX / imageWidth) * rect.w,
-    y: rect.y + (imageY / imageHeight) * rect.h,
-  };
-}
-
-function pointInPolygon(point: Point, polygon: Point[]): boolean {
-  for (let index = 0; index < polygon.length; index += 1) {
-    const a = polygon[index];
-    const b = polygon[(index + 1) % polygon.length];
-    const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x);
-    const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9;
-    const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9;
-    if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true;
-  }
-  let inside = false;
-  for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) {
-    const a = polygon[index];
-    const b = polygon[previous];
-    const intersects =
-      a.y > point.y !== b.y > point.y &&
-      point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x;
-    if (intersects) inside = !inside;
-  }
-  return inside;
-}
-
-export function pointInRegion(point: Point, region: Region): boolean {
-  return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon));
-}

apps/extension/src/tools/surface-target.ts

@@ -1,260 +0,0 @@
-import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
-import type { SessionContext } from "@/session-manager/manager";
-import type { RpcError } from "@/transport/types";
-import { rpcError } from "./errors";
-import { resolveNodeGeometry } from "./frame-geometry";
-import { type CdpRunner, isRpcError, normaliseRef } from "./shared";
-import { resolveSnapshotRef } from "./snapshot-ref";
-import {
-  mapImagePointToViewport,
-  pointInRegion,
-  sameRect,
-  surfaceVisibleRect,
-} from "./surface-coordinate";
-
-export interface SurfaceCaptureEnvironment {
-  navigationIdentity: string;
-  viewportSignature: string;
-  frameProjectionSignature: string;
-}
-
-export interface SurfaceTargetResolverDeps {
-  cdp: CdpRunner;
-  signal?: AbortSignal;
-}
-
-export interface SurfacePointerTargetInput {
-  ref?: string;
-  selector?: string;
-  captureId?: string;
-  imageX?: number;
-  imageY?: number;
-}
-
-export interface ResolvedSurfacePointerTarget {
-  point: { x: number; y: number };
-  usedRef: string;
-  moveSettleMs: number;
-  capture: {
-    id: string;
-    imageX: number;
-    imageY: number;
-  };
-}
-
-// Canvas-style controls commonly update their pointer hit-test on the next
-// animation frame. Leave a scheduling boundary after moving and before pressing.
-export const SURFACE_POINTER_MOVE_SETTLE_MS = 32;
-
-function stale(message: string): RpcError {
-  return rpcError("permission_denied", "surface_capture_stale", message);
-}
-
-function finiteObject(value: unknown): unknown {
-  if (Array.isArray(value)) return value.map(finiteObject);
-  if (typeof value !== "object" || value === null) return value;
-  return Object.fromEntries(
-    Object.entries(value)
-      .filter(
-        ([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)),
-      )
-      .sort(([a], [b]) => a.localeCompare(b))
-      .map(([key, entry]) => [key, finiteObject(entry)]),
-  );
-}
-
-function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null {
-  if (!frameId) return "top";
-  const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
-  const path: Array<{
-    frameId: string;
-    parentFrameId?: string;
-    ownerBackendNodeId?: number;
-    targetSessionId?: string;
-  }> = [];
-  const seen = new Set<string>();
-  let current = frames.get(frameId);
-  while (current) {
-    if (seen.has(current.frameId)) return null;
-    seen.add(current.frameId);
-    path.push({
-      frameId: current.frameId,
-      ...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}),
-      ...(current.ownerBackendNodeId !== undefined
-        ? { ownerBackendNodeId: current.ownerBackendNodeId }
-        : {}),
-      ...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}),
-    });
-    if (!current.parentFrameId) return JSON.stringify(path);
-    const parent = frames.get(current.parentFrameId);
-    if (!parent) return null;
-    current = parent;
-  }
-  return null;
-}
-
-export async function captureSurfaceEnvironment(
-  cdp: CdpRunner,
-  tabId: number,
-  frameId: string | undefined,
-): Promise<SurfaceCaptureEnvironment | RpcError> {
-  try {
-    const [tree, metrics, graph] = await Promise.all([
-      cdp.send<{
-        frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } };
-      }>(tabId, "Page.getFrameTree", {}),
-      cdp.send<Record<string, unknown>>(tabId, "Page.getLayoutMetrics", {}),
-      frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null),
-    ]);
-    const root = tree.frameTree?.frame;
-    if (!root?.id || !root.loaderId) {
-      return { code: "cdp_failed", message: "could not identify the current navigation" };
-    }
-    const frameProjectionSignature = frameId
-      ? graph
-        ? framePathSignature(graph, frameId)
-        : null
-      : "top";
-    if (!frameProjectionSignature) {
-      return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` };
-    }
-    return {
-      navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }),
-      viewportSignature: JSON.stringify(
-        finiteObject({
-          cssLayoutViewport: metrics.cssLayoutViewport,
-          cssVisualViewport: metrics.cssVisualViewport,
-        }),
-      ),
-      frameProjectionSignature,
-    };
-  } catch (error) {
-    return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
-  }
-}
-
-function validatePointParams(input: SurfacePointerTargetInput): RpcError | null {
-  if (!input.captureId || typeof input.captureId !== "string") {
-    return { code: "invalid_params", message: "surface point click requires capture_id" };
-  }
-  if (typeof input.imageX !== "number" || !Number.isFinite(input.imageX)) {
-    return { code: "invalid_params", message: "image_x must be a finite number" };
-  }
-  if (typeof input.imageY !== "number" || !Number.isFinite(input.imageY)) {
-    return { code: "invalid_params", message: "image_y must be a finite number" };
-  }
-  if (!input.ref || input.selector) {
-    return {
-      code: "invalid_params",
-      message: "surface point click requires one Surface ref and does not accept a selector",
-    };
-  }
-  return null;
-}
-
-function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError {
-  const code = reason === "not_found" ? "not_found" : "permission_denied";
-  return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`);
-}
-
-function aborted(signal: AbortSignal | undefined): RpcError | null {
-  return signal?.aborted
-    ? { code: "cancelled", message: "surface target resolution aborted" }
-    : null;
-}
-
-/** Resolve one fresh Surface capture coordinate into a validated top-viewport pointer target. */
-export async function resolveSurfacePointerTarget(
-  ctx: SessionContext,
-  target: { tabId: number; url?: string },
-  input: SurfacePointerTargetInput,
-  deps: SurfaceTargetResolverDeps,
-): Promise<ResolvedSurfacePointerTarget | RpcError> {
-  const invalid = validatePointParams(input);
-  if (invalid) return invalid;
-  const earlyAbort = aborted(deps.signal);
-  if (earlyAbort) return earlyAbort;
-
-  const consumed = ctx.surfaceCaptures.consume(input.captureId as string);
-  if (!consumed.ok) return captureConsumeError(consumed.reason);
-  const capture = consumed.capture;
-  const postConsumeAbort = aborted(deps.signal);
-  if (postConsumeAbort) return postConsumeAbort;
-
-  if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) {
-    return stale("surface capture belongs to a different session or tab");
-  }
-  if (normaliseRef(input.ref as string) !== capture.surface.ref) {
-    return stale("surface ref does not match the capture");
-  }
-  const node = resolveSnapshotRef(ctx, input.ref as string, target.tabId, "screenshot");
-  if (isRpcError(node)) return stale(node.message);
-  if (
-    node.kind !== "surface" ||
-    node.backendNodeId !== capture.surface.backendNodeId ||
-    node.frameId !== capture.surface.frameId ||
-    node.generation !== capture.surface.observationGeneration
-  ) {
-    return stale("surface observation generation or node identity changed");
-  }
-
-  deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-  try {
-    await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
-  } catch (error) {
-    return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
-  }
-  const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId);
-  if (isRpcError(environment)) return environment;
-  if (
-    environment.navigationIdentity !== capture.navigationIdentity ||
-    environment.viewportSignature !== capture.viewportSignature ||
-    environment.frameProjectionSignature !== capture.frameProjectionSignature
-  ) {
-    return stale("navigation, viewport, zoom, scroll, or frame projection changed");
-  }
-
-  const geometry = await resolveNodeGeometry(
-    deps.cdp,
-    target.tabId,
-    {
-      target: {
-        tabId: target.tabId,
-        ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
-      },
-      backendNodeId: node.backendNodeId,
-      ...(node.frameId ? { frameId: node.frameId } : {}),
-    },
-    { scrollIntoView: false },
-  );
-  if (isRpcError(geometry)) return stale(geometry.message);
-  const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect);
-  if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) {
-    return stale("surface visible region changed");
-  }
-  const point = mapImagePointToViewport(
-    capture.topViewportRect,
-    capture.imageWidth,
-    capture.imageHeight,
-    input.imageX as number,
-    input.imageY as number,
-  );
-  if (!point || !pointInRegion(point, geometry.topVisibleRegions)) {
-    return rpcError(
-      "invalid_params",
-      "surface_coordinate_invalid",
-      "image coordinate is outside the captured Surface",
-    );
-  }
-
-  return {
-    point,
-    usedRef: capture.surface.ref,
-    moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
-    capture: {
-      id: capture.id,
-      imageX: input.imageX as number,
-      imageY: input.imageY as number,
-    },
-  };
-}

apps/extension/src/tools/text-input.ts

@@ -1,194 +0,0 @@
-// Browser-native text entry for live editing sessions. Unlike fill, this
-// module does not assume that the focused editable retains a DOM value: a
-// Canvas editor may use an input/textarea only as a transient keyboard/IME
-// sink and clear it after consuming each event.
-
-import type { RpcError } from "@/transport/types";
-import type { CdpRunner } from "./shared";
-
-function cancelled(signal: AbortSignal | undefined): RpcError | null {
-  return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null;
-}
-
-interface KeyDescriptor {
-  key: string;
-  code: string;
-  windowsVirtualKeyCode: number;
-  nativeVirtualKeyCode?: number;
-  modifiers: number;
-}
-
-function asciiKeyDescriptor(character: string): KeyDescriptor {
-  if (/^[a-z]$/.test(character)) {
-    return {
-      key: character,
-      code: `Key${character.toUpperCase()}`,
-      windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0),
-      modifiers: 0,
-    };
-  }
-  if (/^[A-Z]$/.test(character)) {
-    return {
-      key: character,
-      code: `Key${character}`,
-      windowsVirtualKeyCode: character.charCodeAt(0),
-      modifiers: 8,
-    };
-  }
-  if (/^[0-9]$/.test(character)) {
-    return {
-      key: character,
-      code: `Digit${character}`,
-      windowsVirtualKeyCode: character.charCodeAt(0),
-      modifiers: 0,
-    };
-  }
-  if (character === " ") {
-    return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 };
-  }
-  return {
-    key: character,
-    code: "",
-    windowsVirtualKeyCode: character.charCodeAt(0),
-    modifiers: 0,
-  };
-}
-
-/**
- * Run one complete browser key transaction and always release the key. The
- * commit callback owns the text-producing phase: a `char` event for a
- * keyboard character, or `Input.insertText` for committed IME text.
- */
-async function dispatchKeyTransaction(
-  cdp: CdpRunner,
-  tabId: number,
-  descriptor: KeyDescriptor,
-  commit: () => Promise<void>,
-  signal?: AbortSignal,
-): Promise<RpcError | null> {
-  let keyDown = false;
-  try {
-    await cdp.send(tabId, "Input.dispatchKeyEvent", {
-      type: "rawKeyDown",
-      ...descriptor,
-    });
-    keyDown = true;
-    const abortedAfterDown = cancelled(signal);
-    if (!abortedAfterDown) await commit();
-    await cdp.send(tabId, "Input.dispatchKeyEvent", {
-      type: "keyUp",
-      ...descriptor,
-    });
-    keyDown = false;
-    return abortedAfterDown ?? cancelled(signal);
-  } catch (err) {
-    if (keyDown) {
-      try {
-        await cdp.send(tabId, "Input.dispatchKeyEvent", {
-          type: "keyUp",
-          ...descriptor,
-        });
-      } catch {
-        // Best-effort cleanup keeps the page from observing a stuck key.
-      }
-    }
-    throw err;
-  }
-}
-
-async function typeAsciiCharacter(
-  cdp: CdpRunner,
-  tabId: number,
-  character: string,
-  signal?: AbortSignal,
-): Promise<RpcError | null> {
-  const descriptor = asciiKeyDescriptor(character);
-  return dispatchKeyTransaction(
-    cdp,
-    tabId,
-    descriptor,
-    async () => {
-      await cdp.send(tabId, "Input.dispatchKeyEvent", {
-        type: "char",
-        ...descriptor,
-        text: character,
-        unmodifiedText: character,
-      });
-    },
-    signal,
-  );
-}
-
-async function typeImeSegment(
-  cdp: CdpRunner,
-  tabId: number,
-  text: string,
-  signal?: AbortSignal,
-): Promise<RpcError | null> {
-  const abortedBefore = cancelled(signal);
-  if (abortedBefore) return abortedBefore;
-
-  // A committed IME value is still part of a keyboard transaction. The
-  // Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an
-  // IME owns the keystroke. It lets selection-based editors enter text mode
-  // before insertText commits the Unicode value, without inventing a visible
-  // ASCII prefix or leaving a composition transaction open.
-  const processKey: KeyDescriptor = {
-    key: "Process",
-    code: "",
-    windowsVirtualKeyCode: 229,
-    nativeVirtualKeyCode: 229,
-    modifiers: 0,
-  };
-  return dispatchKeyTransaction(
-    cdp,
-    tabId,
-    processKey,
-    () => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined),
-    signal,
-  );
-}
-
-/**
- * Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are
- * delivered as physical-key-shaped events; non-ASCII runs use CDP's committed
- * IME/software-keyboard insertion path. Successful return means the browser
- * accepted the input commands, not that a DOM value retained the text.
- */
-export async function dispatchTextInput(
-  cdp: CdpRunner,
-  tabId: number,
-  text: string,
-  signal?: AbortSignal,
-): Promise<RpcError | null> {
-  const abortedBefore = cancelled(signal);
-  if (abortedBefore) return abortedBefore;
-
-  const segments: Array<{ ime: boolean; text: string }> = [];
-  for (const character of text) {
-    const ime = character.codePointAt(0)! > 0x7f;
-    const previous = segments.at(-1);
-    if (previous?.ime === ime) previous.text += character;
-    else segments.push({ ime, text: character });
-  }
-
-  try {
-    for (const segment of segments) {
-      if (segment.ime) {
-        const error = await typeImeSegment(cdp, tabId, segment.text, signal);
-        if (error) return error;
-        continue;
-      }
-      for (const character of segment.text) {
-        const error = await typeAsciiCharacter(cdp, tabId, character, signal);
-        if (error) return error;
-      }
-    }
-    return null;
-  } catch (err) {
-    return {
-      code: "cdp_failed",
-      message: err instanceof Error ? err.message : String(err),
-    };
-  }
-}

apps/extension/src/tools/vom/__tests__/capture.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it, vi } from "vitest";
 import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
-import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
+import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
 
 // Minimal but format-accurate captureSnapshot reply: a body with one
 // fixed full-screen overlay div carrying a password input.
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
     expect(input?.attrs).toEqual({ type: "password" });
   });
 
-  it("does not run conditional surface probes by default", async () => {
+  it("never hovers the page while capturing", async () => {
     const cdp = makeCdp(fakeSnapshotReply());
-    const result = await captureViewModel(cdp, 4);
+    await captureViewModel(cdp, 4);
 
-    expect(result.surfaceProbes).toEqual([]);
     expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
   });
 
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
       }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
     };
 
-    const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
+    const captured = await captureViewModel(cdp, 4);
+    const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
 
-    expect(result.surfaceProbes).toEqual([
+    expect(surfaceProbes).toEqual([
       {
         triggerBackendNodeId: 12,
         triggerPoint: { x: 956, y: 32 },
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
       }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
     };
 
-    const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
+    const captured = await captureViewModel(cdp, 4);
+    const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
 
-    expect(result.surfaceProbes).toEqual([
+    expect(surfaceProbes).toEqual([
       expect.objectContaining({
         triggerBackendNodeId: 12,
         subItems: ["My profile", "Sign out"],
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
       }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
     };
 
-    await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
+    const captured = await captureViewModel(cdp, 4);
+    await probeHoverSurfaces(cdp, 4, captured.nodes);
 
     expect(hoverMoves).toBe(1);
   });
@@ -722,44 +724,6 @@ describe("captureViewModel", () => {
     expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
   });
 
-  it("propagates fully transparent ancestor suppression to painted descendants", async () => {
-    const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
-    const i = (value: string) => S.indexOf(value);
-    const snapshot = {
-      strings: S,
-      documents: [
-        {
-          nodes: {
-            parentIndex: [-1, 0, 1, 2],
-            nodeName: [i("html"), i("body"), i("div"), i("canvas")],
-            backendNodeId: [10, 11, 12, 13],
-            attributes: [[], [], [], []],
-          },
-          layout: {
-            nodeIndex: [1, 2, 3],
-            styles: [
-              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
-              [i("static"), i("auto"), i("auto"), i("visible"), i("0")],
-              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
-            ],
-            bounds: [
-              [0, 0, 1000, 800],
-              [10, 10, 200, 100],
-              [10, 10, 200, 100],
-            ],
-            paintOrders: [0, 1, 2],
-          },
-        },
-      ],
-    };
-
-    const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
-    const canvas = nodes.find((node) => node.backendNodeId === 13);
-
-    expect(canvas?.rendered).toBe(true);
-    expect(canvas?.visuallySuppressed).toBe(true);
-  });
-
   it("subtracts scroll offset to make rects viewport-relative", async () => {
     const cdp = {
       send: vi.fn(async (_t: number, method: string) => {

apps/extension/src/tools/vom/__tests__/hover-perception.test.ts

@@ -0,0 +1,84 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
+
+afterEach(() => {
+  resetOverlayBypassDepth();
+});
+
+describe("ProbeBudget", () => {
+  it("refuses a candidate that cannot finish inside the budget", () => {
+    const budget = new ProbeBudget(500);
+
+    expect(budget.canAfford(400)).toBe(true);
+    expect(budget.canAfford(600)).toBe(false);
+  });
+
+  it("stops admitting candidates once the remaining budget is smaller than one", async () => {
+    const budget = new ProbeBudget(60);
+    await new Promise((resolve) => setTimeout(resolve, 40));
+
+    // A top-of-loop `elapsed > total` check would still admit this candidate
+    // and then overshoot by its full cost.
+    expect(budget.canAfford(50)).toBe(false);
+  });
+});
+
+describe("withOverlayBypass", () => {
+  it("toggles the overlay once around nested probe phases", async () => {
+    const bypass = vi.fn(async () => {});
+
+    await withOverlayBypass(bypass, 4, async () => {
+      await withOverlayBypass(bypass, 4, async () => {
+        expect(bypass).toHaveBeenCalledTimes(1);
+      });
+      expect(bypass).toHaveBeenCalledTimes(1);
+    });
+
+    expect(bypass.mock.calls).toEqual([
+      [4, true],
+      [4, false],
+    ]);
+  });
+
+  it("restores the overlay when the probe throws", async () => {
+    const bypass = vi.fn(async () => {});
+
+    await expect(
+      withOverlayBypass(bypass, 4, async () => {
+        throw new Error("probe exploded");
+      }),
+    ).rejects.toThrow("probe exploded");
+
+    expect(bypass).toHaveBeenLastCalledWith(4, false);
+  });
+
+  it("reports a failed restore instead of swallowing it", async () => {
+    const onRestoreFailure = vi.fn();
+    const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
+      if (!enabled) throw new Error("tab is gone");
+    });
+
+    await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
+
+    expect(onRestoreFailure).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not pin the overlay off for the tab when a restore fails", async () => {
+    const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
+      if (!enabled) throw new Error("tab is gone");
+    });
+    await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
+
+    const bypass = vi.fn(async () => {});
+    await withOverlayBypass(bypass, 4, async () => undefined);
+
+    expect(bypass.mock.calls).toEqual([
+      [4, true],
+      [4, false],
+    ]);
+  });
+
+  it("runs the probe untouched when no bypass is wired up", async () => {
+    await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
+  });
+});

apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts

@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
     expect(cdp.sendToTarget).not.toHaveBeenCalled();
   });
 
+  it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
+    const cdp: CdpRunner = {
+      send: vi.fn(async () => ({})) as CdpRunner["send"],
+      sendToTarget: vi.fn(async (_target, method) => {
+        if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
+        if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
+        if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
+        return {};
+      }) as CdpRunner["sendToTarget"],
+    };
+
+    const documents = [
+      document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
+    ];
+    const graph = semantics(documents);
+
+    const first = await probeTooltipNames(cdp, 11, documents, graph);
+    expect(first.size).toBe(0);
+    const hoversAfterFirst = vi
+      .mocked(cdp.send)
+      .mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
+    expect(hoversAfterFirst).toBeGreaterThan(0);
+
+    const second = await probeTooltipNames(cdp, 11, documents, graph);
+    expect(second.size).toBe(0);
+    expect(
+      vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
+        .length,
+    ).toBe(hoversAfterFirst);
+  });
+
   it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
     let hoveredX = -10;
     const cdp: CdpRunner = {

apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts

@@ -1,256 +0,0 @@
-import type { VomScene } from "@browser-skill/vom";
-import { describe, expect, it } from "vitest";
-import type { CapturedNode } from "../capture";
-import type { CapturedFrameDocument } from "../frame-capture";
-import {
-  clusterRenderedSurfaces,
-  discoverRenderedSurfaces,
-  projectRenderedSurfaces,
-} from "../rendered-surfaces";
-import type { SemanticAxNode } from "../semantic-graph";
-
-function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
-  return {
-    backendNodeId,
-    parentBackendNodeId: null,
-    frameId: "main",
-    ownerFrameBackendNodeId: null,
-    tag: "canvas",
-    attrs: {},
-    rect: { x: 10, y: 20, w: 800, h: 500 },
-    localRect: { x: 10, y: 20, w: 800, h: 500 },
-    paintOrder: 1,
-    position: "static",
-    pointerEvents: "auto",
-    rendered: true,
-    ...overrides,
-  };
-}
-
-function document(
-  domNodes: CapturedNode[],
-  axNodes: SemanticAxNode[] = [],
-): CapturedFrameDocument<SemanticAxNode> {
-  return {
-    frameId: domNodes[0]?.frameId ?? "main",
-    contextScopeId: domNodes[0]?.frameId ?? "main",
-    target: { tabId: 4 },
-    domNodes,
-    axNodes,
-  };
-}
-
-describe("rendered surface discovery", () => {
-  it("finds prominent rendered canvases and excludes hidden descendants", () => {
-    const visible = domNode(2, { parentBackendNodeId: 1 });
-    const hidden = domNode(3, { parentBackendNodeId: 4 });
-    const surfaces = discoverRenderedSurfaces(
-      [
-        document([
-          domNode(1, { tag: "main", rect: null, localRect: null }),
-          visible,
-          domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
-          hidden,
-        ]),
-      ],
-      { width: 1280, height: 720 },
-    );
-
-    expect(surfaces).toHaveLength(1);
-    expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
-  });
-
-  it("keeps tiny canvases as individually addressable exact discoveries", () => {
-    const surfaces = discoverRenderedSurfaces(
-      [document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
-      { width: 1280, height: 720 },
-    );
-    expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
-  });
-
-  it("keeps a surface when the canvas only has partial native accessibility content", () => {
-    const axNodes = [
-      {
-        nodeId: "canvas",
-        backendDOMNodeId: 2,
-        role: { type: "role", value: "presentation" },
-        childIds: ["button"],
-        frameId: "main",
-      },
-      {
-        nodeId: "button",
-        backendDOMNodeId: 3,
-        role: { type: "role", value: "button" },
-        frameId: "main",
-      },
-    ] as SemanticAxNode[];
-    expect(
-      discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
-    ).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
-  });
-
-  it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
-    const canvas = domNode(2, {
-      attrs: {
-        id: "must-not-be-a-label",
-        class: "must-not-be-a-label",
-        "data-testid": "must-not-be-a-label",
-      },
-    });
-
-    expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
-      expect.not.objectContaining({
-        label: expect.anything(),
-      }),
-    ]);
-  });
-
-  it("treats blank explicit Canvas names as unnamed", () => {
-    expect(
-      discoverRenderedSurfaces(
-        [document([domNode(2, { attrs: { "aria-label": "   ", title: "\n\t" } })])],
-        { width: 1280, height: 720 },
-      ),
-    ).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
-  });
-
-  it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
-    const owner = domNode(1, { tag: "div", rect: null, localRect: null });
-    const canvas = domNode(2, { parentBackendNodeId: 1 });
-
-    expect(
-      discoverRenderedSurfaces(
-        [document([owner, canvas])],
-        { width: 1280, height: 720 },
-        new Set([1]),
-      ),
-    ).toEqual([]);
-  });
-
-  it("excludes canvases suppressed by a fully transparent ancestor", () => {
-    const owner = domNode(1, {
-      tag: "div",
-      rect: { x: 0, y: 0, w: 900, h: 600 },
-      visuallySuppressed: true,
-    });
-    const canvas = domNode(2, { parentBackendNodeId: 1 });
-
-    expect(
-      discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
-    ).toEqual([]);
-  });
-
-  it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
-    const owner = domNode(1, {
-      tag: "div",
-      rect: { x: 10, y: 20, w: 100, h: 70 },
-      overflowX: "hidden",
-      overflowY: "hidden",
-    });
-    const partial = domNode(2, {
-      parentBackendNodeId: 1,
-      rect: { x: 55, y: 30, w: 140, h: 60 },
-    });
-    const fullyClipped = domNode(3, {
-      parentBackendNodeId: 1,
-      rect: { x: 140, y: 30, w: 80, h: 50 },
-    });
-
-    expect(
-      discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
-        width: 1280,
-        height: 720,
-      }),
-    ).toEqual([
-      expect.objectContaining({
-        backendNodeId: 2,
-        visibleRect: { x: 55, y: 30, w: 55, h: 60 },
-      }),
-    ]);
-  });
-});
-
-describe("rendered surface clustering and projection", () => {
-  it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
-    const body = domNode(1, { tag: "body", rect: null, localRect: null });
-    const surfaces = discoverRenderedSurfaces(
-      [
-        document([
-          body,
-          domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
-          domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
-        ]),
-      ],
-      { width: 1280, height: 720 },
-    );
-    const groups = clusterRenderedSurfaces(surfaces);
-    const scene: VomScene = {
-      viewport: { width: 1280, height: 720 },
-      nodes: [
-        {
-          id: 10,
-          parentId: null,
-          backendNodeId: 1,
-          frameId: "main",
-          tag: "body",
-          role: "RootWebArea",
-          rect: null,
-          paintOrder: 0,
-          position: "static",
-          pointerEvents: "auto",
-        },
-      ],
-    };
-
-    expect(groups).toHaveLength(1);
-    expect(groups[0]?.members).toHaveLength(2);
-    expect(groups[0]?.label).toBeUndefined();
-    expect(
-      projectRenderedSurfaces(
-        scene,
-        [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
-        groups,
-      ),
-    ).toMatchObject({
-      visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
-    });
-  });
-
-  it("does not merge a contained canvas with a larger independent canvas", () => {
-    const groups = clusterRenderedSurfaces([
-      ...discoverRenderedSurfaces(
-        [
-          document([
-            domNode(1, { tag: "main", rect: null, localRect: null }),
-            domNode(2, { parentBackendNodeId: 1 }),
-            domNode(3, {
-              parentBackendNodeId: 1,
-              rect: { x: 30, y: 40, w: 200, h: 100 },
-            }),
-          ]),
-        ],
-        { width: 1280, height: 720 },
-      ),
-    ]);
-
-    expect(groups).toHaveLength(2);
-  });
-
-  it("does not merge coincident canvases owned by different containers", () => {
-    const groups = clusterRenderedSurfaces(
-      discoverRenderedSurfaces(
-        [
-          document([
-            domNode(1, { tag: "section", rect: null, localRect: null }),
-            domNode(2, { tag: "section", rect: null, localRect: null }),
-            domNode(3, { parentBackendNodeId: 1 }),
-            domNode(4, { parentBackendNodeId: 2 }),
-          ]),
-        ],
-        { width: 1280, height: 720 },
-      ),
-    );
-
-    expect(groups).toHaveLength(2);
-  });
-});

apps/extension/src/tools/vom/capture.ts

@@ -9,17 +9,9 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
 import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
 import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
 import type { CdpRunner } from "../shared";
-import { clearHover, waitForHover } from "./hover-perception";
+import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
 
-const REQUESTED_STYLES = [
+const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
-  "position",
-  "pointer-events",
-  "cursor",
-  "visibility",
-  "opacity",
-  "overflow-x",
-  "overflow-y",
-] as const;
 const STYLE_COL = Object.fromEntries(
   REQUESTED_STYLES.map((name, index) => [name, index]),
 ) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -47,10 +39,6 @@ export interface CapturedNode {
    * `textContent`: the live parser always sets it, hand-built fixtures may not.
    */
   cursor?: string;
-  overflowX?: string;
-  overflowY?: string;
-  /** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
-  visuallySuppressed?: boolean;
   /**
    * Whether the live DOM snapshot provides a painted, non-hidden box for this
    * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
@@ -83,14 +71,11 @@ export interface CapturedViewModel {
   /** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
   frameParentIds?: Map<string, string>;
   rootFrameId?: string;
-  surfaceProbes?: CapturedSurfaceProbe[];
   /** Backend node ids belonging to the agent overlay host + its shadow subtree. */
   excludedBackendNodeIds: Set<number>;
 }
 
 export interface CaptureViewModelOptions {
-  conditionalSurfaceProbe?: boolean;
-  hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
   signal?: AbortSignal;
 }
 
@@ -380,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
   }
 }
 
-const MAX_HOVER_PROBE_MS = 2_000;
+/**
+ * Ceiling for the whole hover-surface phase.
+ *
+ * The previous 2000 was only a floor: the loop checked elapsed time at the top,
+ * so a candidate could start with 1ms left and still run its two settle
+ * windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
+ * enforced by an up-front affordability check, so the same number of candidates
+ * get probed under a bound that no longer lies.
+ */
+const MAX_HOVER_PROBE_MS = 2_600;
 const MAX_HOVER_TRIGGERS = 6;
 const MAX_HOVER_SURFACES = 3;
 const HOVER_SETTLE_MS = 300;
@@ -626,13 +620,35 @@ function confidenceForHover(
   return "low";
 }
 
-async function probeHoverSurfaces(
+/**
+ * One candidate costs two settle windows (baseline + post-hover) plus a few
+ * CDP round trips. Used to decide whether the next candidate still fits the
+ * budget before paying for it.
+ */
+const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
+
+export interface HoverSurfaceProbeOptions {
+  signal?: AbortSignal;
+}
+
+/**
+ * Hovers a bounded set of likely menu triggers and reports the sub-items each
+ * one reveals.
+ *
+ * Runs after DOM *and* accessibility capture. Hovering can open menus and
+ * change layout, so probing between the two captures would leave the DOM half
+ * of an observation describing the page before the change and the AX half
+ * describing it after.
+ *
+ * The caller owns the overlay bypass span (see `withOverlayBypass`).
+ */
+export async function probeHoverSurfaces(
   cdp: CdpRunner,
   tabId: number,
   nodes: CapturedNode[],
-  options: CaptureViewModelOptions,
+  options: HoverSurfaceProbeOptions = {},
 ): Promise<CapturedSurfaceProbe[]> {
-  const started = Date.now();
+  const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
   try {
     throwIfAborted(options.signal);
     const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
@@ -647,60 +663,54 @@ async function probeHoverSurfaces(
     const results: CapturedSurfaceProbe[] = [];
     const seen = new Set<number>();
     throwIfAborted(options.signal);
-    await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
+    for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
-    try {
       throwIfAborted(options.signal);
-      for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
+      if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
+      if (results.length >= MAX_HOVER_SURFACES) break;
+      if (seen.has(candidate.backendNodeId)) continue;
+      try {
+        await clearHover(cdp, tabId);
         throwIfAborted(options.signal);
-        if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
+        await waitForHover(HOVER_SETTLE_MS, options.signal);
-        if (results.length >= MAX_HOVER_SURFACES) break;
+        const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
-        if (seen.has(candidate.backendNodeId)) continue;
+          expression: hoverStateExpression(),
-        try {
+          returnByValue: true,
-          await clearHover(cdp, tabId);
+        });
-          throwIfAborted(options.signal);
+        throwIfAborted(options.signal);
-          await waitForHover(HOVER_SETTLE_MS, options.signal);
+        const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
-          const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
-            expression: hoverStateExpression(),
-            returnByValue: true,
-          });
-          throwIfAborted(options.signal);
-          const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
 
-          throwIfAborted(options.signal);
+        throwIfAborted(options.signal);
-          await cdp.send(tabId, "Input.dispatchMouseEvent", {
+        await cdp.send(tabId, "Input.dispatchMouseEvent", {
-            type: "mouseMoved",
+          type: "mouseMoved",
-            x: candidate.x,
+          x: candidate.x,
-            y: candidate.y,
+          y: candidate.y,
-          });
+        });
-          throwIfAborted(options.signal);
+        throwIfAborted(options.signal);
-          await waitForHover(HOVER_SETTLE_MS, options.signal);
+        await waitForHover(HOVER_SETTLE_MS, options.signal);
-          const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
+        const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
-            expression: hoverStateExpression(),
+          expression: hoverStateExpression(),
-            returnByValue: true,
+          returnByValue: true,
-          });
+        });
-          throwIfAborted(options.signal);
+        throwIfAborted(options.signal);
-          const subItems = diffHoverItems(
+        const subItems = diffHoverItems(
-            baselineItems,
+          baselineItems,
-            runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
+          runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
-          );
+        );
-          if (subItems.length === 0) continue;
+        if (subItems.length === 0) continue;
-          seen.add(candidate.backendNodeId);
+        seen.add(candidate.backendNodeId);
-          results.push({
+        results.push({
-            triggerBackendNodeId: candidate.backendNodeId,
+          triggerBackendNodeId: candidate.backendNodeId,
-            triggerPoint: { x: candidate.x, y: candidate.y },
+          triggerPoint: { x: candidate.x, y: candidate.y },
-            triggerAction: "hover",
+          triggerAction: "hover",
-            subItems,
+          subItems,
-            confidence: confidenceForHover(candidate, subItems),
+          confidence: confidenceForHover(candidate, subItems),
-          });
+        });
-        } catch (error) {
+      } catch (error) {
-          if (isAbortError(error)) throw error;
+        if (isAbortError(error)) throw error;
-          continue;
+        continue;
-        } finally {
+      } finally {
-          await clearHover(cdp, tabId);
+        await clearHover(cdp, tabId);
-        }
       }
-    } finally {
-      await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
     }
     return results;
   } catch (err) {
@@ -808,8 +818,6 @@ function parseDocumentNodes(
   const cursorCol = STYLE_COL.cursor;
   const visibilityCol = STYLE_COL.visibility;
   const opacityCol = STYLE_COL.opacity;
-  const overflowXCol = STYLE_COL["overflow-x"];
-  const overflowYCol = STYLE_COL["overflow-y"];
   const inputValues = sparseIndexMap(dn.inputValue);
   const textValues = sparseIndexMap(dn.textValue);
   const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -835,7 +843,6 @@ function parseDocumentNodes(
 
   // Build CapturedNodes.
   const nodes: CapturedNode[] = [];
-  const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
   for (let n = 0; n < count; n++) {
     if (excluded[n]) continue;
     const backendNodeId = dn.backendNodeId[n];
@@ -855,11 +862,7 @@ function parseDocumentNodes(
     let position = "static";
     let pointerEvents = "auto";
     let cursor = "auto";
-    let overflowX = "visible";
-    let overflowY = "visible";
     let rendered = false;
-    let visuallySuppressed =
-      parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
     const li = layoutByNode.get(n);
     if (li !== undefined) {
       const b = dl?.bounds?.[li];
@@ -882,19 +885,12 @@ function parseDocumentNodes(
       cursor = str(strings, styleRow[cursorCol]) || "auto";
       const visibility = str(strings, styleRow[visibilityCol]) || "visible";
       const opacity = str(strings, styleRow[opacityCol]) || "1";
-      overflowX = str(strings, styleRow[overflowXCol]) || "visible";
-      overflowY = str(strings, styleRow[overflowYCol]) || "visible";
-      visuallySuppressed ||=
-        visibility === "hidden" ||
-        visibility === "collapse" ||
-        (Number.parseFloat(opacity) || 0) <= 0;
       rendered =
         localRect !== null &&
         visibility !== "hidden" &&
         visibility !== "collapse" &&
         (Number.parseFloat(opacity) || 0) > 0;
     }
-    visuallySuppressedByNodeIndex[n] = visuallySuppressed;
 
     // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
     // geometry and are never queried by callers.
@@ -929,9 +925,6 @@ function parseDocumentNodes(
       position,
       pointerEvents,
       cursor,
-      overflowX,
-      overflowY,
-      visuallySuppressed,
       rendered,
       textContent,
       ...(formValue !== undefined ? { formValue } : {}),
@@ -1071,7 +1064,6 @@ export async function captureViewModel(
       nodes: [],
       viewport,
       iframeNodes: new Map(),
-      surfaceProbes: [],
       excludedBackendNodeIds: new Set(),
     };
   }
@@ -1100,11 +1092,6 @@ export async function captureViewModel(
   await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
   throwIfAborted(options.signal);
 
-  const surfaceProbes = options.conditionalSurfaceProbe
-    ? await probeHoverSurfaces(cdp, tabId, nodes, options)
-    : [];
-  throwIfAborted(options.signal);
-
   const frameNodes = new Map<string, CapturedNode[]>();
   if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
   for (const iframeFrameNodes of iframeNodes.values()) {
@@ -1125,7 +1112,6 @@ export async function captureViewModel(
     frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
     frameParentIds: frameParsed.frameParentIds,
     ...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
-    surfaceProbes,
     excludedBackendNodeIds,
   };
 }

apps/extension/src/tools/vom/hover-perception.ts

@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
       }),
     );
 }
+
+/**
+ * Wall-clock budget for a probe phase.
+ *
+ * Callers ask whether the *next* candidate still fits before starting it,
+ * rather than checking elapsed time at the top of the loop. A top-of-loop
+ * check lets a candidate start with 1ms of budget left and then run to
+ * completion, overshooting by a full settle window.
+ */
+export class ProbeBudget {
+  private readonly startedAt = Date.now();
+
+  constructor(private readonly totalMs: number) {}
+
+  get elapsedMs(): number {
+    return Date.now() - this.startedAt;
+  }
+
+  /** True when a candidate costing `estimatedMs` still fits in the budget. */
+  canAfford(estimatedMs: number): boolean {
+    return this.elapsedMs + estimatedMs <= this.totalMs;
+  }
+}
+
+export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
+
+export interface OverlayBypassScopeOptions {
+  /**
+   * Called when the overlay could not be restored. Defaults to `console.error`
+   * so a stuck-bypassed overlay is never silent; tests inject a spy.
+   */
+  onRestoreFailure?: (error: unknown) => void;
+}
+
+const overlayBypassDepth = new Map<number, number>();
+
+/**
+ * Runs `body` with the agent overlay bypassed, restoring it afterwards.
+ *
+ * Reference counted per tab so nested probe phases share a single bypass span
+ * instead of toggling the overlay once per phase. Restore failures are
+ * reported rather than swallowed, and the depth is released even when restore
+ * throws so a failure cannot pin the overlay off for the tab's lifetime.
+ */
+export async function withOverlayBypass<T>(
+  bypass: OverlayBypass | undefined,
+  tabId: number,
+  body: () => Promise<T>,
+  options: OverlayBypassScopeOptions = {},
+): Promise<T> {
+  if (!bypass) return body();
+
+  const depth = overlayBypassDepth.get(tabId) ?? 0;
+  overlayBypassDepth.set(tabId, depth + 1);
+  if (depth === 0) {
+    try {
+      await bypass(tabId, true);
+    } catch (error) {
+      // Failing to hide the overlay only risks probing our own UI, so continue.
+      console.debug("[bsk hover] overlay bypass could not be enabled", error);
+    }
+  }
+
+  try {
+    return await body();
+  } finally {
+    const current = overlayBypassDepth.get(tabId) ?? 1;
+    if (current <= 1) {
+      overlayBypassDepth.delete(tabId);
+      try {
+        await bypass(tabId, false);
+      } catch (error) {
+        (options.onRestoreFailure ?? defaultRestoreFailure)(error);
+      }
+    } else {
+      overlayBypassDepth.set(tabId, current - 1);
+    }
+  }
+}
+
+function defaultRestoreFailure(error: unknown): void {
+  console.error("[bsk hover] agent overlay could not be restored after probing", error);
+}
+
+/** Test seam: drops any bypass depth left behind by a crashed probe. */
+export function resetOverlayBypassDepth(): void {
+  overlayBypassDepth.clear();
+}

apps/extension/src/tools/vom/name-enrichment.ts

@@ -1,7 +1,7 @@
 import { isVomReferenceNode } from "@browser-skill/vom";
 import { type CdpRunner, sendToCdpTarget } from "../shared";
 import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
-import { clearHover, waitForHover } from "./hover-perception";
+import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
 import { stableIdentifierName } from "./semantic-graph/name-evidence";
 import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
 
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
 const MAX_TOOLTIP_PROBE_MS = 2_400;
 const TOOLTIP_SETTLE_MS = 250;
 const CACHE_TTL_MS = 5 * 60_000;
+/**
+ * Misses expire sooner than hits: a control that gains a tooltip once the app
+ * finishes booting should still be discoverable, but not at the cost of
+ * re-probing every unlabelled control on every observation.
+ */
+const MISS_CACHE_TTL_MS = 60_000;
 const MAX_CACHE_ENTRIES = 500;
 const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
 
 interface TooltipCacheEntry {
-  name: string;
+  /** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
+  name: string | null;
   expiresAt: number;
 }
 
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
 
 export interface TooltipNameProbeOptions {
   signal?: AbortSignal;
-  hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
 }
 
 const tooltipCache = new Map<string, TooltipCacheEntry>();
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
   return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
 }
 
-function cachedName<T extends FrameOwnedAxNode>(
+/** Returns the cache entry, or `undefined` when this control was never probed. */
+function cachedEntry<T extends FrameOwnedAxNode>(
   document: FrameDocument<T>,
   backendNodeId: number,
-): string | undefined {
+): TooltipCacheEntry | undefined {
   const key = cacheKey(document, backendNodeId);
   const entry = tooltipCache.get(key);
   if (!entry) return undefined;
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
     tooltipCache.delete(key);
     return undefined;
   }
-  return entry.name;
+  return entry;
 }
 
 function storeName<T extends FrameOwnedAxNode>(
   document: FrameDocument<T>,
   backendNodeId: number,
-  name: string,
+  name: string | null,
 ): void {
   if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
     const oldest = tooltipCache.keys().next().value;
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
   }
   tooltipCache.set(cacheKey(document, backendNodeId), {
     name,
-    expiresAt: Date.now() + CACHE_TTL_MS,
+    expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
   });
 }
 
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
   const names = new Map<string, string>();
   const pending: TooltipCandidate<T>[] = [];
   for (const candidate of candidates(documents, graph)) {
-    const cached = cachedName(candidate.document, candidate.backendNodeId);
+    const cached = cachedEntry(candidate.document, candidate.backendNodeId);
     if (cached) {
-      names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
+      // A cached miss is still a decision: skip the control instead of re-probing.
+      if (cached.name !== null) {
+        names.set(
+          frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
+          cached.name,
+        );
+      }
     } else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
       pending.push(candidate);
     }
   }
   if (pending.length === 0) return names;
 
-  const started = Date.now();
+  const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
   const contexts = new Map<string, number>();
-  await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
   try {
     for (const candidate of pending) {
       if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
-      if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
+      if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
       let objectId: string | undefined;
       try {
         let contextId = contexts.get(candidate.document.frameId);
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
         await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
         const after = await tooltipTexts(cdp, candidate.document, objectId);
         const name = cleanTooltipDiff(before, after);
+        storeName(candidate.document, candidate.backendNodeId, name ?? null);
         if (!name) continue;
         names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
-        storeName(candidate.document, candidate.backendNodeId, name);
       } catch (error) {
         if (error instanceof Error && error.name === "AbortError") throw error;
       } finally {
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
     }
   } finally {
     await clearHover(cdp, tabId);
-    await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
   }
   return names;
 }

apps/extension/src/tools/vom/record-safe-observation.ts

@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
   frames: CaptureVomFrame[];
   matchNodes: CaptureVomMatchNode[];
   surfaceProbes?: CapturedSurfaceProbe[];
+  /**
+   * Whether this observation actively hovered the page, and whether that
+   * revealed content the static tree does not contain. Lets callers judge how
+   * far the page may have drifted from the returned snapshot.
+   */
+  hoverProbe?: HoverProbeReport;
+}
+
+export interface HoverProbeReport {
+  performed: boolean;
+  revealedContent: boolean;
 }
 
 function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
   frameDocuments: CapturedFrameDocument<FrameAxNode>[];
   rendered: VomResult;
   surfaceProbes?: CapturedSurfaceProbe[];
+  hoverProbe?: HoverProbeReport;
 }): CaptureVomObservationResult {
   return {
     text: input.rendered.text,
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
     frames: projectFrames(input.frameDocuments),
     matchNodes: projectMatchNodes(input.frameDocuments),
     surfaceProbes: input.surfaceProbes,
+    ...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
   };
 }

apps/extension/src/tools/vom/rendered-surfaces/cluster.ts

@@ -1,65 +0,0 @@
-import type { Rect } from "@browser-skill/vom";
-import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
-
-const STACK_IOU_RATIO = 0.9;
-const STACK_SIZE_RATIO = 0.9;
-
-function area(rect: Rect): number {
-  return Math.max(0, rect.w) * Math.max(0, rect.h);
-}
-
-function intersectionArea(a: Rect, b: Rect): number {
-  const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
-  const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
-  return width * height;
-}
-
-function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
-  if (a.frameId !== b.frameId) return false;
-  if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
-  const aArea = area(a.visibleRect);
-  const bArea = area(b.visibleRect);
-  const intersection = intersectionArea(a.visibleRect, b.visibleRect);
-  const union = aArea + bArea - intersection;
-  const widthRatio =
-    Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
-  const heightRatio =
-    Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
-  return (
-    union > 0 &&
-    intersection / union >= STACK_IOU_RATIO &&
-    widthRatio >= STACK_SIZE_RATIO &&
-    heightRatio >= STACK_SIZE_RATIO
-  );
-}
-
-function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
-  const representative = [...members].sort(
-    (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
-  )[0];
-  return {
-    frameId: representative.frameId,
-    members,
-    representative,
-    label: members.map((surface) => surface.label).find((label) => label !== undefined),
-  };
-}
-
-export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
-  const groups: RenderedSurface[][] = [];
-  for (const surface of surfaces) {
-    const matching = groups.find((group) =>
-      group.some((member) => sameVisualStack(member, surface)),
-    );
-    if (matching) matching.push(surface);
-    else groups.push([surface]);
-  }
-  return groups
-    .map(toGroup)
-    .sort(
-      (a, b) =>
-        a.frameId.localeCompare(b.frameId) ||
-        a.representative.visibleRect.y - b.representative.visibleRect.y ||
-        a.representative.visibleRect.x - b.representative.visibleRect.x,
-    );
-}

apps/extension/src/tools/vom/rendered-surfaces/discovery.ts

@@ -1,113 +0,0 @@
-import type { Rect, Viewport } from "@browser-skill/vom";
-import type { CapturedNode } from "../capture";
-import type { CapturedFrameDocument } from "../frame-capture";
-import type { SemanticAxNode } from "../semantic-graph";
-import type { RenderedSurface } from "./types";
-
-function clean(value: string | undefined): string | undefined {
-  const normalized = value?.replace(/\s+/g, " ").trim();
-  return normalized || undefined;
-}
-
-function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
-  return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
-}
-
-function intersectRect(a: Rect, b: Rect): Rect | null {
-  const x = Math.max(a.x, b.x);
-  const y = Math.max(a.y, b.y);
-  const right = Math.min(a.x + a.w, b.x + b.w);
-  const bottom = Math.min(a.y + a.h, b.y + b.h);
-  if (right <= x || bottom <= y) return null;
-  return { x, y, w: right - x, h: bottom - y };
-}
-
-const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
-
-function clipToOverflowAncestors(
-  node: CapturedNode,
-  rect: Rect,
-  nodesByBackend: ReadonlyMap<number, CapturedNode>,
-): Rect | null {
-  let clipped: Rect | null = rect;
-  let parentBackendNodeId = node.parentBackendNodeId;
-  let guard = 0;
-  while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
-    const parent = nodesByBackend.get(parentBackendNodeId);
-    if (!parent) break;
-    if (parent.rect) {
-      const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
-      const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
-      if (clipX || clipY) {
-        const clipRect = {
-          x: clipX ? parent.rect.x : clipped.x,
-          y: clipY ? parent.rect.y : clipped.y,
-          w: clipX ? parent.rect.w : clipped.w,
-          h: clipY ? parent.rect.h : clipped.h,
-        };
-        clipped = intersectRect(clipped, clipRect);
-      }
-    }
-    parentBackendNodeId = parent.parentBackendNodeId;
-    guard += 1;
-  }
-  return clipped;
-}
-
-function excludedByDomPolicy(
-  node: CapturedNode,
-  nodesByBackend: Map<number, CapturedNode>,
-  excludedBackendNodeIds: ReadonlySet<number>,
-): boolean {
-  let current: CapturedNode | undefined = node;
-  let guard = 0;
-  while (current && guard <= nodesByBackend.size) {
-    if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
-    if (current.visuallySuppressed === true) return true;
-    const attrs = current.attrs;
-    if (
-      Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
-      Object.prototype.hasOwnProperty.call(attrs, "inert") ||
-      (attrs["aria-hidden"] ?? "").toLowerCase() === "true"
-    ) {
-      return true;
-    }
-    current =
-      current.parentBackendNodeId === null
-        ? undefined
-        : nodesByBackend.get(current.parentBackendNodeId);
-    guard += 1;
-  }
-  return false;
-}
-
-export function discoverRenderedSurfaces(
-  documents: CapturedFrameDocument<SemanticAxNode>[],
-  viewport: Viewport,
-  excludedBackendNodeIds: ReadonlySet<number> = new Set(),
-): RenderedSurface[] {
-  const surfaces: RenderedSurface[] = [];
-  for (const document of documents) {
-    const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
-    for (const node of document.domNodes) {
-      if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
-      if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
-      const viewportRect = intersectViewport(node.rect, viewport);
-      const visibleRect = viewportRect
-        ? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
-        : null;
-      if (!visibleRect) continue;
-      const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
-      surfaces.push({
-        renderingKind: "canvas",
-        frameId: document.frameId,
-        backendNodeId: node.backendNodeId,
-        parentBackendNodeId: node.parentBackendNodeId,
-        visibleRect,
-        paintOrder: node.paintOrder,
-        ...(label ? { label } : {}),
-      });
-    }
-  }
-  return surfaces;
-}

apps/extension/src/tools/vom/rendered-surfaces/index.ts

@@ -1,4 +0,0 @@
-export { clusterRenderedSurfaces } from "./cluster";
-export { discoverRenderedSurfaces } from "./discovery";
-export { projectRenderedSurfaces } from "./project";
-export type { RenderedSurface, RenderedSurfaceGroup } from "./types";

apps/extension/src/tools/vom/rendered-surfaces/project.ts

@@ -1,100 +0,0 @@
-import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
-import type { CapturedFrameDocument } from "../frame-capture";
-import type { SemanticAxNode } from "../semantic-graph";
-import type { RenderedSurfaceGroup } from "./types";
-
-function frameBackendKey(frameId: string, backendNodeId: number): string {
-  return `${frameId}\u0000${backendNodeId}`;
-}
-
-interface ProjectionIndex {
-  sceneByBackend: ReadonlyMap<string, number>;
-  documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
-  domByFrame: ReadonlyMap<
-    string,
-    ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
-  >;
-  rootSceneIdByFrame: ReadonlyMap<string, number>;
-}
-
-function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
-  const document = index.documentByFrame.get(group.frameId);
-  const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
-  let parentBackendNodeId = group.representative.parentBackendNodeId;
-  let guard = 0;
-  while (parentBackendNodeId !== null && guard <= domByBackend.size) {
-    const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
-    if (sceneId !== undefined) return sceneId;
-    parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
-    guard += 1;
-  }
-
-  if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
-    const owner = index.sceneByBackend.get(
-      frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
-    );
-    if (owner !== undefined) return owner;
-  }
-
-  return index.rootSceneIdByFrame.get(group.frameId) ?? null;
-}
-
-function buildProjectionIndex(
-  scene: VomScene,
-  documents: CapturedFrameDocument<SemanticAxNode>[],
-): ProjectionIndex {
-  const rootSceneIdByFrame = new Map<string, number>();
-  for (const node of scene.nodes) {
-    if (
-      node.frameId &&
-      !rootSceneIdByFrame.has(node.frameId) &&
-      ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
-    ) {
-      rootSceneIdByFrame.set(node.frameId, node.id);
-    }
-  }
-  return {
-    sceneByBackend: new Map(
-      scene.nodes.flatMap((node) =>
-        node.backendNodeId === undefined
-          ? []
-          : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
-      ),
-    ),
-    documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
-    domByFrame: new Map(
-      documents.map((document) => [
-        document.frameId,
-        new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
-      ]),
-    ),
-    rootSceneIdByFrame,
-  };
-}
-
-export function projectRenderedSurfaces(
-  scene: VomScene,
-  documents: CapturedFrameDocument<SemanticAxNode>[],
-  groups: RenderedSurfaceGroup[],
-): VomScene {
-  const index = buildProjectionIndex(scene, documents);
-  const visualSurfaces: VomVisualSurface[] = [];
-
-  for (const group of groups) {
-    const parentId = nearestSceneParent(index, group);
-    visualSurfaces.push({
-      parentId,
-      backendNodeId: group.representative.backendNodeId,
-      frameId: group.frameId,
-      renderingKind: "canvas",
-      visibleRect: group.representative.visibleRect,
-      ...(group.label ? { label: group.label } : {}),
-      memberCount: group.members.length,
-    });
-  }
-
-  return {
-    ...scene,
-    ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
-  };
-}

apps/extension/src/tools/vom/rendered-surfaces/types.ts

@@ -1,18 +0,0 @@
-import type { Rect } from "@browser-skill/vom";
-
-export interface RenderedSurface {
-  renderingKind: "canvas";
-  frameId: string;
-  backendNodeId: number;
-  parentBackendNodeId: number | null;
-  visibleRect: Rect;
-  paintOrder: number;
-  label?: string;
-}
-
-export interface RenderedSurfaceGroup {
-  frameId: string;
-  members: RenderedSurface[];
-  representative: RenderedSurface;
-  label?: string;
-}

apps/extension/src/transport/types.ts

@@ -23,12 +23,9 @@ export type ErrorCode =
 export type RpcErrorReason =
   | "agent_window_scope"
   | "element_not_visible"
-  | "ref_capability_denied"
   | "ref_not_found"
   | "selector_not_found"
   | "target_not_fillable"
-  | "focus_target_unresolved"
-  | "input_not_applied"
   | "target_not_select"
   | "option_not_found"
   | "single_select_value_count"
@@ -36,11 +33,6 @@ export type RpcErrorReason =
   | "restricted_tab_url"
   | "borrow_conflict"
   | "screenshot_capture_failed"
-  | "surface_capture_not_found"
-  | "surface_capture_expired"
-  | "surface_capture_consumed"
-  | "surface_capture_stale"
-  | "surface_coordinate_invalid"
   | "cleanup_failed";
 
 export interface RpcErrorData {
@@ -306,17 +298,9 @@ export interface ScreenshotResult {
   height: number;
   format: string;
   tab_id: number;
-  capture?: SurfaceCaptureInfo;
   dialogs?: JavaScriptDialogInfo[];
 }
 
-export interface SurfaceCaptureInfo {
-  id: string;
-  surface_ref: string;
-  coordinate_space: "capture-image-pixel";
-  expires_at: number;
-}
-
 export interface SnapshotParams {
   session_id: string;
   tab_id?: number;
@@ -334,9 +318,14 @@ export interface SnapshotResult {
 
 export interface ObserveParams extends SnapshotParams {
   debug_surfaces?: boolean;
+  probe_hover?: boolean;
 }
 
 export interface ObserveResult extends SnapshotResult {
+  hover_probe?: {
+    performed: boolean;
+    revealed_content: boolean;
+  };
   debug?: {
     surface_probes?: Array<{
       trigger_backend_node_id: number;
@@ -430,9 +419,6 @@ export interface ClickParams {
   click_count?: number;
   modifiers?: KeyModifier[];
   timeout_ms?: number;
-  capture_id?: string;
-  image_x?: number;
-  image_y?: number;
 }
 
 export interface ClickResult {
@@ -441,9 +427,6 @@ export interface ClickResult {
   used_selector?: string;
   x: number;
   y: number;
-  capture_id?: string;
-  image_x?: number;
-  image_y?: number;
   dialogs?: JavaScriptDialogInfo[];
 }
 
@@ -484,26 +467,6 @@ export interface FillResult {
   dialogs?: JavaScriptDialogInfo[];
 }
 
-export interface TypeParams {
-  session_id: string;
-  text: string;
-  ref?: string;
-  selector?: string;
-  /** Explicitly dispatch to the unique focused text-entry target. */
-  focused?: boolean;
-  tab_id?: number;
-  timeout_ms?: number;
-}
-
-export interface TypeResult {
-  tab_id: number;
-  used_ref?: string;
-  used_selector?: string;
-  text_length: number;
-  verification: "dispatched";
-  dialogs?: JavaScriptDialogInfo[];
-}
-
 export interface PressParams {
   session_id: string;
   key: string;

apps/extension/test-fixtures/rendered-surfaces/README.md

@@ -1,147 +0,0 @@
-# Rendered Surface browser fixtures
-
-These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
-
-## Serve
-
-From the repository root:
-
-```bash
-python3 -m http.server 4173 --bind 127.0.0.1 \
-  --directory apps/extension/test-fixtures/rendered-surfaces
-```
-
-Use an Agent Window large enough to keep the primary matrix visible:
-
-```bash
-bsk session start --width 1440 --height 1100
-bsk navigate http://127.0.0.1:4173/index.html --session <id>
-bsk observe --session <id>
-```
-
-Always stop the session after the run.
-
-## Pages
-
-### `index.html`
-
-Expected individual Surface labels:
-
-- `fixture-basic-large`
-- `fixture-tiny-8px`
-- `fixture-pointer-events-none`
-- `fixture-partial-ax`
-- `fixture-contained-outer`
-- `fixture-contained-inner`
-- `fixture-different-parent-a`
-- `fixture-different-parent-b`
-
-Expected one grouped Surface:
-
-- `fixture-layered-group`, with `layers=2`
-
-Expected absent:
-
-- `fixture-display-none`
-- `fixture-visibility-hidden`
-- `fixture-opacity-zero`
-- `fixture-parent-opacity-zero`
-- `fixture-hidden-attribute`
-- `fixture-inert-parent`
-- `fixture-aria-hidden-parent`
-- `fixture-zero-size`
-- `fixture-offscreen-right`
-- `fixture-fully-overflow-clipped`
-
-`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
-
-### `unlabeled.html`
-
-This page contains eight Canvas elements but must render seven generic Surface markers because
-the two coincident layers form one group.
-
-- Every marker uses the stable fallback label `canvas visual surface`.
-- Exactly one marker has `layers=2`.
-- The two contained canvases remain separate markers.
-- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
-- Blank `aria-label` and `title` values behave as no label.
-- Fallback AX content does not suppress the corresponding Surface.
-
-### `frames.html`
-
-Expected Surface labels:
-
-- `fixture-same-origin-frame`
-- `fixture-cross-origin-frame`
-- `fixture-nested-frame`
-
-The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
-
-Each frame Canvas also exposes a `pointer status` live region. A successful
-screenshot-bound point click must report
-`down button=0 buttons=1 hoverReady=true; click received` and turn the Canvas
-yellow. The hover flag is set on the animation frame after `pointermove`, so the
-fixture verifies both child-document delivery and the scheduling boundary
-required before `pointerdown`, rather than merely validating projected
-coordinates or a successful CDP response.
-
-### `viewport.html`
-
-Run with a `900x700` Agent Window.
-
-- `fixture-partial-viewport` is partly visible and must be represented.
-- `fixture-below-fold` is absent initially.
-- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
-
-### `input-frames.html`
-
-This page exercises generic frame-aware input routing. It contains a top-level
-input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
-frame includes an input, textarea, contenteditable editor, an input that
-replaces its DOM node after accepted input, an input that rejects synthetic
-DOM events, a transient keyboard/IME input proxy that consumes and clears its
-DOM value, and a controlled input that resets its value on the next animation
-frame.
-
-- `fill` must route focus, text insertion, DOM events, and readback through the
-  ref's owning CDP target.
-- `press --ref` must focus and dispatch every key event through that same target.
-- Untargeted `press` must follow the actual focused target or refuse when focus
-  ownership cannot be determined uniquely.
-- The remounting input must retain the value through its replacement node.
-- The trusted input must be filled without synthetic `input` / `change` events.
-- `type` must deliver ASCII key events and committed non-ASCII IME/software
-  keyboard insertion to the input proxy. Its DOM value remains empty while
-  `proxy output` retains the consumed text; the result reports
-  `verification=dispatched`, not a retained DOM value.
-- `type` must reject an omitted target unless `focused=true` is explicit, and
-  focused mode must reject focus that is not owned by a text-entry target.
-- The rejecting input must return `input_not_applied`, never a successful
-  requested-value length.
-
-### `canvas-input.html`
-
-This is the PR3 end-to-end fixture. Its cross-origin frame contains a
-visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
-focuses a standard textbox; ordinary `fill` must then write and read back its
-real value through the textbox's owning target. The fixture deliberately does
-not provide a Canvas-specific fill action.
-
-### `dynamic.html`
-
-Use the normal DOM buttons and re-observe after each action:
-
-- Initial state: `fixture-dynamic` absent.
-- After **Insert canvas**: present.
-- After **Resize to zero**: absent.
-- After **Restore size**: present with a fresh ref.
-- After **Remove canvas**: absent.
-
-## Invariants
-
-- `snapshot` must not emit any of these Surface markers.
-- `observe` must emit only currently visible Canvas surfaces.
-- Existing DOM/AX controls remain readable alongside Surface markers.
-- Missing labels never prevent discovery; naming and discovery are independent.
-- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
-- No fixture URL, id, label, or layout constant may appear in production discovery code.

apps/extension/test-fixtures/rendered-surfaces/canvas-input.html

@@ -1,20 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Canvas to standard input fixture</title>
-    <style>
-      body { padding: 16px; font: 14px system-ui, sans-serif; }
-      iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
-    </style>
-  </head>
-  <body>
-    <h1>Canvas to standard input fixture</h1>
-    <p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
-    <iframe
-      title="cross-origin Canvas editor"
-      src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
-    ></iframe>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/dynamic.html

@@ -1,46 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Rendered Surface dynamic matrix</title>
-    <style>
-      html, body { margin: 0; font: 14px system-ui, sans-serif; }
-      body { padding: 16px; }
-      .controls { display: flex; gap: 8px; margin-bottom: 16px; }
-      button { padding: 8px 12px; }
-      canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
-    </style>
-  </head>
-  <body>
-    <h1>Dynamic Canvas lifecycle</h1>
-    <div class="controls">
-      <button id="insert" type="button">Insert canvas</button>
-      <button id="zero" type="button">Resize to zero</button>
-      <button id="restore" type="button">Restore size</button>
-      <button id="remove" type="button">Remove canvas</button>
-    </div>
-    <div id="mount"></div>
-    <script>
-      const mount = document.querySelector("#mount");
-      const getCanvas = () => mount.querySelector("canvas");
-      document.querySelector("#insert").addEventListener("click", () => {
-        if (getCanvas()) return;
-        const canvas = document.createElement("canvas");
-        canvas.width = 260;
-        canvas.height = 120;
-        canvas.setAttribute("aria-label", "fixture-dynamic");
-        mount.append(canvas);
-      });
-      document.querySelector("#zero").addEventListener("click", () => {
-        const canvas = getCanvas();
-        if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
-      });
-      document.querySelector("#restore").addEventListener("click", () => {
-        const canvas = getCanvas();
-        if (canvas) canvas.removeAttribute("style");
-      });
-      document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
-    </script>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/frame.html

@@ -1,80 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <style>
-      html, body { margin: 0; font: 14px system-ui, sans-serif; }
-      body { padding: 12px; background: #fdfefe; }
-      canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
-      iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
-      #pointer-status { margin: 8px 0 0; min-height: 20px; }
-    </style>
-  </head>
-  <body>
-    <canvas id="frame-canvas" width="220" height="90"></canvas>
-    <script>
-      const params = new URLSearchParams(location.search);
-      const canvas = document.querySelector("#frame-canvas");
-      if (params.get("canvas") === "0") canvas.remove();
-      else {
-        canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
-        if (params.get("editor") === "1") {
-          const context = canvas.getContext("2d");
-          context.font = "14px system-ui, sans-serif";
-          context.textBaseline = "middle";
-          context.strokeStyle = "#1e8449";
-          context.fillStyle = "#145a32";
-          for (let row = 0; row < 3; row += 1) {
-            const top = row * 30;
-            context.strokeRect(0.5, top + 0.5, 219, 29);
-            context.fillText(`Row ${row + 1}`, 12, top + 15);
-          }
-        }
-        if (params.get("interactive") === "1") {
-          const status = document.createElement("p");
-          status.id = "pointer-status";
-          status.setAttribute("aria-live", "polite");
-          status.textContent = "pointer status: idle";
-          canvas.insertAdjacentElement("afterend", status);
-          let hoverReady = false;
-          canvas.addEventListener("pointermove", () => {
-            hoverReady = false;
-            requestAnimationFrame(() => {
-              hoverReady = true;
-            });
-          });
-          canvas.addEventListener("pointerdown", (event) => {
-            status.textContent = `pointer status: down button=${event.button} buttons=${event.buttons} hoverReady=${hoverReady}`;
-            if (event.buttons === 1 && hoverReady) canvas.style.background = "#f9e79f";
-          });
-          canvas.addEventListener("click", () => {
-            status.textContent += "; click received";
-          });
-          if (params.get("editor") === "1") {
-            canvas.addEventListener("click", (event) => {
-              const rect = canvas.getBoundingClientRect();
-              const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
-              const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
-              status.textContent += `; selected row=${row + 1}`;
-              document.querySelector("#canvas-cell-editor")?.remove();
-              if (row !== 2) return;
-              const editor = document.createElement("input");
-              editor.id = "canvas-cell-editor";
-              editor.setAttribute("aria-label", "selected canvas cell editor");
-              editor.placeholder = "Type into selected row 3";
-              canvas.insertAdjacentElement("afterend", editor);
-              editor.focus();
-            });
-          }
-        }
-      }
-      if (params.get("nested") === "1") {
-        const iframe = document.createElement("iframe");
-        iframe.title = "nested fixture frame";
-        iframe.src = "frame.html?label=fixture-nested-frame&interactive=1";
-        document.body.append(iframe);
-      }
-    </script>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/frames.html

@@ -1,22 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Rendered Surface frame matrix</title>
-    <style>
-      html, body { margin: 0; font: 14px system-ui, sans-serif; }
-      body { padding: 16px; }
-      .frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
-      iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
-    </style>
-  </head>
-  <body>
-    <h1>Frame matrix</h1>
-    <div class="frames">
-      <iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame&interactive=1"></iframe>
-      <iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame&interactive=1"></iframe>
-      <iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
-    </div>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/index.html

@@ -1,118 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Rendered Surface discovery matrix</title>
-    <style>
-      * { box-sizing: border-box; }
-      html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
-      body { padding: 16px; background: #f4f6f8; color: #17202a; }
-      h1 { margin: 0 0 12px; font-size: 20px; }
-      .matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
-      .case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
-      .case h2 { margin: 0 0 8px; font-size: 13px; }
-      canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
-      .stack { position: relative; width: 180px; height: 90px; }
-      .stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
-      .stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
-      .contained { position: relative; width: 190px; height: 100px; }
-      .contained > canvas:first-child { width: 190px; height: 100px; }
-      .contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
-      .same-place { position: relative; width: 180px; height: 90px; }
-      .same-place > div { position: absolute; inset: 0; }
-      .same-place canvas { width: 180px; height: 90px; }
-      .same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
-      .clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
-      .partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
-      .fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
-      .tiny { width: 8px; height: 8px; border-width: 1px; }
-    </style>
-  </head>
-  <body>
-    <h1>Rendered Surface discovery matrix</h1>
-    <main class="matrix">
-      <section class="case">
-        <h2>Visible named canvas</h2>
-        <canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Tiny canvas remains addressable</h2>
-        <canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Visible but pointer-events none</h2>
-        <canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Partial AX/fallback content</h2>
-        <canvas aria-label="fixture-partial-ax" width="180" height="80">
-          <button type="button">Fallback button must not suppress the surface</button>
-        </canvas>
-      </section>
-
-      <section class="case">
-        <h2>Two real rendering layers</h2>
-        <div class="stack">
-          <canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
-          <canvas width="180" height="90"></canvas>
-        </div>
-      </section>
-
-      <section class="case">
-        <h2>Contained canvases are independent</h2>
-        <div class="contained">
-          <canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
-          <canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
-        </div>
-      </section>
-
-      <section class="case">
-        <h2>Same geometry, different parents</h2>
-        <div class="same-place">
-          <div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
-          <div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
-        </div>
-      </section>
-
-      <section class="case">
-        <h2>Partially overflow-clipped</h2>
-        <div class="clip-box">
-          <canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
-        </div>
-      </section>
-
-      <section class="case">
-        <h2>Fully overflow-clipped</h2>
-        <div class="clip-box">
-          <canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
-        </div>
-      </section>
-
-      <section class="case">
-        <h2>CSS-hidden variants</h2>
-        <canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
-        <canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
-        <canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Hidden ancestor variants</h2>
-        <div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
-        <div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
-        <div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
-        <div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
-      </section>
-
-      <section class="case">
-        <h2>Zero-size and offscreen</h2>
-        <canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
-        <canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
-      </section>
-    </main>
-  </body>
-</html>
-

apps/extension/test-fixtures/rendered-surfaces/input-frame.html

@@ -1,59 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <style>
-      body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
-      label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
-      input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
-      iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
-    </style>
-  </head>
-  <body>
-    <label>Frame input <input aria-label="frame input" /></label>
-    <label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
-    <div contenteditable="true" aria-label="frame editor"></div>
-    <label>Remounting input <span id="remount-host"><input aria-label="remounting input" /></span></label>
-    <label>Trusted input <input id="trusted" aria-label="trusted input" /></label>
-    <label>Input proxy <input id="proxy" aria-label="input proxy" /></label>
-    <output id="proxy-output" aria-label="proxy output">proxy output: empty</output>
-    <label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
-    <script>
-      const params = new URLSearchParams(location.search);
-      const remountHost = document.querySelector("#remount-host");
-      const armRemount = (input) => {
-        input.addEventListener("input", () => {
-          const replacement = document.createElement("input");
-          replacement.setAttribute("aria-label", "remounting input");
-          replacement.value = input.value;
-          armRemount(replacement);
-          input.replaceWith(replacement);
-          replacement.focus();
-        }, { once: true });
-      };
-      armRemount(remountHost.querySelector("input"));
-      const trusted = document.querySelector("#trusted");
-      trusted.addEventListener("input", (event) => {
-        if (!event.isTrusted) requestAnimationFrame(() => { trusted.value = ""; });
-      });
-      const proxy = document.querySelector("#proxy");
-      const proxyOutput = document.querySelector("#proxy-output");
-      let proxyValue = "";
-      proxy.addEventListener("input", (event) => {
-        if (event.isComposing) return;
-        proxyValue += event.data ?? proxy.value;
-        proxy.value = "";
-        proxyOutput.textContent = `proxy output: ${proxyValue || "empty"}`;
-      });
-      const rejecting = document.querySelector("#rejecting");
-      rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
-      if (params.get("nested") === "1") {
-        const iframe = document.createElement("iframe");
-        iframe.title = "nested input frame";
-        iframe.src = "input-frame.html";
-        document.body.append(iframe);
-      }
-    </script>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/input-frames.html

@@ -1,23 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Frame input routing matrix</title>
-    <style>
-      body { padding: 16px; font: 14px system-ui, sans-serif; }
-      .frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
-      iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
-      input { padding: 6px; }
-    </style>
-  </head>
-  <body>
-    <h1>Frame input routing matrix</h1>
-    <label>Top input <input aria-label="top input" /></label>
-    <div class="frames">
-      <iframe title="same-origin inputs" src="input-frame.html"></iframe>
-      <iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
-      <iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
-    </div>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/unlabeled.html

@@ -1,68 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Unlabeled Rendered Surface matrix</title>
-    <style>
-      * { box-sizing: border-box; }
-      html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
-      body { padding: 16px; background: #f4f6f8; color: #17202a; }
-      h1 { margin: 0 0 12px; font-size: 20px; }
-      .matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
-      .case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
-      .case h2 { margin: 0 0 8px; font-size: 13px; }
-      .case p { margin: 0 0 8px; }
-      canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
-      .stack { position: relative; width: 180px; height: 90px; }
-      .stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
-      .stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
-      .contained { position: relative; width: 190px; height: 100px; }
-      .contained > canvas:first-child { width: 190px; height: 100px; }
-      .contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
-    </style>
-  </head>
-  <body>
-    <h1>Unlabeled Rendered Surface matrix</h1>
-    <main class="matrix">
-      <section class="case">
-        <h2>Only engineering attributes</h2>
-        <canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Blank explicit names</h2>
-        <canvas aria-label="   " title="   " width="180" height="80"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Nearby prose is not a Canvas name</h2>
-        <p>must-not-be-inferred-as-a-surface-label</p>
-        <canvas width="180" height="80"></canvas>
-      </section>
-
-      <section class="case">
-        <h2>Fallback AX content does not name or suppress the Canvas</h2>
-        <canvas width="180" height="80">
-          <button type="button">Fallback control remains separate</button>
-        </canvas>
-      </section>
-
-      <section class="case">
-        <h2>Unnamed rendering layers still cluster</h2>
-        <div class="stack">
-          <canvas width="180" height="90"></canvas>
-          <canvas width="180" height="90"></canvas>
-        </div>
-      </section>
-
-      <section class="case">
-        <h2>Unnamed contained canvases stay independent</h2>
-        <div class="contained">
-          <canvas width="190" height="100"></canvas>
-          <canvas width="70" height="40"></canvas>
-        </div>
-      </section>
-    </main>
-  </body>
-</html>

apps/extension/test-fixtures/rendered-surfaces/viewport.html

@@ -1,22 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Rendered Surface viewport matrix</title>
-    <style>
-      html, body { margin: 0; font: 14px system-ui, sans-serif; }
-      body { min-height: 1800px; padding: 16px; }
-      canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
-      #partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
-      #below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
-    </style>
-  </head>
-  <body>
-    <h1>Viewport matrix</h1>
-    <p>The orange canvas on the left is intentionally clipped by the viewport.</p>
-    <canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
-    <canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
-  </body>
-</html>
-

crates/bsk-cli/skill/SKILL.md

@@ -67,58 +67,30 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
 
 ```
 bsk navigate <url> --session <id>
-bsk observe --session <id>           → primary semantic VOM view; reveals hover/focus surfaces
+bsk observe --session <id>           → primary semantic VOM view
 bsk snapshot --session <id>          → static aria tree fallback when VOM is insufficient
 bsk hover @e3 --session <id>          → reveal hover-triggered menus before re-observing/clicking
-bsk click @e4 --session <id>          → or bsk fill, bsk type, bsk select, bsk press
+bsk click @e4 --session <id>          → or bsk fill, bsk select, bsk press
 bsk observe --session <id>             → again after navigation / DOM change
 ```
 
-### Choosing `fill`, `type`, and `press`
-
-Choose by the intended postcondition, not by whether observation happens to
-render `textbox [empty]`:
-
-- `fill` sets a field's retained value. Use it when success means the target
-  value must equal the requested text; it replaces existing content by default.
-- `type` inserts text at the active caret/selection in a live editing session.
-  Use it when the application must consume keyboard/IME text; it never clears
-  first and reports dispatch only.
-- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
-  Do not encode special keys inside `type --text`.
-
-`type` requires an explicit targeting mode: pass a ref/selector when one is
-available, or pass `--focused` only after an immediately preceding action in
-the same tab deliberately established a text-entry focus. Never use `fill`
-as a probe and never automatically retry a failed `fill` with `type`: an
-application may already have consumed the text before clearing its proxy DOM
-value. Re-observe first. After `type`, verify the application output with one
-observation or screenshot; `verification=dispatched` is not business success.
-
 **Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
 
 Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
 
-When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
+When VOM marks a control `[has-submenu]` or `[expanded]`, its menu items are not in the observation yet. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. `[hover first: …]` means the same thing and additionally lists what that trigger revealed.
 
-When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
+`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have good reason to expect is absent from the observation **and** no `[has-submenu]` marker points at a trigger — that combination is what a CSS-only hover menu looks like from here. Try it before falling back to `bsk get-html` or `bsk screenshot`. It hovers a bounded set of likely triggers to surface hover-only menus and tooltips; it costs a few seconds and touches the live page, so once you know which element hides the menu, `bsk hover <ref>` is cheaper and more precise.
-
-If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
-
-```bash
-bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
-```
-
-Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
 
 ## Observation priority
 
-Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
+Start with `bsk observe` to understand page structure, text, controls, and element refs. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
 
-1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
+1. `bsk observe` — primary semantic VOM observation
-2. `bsk snapshot` — strict static accessibility tree fallback
+2. `bsk observe --probe-hover` — re-run once when an expected control is missing and nothing marks a hover trigger
-3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
+3. `bsk snapshot` — strict static accessibility tree fallback
-4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
+4. `bsk get-html` — when hidden DOM, metadata, or markup details are required
+5. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
 
 Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
 
@@ -203,9 +175,9 @@ bsk emulate --session <id> --off
 | Command | Summary |
 |---------|---------|
 | `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
-| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
+| `bsk observe` | Semantic VOM observation; `--probe-hover` adds bounded active hover probing for conditional surfaces |
 | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
-| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
+| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
 
 ### Console & network debugging (read-only; require `--session`)
 
@@ -231,11 +203,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
 
 | Command | Summary |
 |---------|---------|
-| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
+| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
 | `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
-| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
+| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
-| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
-| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
 | `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
 | `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
 
@@ -352,7 +322,7 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
 3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
 4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
-5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
+5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
 6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
 
 ---

crates/bsk-cli/src/cli/interaction.rs

@@ -11,7 +11,7 @@ use anyhow::Context;
 use bsk_protocol::Method;
 use bsk_protocol::tools::{
     ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
-    MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
+    MouseButton, PressParams, PressResult, SelectParams, SelectResult,
 };
 use clap::{Args, ValueEnum};
 
@@ -95,27 +95,6 @@ fn split_target(
     }
 }
 
-fn split_type_target(
-    positional: Option<String>,
-    explicit_ref: Option<String>,
-    explicit_selector: Option<String>,
-    focused: bool,
-) -> Result<(Option<String>, Option<String>), CliError> {
-    if focused {
-        if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
-            return Err(CliError::Local(anyhow::anyhow!(
-                "pass a target or --focused, not both"
-            )));
-        }
-        return Ok((None, None));
-    }
-    split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
-        CliError::Local(anyhow::anyhow!(
-            "type requires exactly one of: <target>, --ref, --selector, or --focused"
-        ))
-    })
-}
-
 // ---------------------------------------------------------------------------
 // bsk click
 // ---------------------------------------------------------------------------
@@ -157,18 +136,6 @@ pub struct ClickArgs {
 
     #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
     pub timeout: u32,
-
-    /// Capture id returned by `bsk screenshot --ref <surface>`.
-    #[arg(long)]
-    pub capture: Option<String>,
-
-    /// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
-    #[arg(long = "image-x", allow_hyphen_values = true)]
-    pub image_x: Option<f64>,
-
-    /// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
-    #[arg(long = "image-y", allow_hyphen_values = true)]
-    pub image_y: Option<f64>,
 }
 
 pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
@@ -180,14 +147,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
     )?;
     let modifiers = parse_modifiers(&args.modifiers)
         .map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
-    let point_arg_count = usize::from(args.capture.is_some())
-        + usize::from(args.image_x.is_some())
-        + usize::from(args.image_y.is_some());
-    if point_arg_count != 0 && point_arg_count != 3 {
-        return Err(CliError::Local(anyhow::anyhow!(
-            "--capture, --image-x, and --image-y must be given together"
-        )));
-    }
     let params = ClickParams {
         session_id: args.session.clone(),
         ref_,
@@ -201,9 +160,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
             Some(modifiers)
         },
         timeout_ms: Some(args.timeout),
-        capture_id: args.capture.clone(),
-        image_x: args.image_x,
-        image_y: args.image_y,
     };
     let reply: ClickResult = call(
         info.sock_path,
@@ -227,14 +183,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
                 "click ok tab={} target={target} at=({}, {})",
                 reply.tab_id, reply.x, reply.y
             );
-            if let Some(capture_id) = &reply.capture_id {
-                println!(
-                    "capture={} image=({}, {}) coordinate_space=capture-image-pixel",
-                    capture_id,
-                    reply.image_x.unwrap_or_default(),
-                    reply.image_y.unwrap_or_default()
-                );
-            }
             print_dialog_summaries(&reply.dialogs);
         }
     }
@@ -404,95 +352,6 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
     Ok(())
 }
 
-// ---------------------------------------------------------------------------
-// bsk type
-// ---------------------------------------------------------------------------
-
-#[derive(Debug, Clone, Args)]
-#[group(id = "type_destination", required = true, multiple = false)]
-pub struct TypeDestinationArgs {
-    /// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
-    #[arg(value_name = "TARGET")]
-    pub target: Option<String>,
-
-    #[arg(long = "ref")]
-    pub ref_: Option<String>,
-
-    #[arg(long = "selector")]
-    pub selector: Option<String>,
-
-    /// Insert into the unique focused text-entry target instead of focusing a target.
-    #[arg(long)]
-    pub focused: bool,
-}
-
-#[derive(Debug, Clone, Args)]
-pub struct TypeArgs {
-    #[command(flatten)]
-    pub destination: TypeDestinationArgs,
-
-    /// Text to dispatch through the live keyboard/IME editing session.
-    #[arg(long)]
-    pub text: String,
-
-    #[arg(long)]
-    pub session: String,
-
-    #[arg(long = "tab-id")]
-    pub tab_id: Option<i64>,
-
-    #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
-    pub timeout: u32,
-}
-
-pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
-    let (ref_, selector) = split_type_target(
-        args.destination.target.clone(),
-        args.destination.ref_.clone(),
-        args.destination.selector.clone(),
-        args.destination.focused,
-    )?;
-    let info = ensure_daemon().context("ensure daemon is running")?;
-    let params = TypeParams {
-        session_id: args.session.clone(),
-        text: args.text.clone(),
-        ref_,
-        selector,
-        focused: args.destination.focused,
-        tab_id: args.tab_id,
-        timeout_ms: Some(args.timeout),
-    };
-    let reply: TypeResult = call(
-        info.sock_path,
-        Method::ToolType,
-        params,
-        "type-1",
-        args.timeout,
-    )?;
-    match format {
-        Format::Json => {
-            println!(
-                "{}",
-                serde_json::to_string_pretty(&reply)
-                    .map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
-            );
-        }
-        Format::Human => {
-            let target = if args.destination.focused {
-                "focused".into()
-            } else {
-                format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
-            };
-            println!(
-                "type dispatched tab={} target={target} length={} verification=dispatched",
-                reply.tab_id, reply.text_length
-            );
-            print_dialog_summaries(&reply.dialogs);
-        }
-    }
-    Ok(())
-}
-
 // ---------------------------------------------------------------------------
 // bsk press
 // ---------------------------------------------------------------------------
@@ -769,52 +628,6 @@ mod tests {
         assert_eq!(parsed.args.session, "ohli");
     }
 
-    #[test]
-    fn type_clap_accepts_explicit_target_and_text_flag() {
-        use clap::Parser;
-
-        #[derive(Parser)]
-        #[command(name = "type")]
-        struct Wrapper {
-            #[command(flatten)]
-            args: TypeArgs,
-        }
-
-        let parsed =
-            Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
-                .expect("type args should parse");
-        assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
-        assert_eq!(parsed.args.text, "测试");
-        assert_eq!(parsed.args.session, "ohli");
-    }
-
-    #[test]
-    fn type_targeting_requires_exactly_one_explicit_mode() {
-        use clap::Parser;
-
-        #[derive(Parser)]
-        #[command(name = "type")]
-        struct Wrapper {
-            #[command(flatten)]
-            args: TypeArgs,
-        }
-
-        let parsed =
-            Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
-                .expect("focused type args should parse");
-        assert!(parsed.args.destination.focused);
-        assert!(parsed.args.destination.target.is_none());
-
-        let focused = split_type_target(None, None, None, true).unwrap();
-        assert_eq!(focused, (None, None));
-
-        let missing = split_type_target(None, None, None, false).unwrap_err();
-        assert!(missing.to_string().contains("--focused"));
-
-        let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
-        assert!(mixed.to_string().contains("not both"));
-    }
-
     #[test]
     fn select_clap_accepts_target_positional_with_repeated_value_flags() {
         use clap::Parser;

crates/bsk-cli/src/cli/mod.rs

@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
 use crate::cli::get_html::GetHtmlArgs;
 use crate::cli::human_loop::RequestHelpArgs;
 use crate::cli::install_skill::InstallSkillArgs;
-use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
+use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
 use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
 use crate::cli::network::NetworkArgs;
 use crate::cli::observe::ObserveArgs;
@@ -171,9 +171,6 @@ pub enum Command {
     /// Fill an input / textarea / contenteditable.
     Fill(FillArgs),
 
-    /// Insert text into an explicit target or focused keyboard/IME editing session.
-    Type(TypeArgs),
-
     /// Dispatch a keyboard key combo.
     Press(PressArgs),
 

crates/bsk-cli/src/cli/observe.rs

@@ -33,6 +33,12 @@ pub struct ObserveArgs {
     /// Include conditional surface probe diagnostics in JSON output.
     #[arg(long = "debug-surfaces")]
     pub debug_surfaces: bool,
+
+    /// Actively hover page controls to reveal hover-only menus and tooltips.
+    /// Costs seconds of wall clock and touches the live page, so it is off
+    /// unless a static observation looks like it is missing hover content.
+    #[arg(long = "probe-hover")]
+    pub probe_hover: bool,
 }
 
 pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
@@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
         max_depth: args.max_depth,
         max_tokens: args.max_tokens,
         debug_surfaces: args.debug_surfaces,
+        probe_hover: args.probe_hover,
     };
     let reply: ObserveResult = call(sock, params)?;
     match format {

crates/bsk-cli/src/cli/render_error.rs

@@ -40,23 +40,15 @@ use bsk_protocol::ErrorCode;
 pub mod reason {
     pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
     pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
-    pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
     pub const REF_NOT_FOUND: &str = "ref_not_found";
     pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
     pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
-    pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
-    pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
     pub const TARGET_NOT_SELECT: &str = "target_not_select";
     pub const OPTION_NOT_FOUND: &str = "option_not_found";
     pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
     pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
     pub const BORROW_CONFLICT: &str = "borrow_conflict";
     pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
-    pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
-    pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
-    pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
-    pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
-    pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
     pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
     pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
 }
@@ -236,27 +228,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
             ),
             exit_code: base.exit_code,
         },
-        (ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
-            summary: "focused frame target could not be determined",
-            hint: Some(
-                "pass an explicit --ref or --selector so the key event can be routed safely",
-            ),
-            exit_code: base.exit_code,
-        },
-        (ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
-            summary: "the page did not retain the requested input",
-            hint: Some(
-                "refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
-            ),
-            exit_code: base.exit_code,
-        },
-        (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
-            summary: "snapshot ref does not support this operation",
-            hint: Some(
-                "visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
-            ),
-            exit_code: base.exit_code,
-        },
         (ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
             summary: "target element is not a <select>",
             hint: Some(
@@ -299,41 +270,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
             ),
             exit_code: base.exit_code,
         },
-        (ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
-            summary: "surface capture transaction was not found",
-            hint: Some(
-                "take a fresh surface screenshot and use its capture id with coordinates from that image",
-            ),
-            exit_code: base.exit_code,
-        },
-        (ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
-            summary: "surface capture transaction has expired",
-            hint: Some(
-                "take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
-            ),
-            exit_code: base.exit_code,
-        },
-        (ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
-            summary: "surface capture transaction was already used",
-            hint: Some(
-                "each capture id is single-use; take a fresh surface screenshot before another point action",
-            ),
-            exit_code: base.exit_code,
-        },
-        (ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
-            summary: "page state changed since the surface screenshot",
-            hint: Some(
-                "take a fresh surface screenshot and choose coordinates from the new image before retrying",
-            ),
-            exit_code: base.exit_code,
-        },
-        (ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
-            summary: "surface image coordinate is invalid",
-            hint: Some(
-                "use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
-            ),
-            exit_code: base.exit_code,
-        },
         (
             ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
             reason::RECORD_START_PAGE_UNREACHABLE,
@@ -441,18 +377,6 @@ mod tests {
         assert!(!info.summary.contains("sandbox"));
     }
 
-    #[test]
-    fn ref_capability_denied_explains_visual_surface_refs() {
-        let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
-        let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
-        assert_eq!(info.summary, "snapshot ref does not support this operation");
-        assert!(
-            info.hint.unwrap().contains("capture id"),
-            "expected visual-surface-specific hint"
-        );
-        assert_eq!(info.exit_code, 1);
-    }
-
     #[test]
     fn session_busy_overrides_timeout_copy() {
         let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
@@ -495,25 +419,6 @@ mod tests {
         assert_eq!(info.exit_code, 3);
     }
 
-    #[test]
-    fn surface_capture_stale_requests_a_fresh_screenshot() {
-        let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
-        let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
-        assert_eq!(
-            info.summary,
-            "page state changed since the surface screenshot"
-        );
-        assert!(info.hint.unwrap().contains("fresh surface screenshot"));
-    }
-
-    #[test]
-    fn surface_coordinate_invalid_explains_image_pixel_bounds() {
-        let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
-        let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
-        assert_eq!(info.summary, "surface image coordinate is invalid");
-        assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
-    }
-
     #[test]
     fn ref_not_found_overrides_not_found_copy() {
         let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
@@ -548,20 +453,4 @@ mod tests {
         );
         assert!(info.hint.unwrap().contains("exactly one"));
     }
-
-    #[test]
-    fn focus_target_unresolved_overrides_cdp_copy() {
-        let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
-        let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
-        assert_eq!(info.summary, "focused frame target could not be determined");
-        assert!(info.hint.unwrap().contains("--ref"));
-    }
-
-    #[test]
-    fn input_not_applied_overrides_cdp_copy() {
-        let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
-        let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
-        assert_eq!(info.summary, "the page did not retain the requested input");
-        assert!(info.hint.unwrap().contains("rejecting"));
-    }
 }

crates/bsk-cli/src/cli/screenshot.rs

@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
         .map_err(CliError::Local)?;
     match format {
         Format::Json => {
-            let mut json = serde_json::json!({
+            let json = serde_json::json!({
                 "tab_id": reply.tab_id,
                 "width": reply.width,
                 "height": reply.height,
@@ -67,10 +67,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
                 "path": out_path.to_string_lossy(),
                 "byte_size": bytes.len(),
             });
-            if let Some(capture) = &reply.capture {
-                json["capture"] = serde_json::to_value(capture)
-                    .map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
-            }
             println!(
                 "{}",
                 serde_json::to_string_pretty(&json)
@@ -79,12 +75,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
         }
         Format::Human => {
             println!("{}", out_path.display());
-            if let Some(capture) = &reply.capture {
-                println!(
-                    "capture={} surface={} coordinate_space={} expires_at={}",
-                    capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
-                );
-            }
             print_dialog_summaries(&reply.dialogs);
         }
     }

crates/bsk-cli/src/daemon/ipc.rs

@@ -244,7 +244,6 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
                 | Method::ToolClick
                 | Method::ToolHover
                 | Method::ToolFill
-                | Method::ToolType
                 | Method::ToolPress
                 | Method::ToolSelect
                 | Method::ToolEvaluate

crates/bsk-cli/src/main.rs

@@ -95,7 +95,6 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
         Command::Click(args) => cli::interaction::dispatch_click(args, format),
         Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
         Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
-        Command::Type(args) => cli::interaction::dispatch_type(args, format),
         Command::Press(args) => cli::interaction::dispatch_press(args, format),
         Command::Select(args) => cli::interaction::dispatch_select(args, format),
         Command::Evaluate(args) => cli::evaluate::dispatch(args, format),

crates/bsk-cli/tests/cli_parse.rs

@@ -188,29 +188,6 @@ fn parses_click_count_alias() {
     assert_eq!(args.click_count, 2);
 }
 
-#[test]
-fn parses_surface_capture_image_coordinates() {
-    let cli = parse(&[
-        "bsk",
-        "click",
-        "@e1",
-        "--session",
-        "s1",
-        "--capture",
-        "sc_test",
-        "--image-x",
-        "160.5",
-        "--image-y",
-        "-1",
-    ]);
-    let Command::Click(args) = cli.command else {
-        panic!("expected click command");
-    };
-    assert_eq!(args.capture.as_deref(), Some("sc_test"));
-    assert_eq!(args.image_x, Some(160.5));
-    assert_eq!(args.image_y, Some(-1.0));
-}
-
 #[test]
 fn parses_hover_with_settle() {
     let cli = parse(&[

crates/bsk-cli/tests/tools_ipc.rs

@@ -238,7 +238,6 @@ async fn screenshot_returns_image_base64_with_dimensions() {
                 height: 600,
                 format: "png".into(),
                 tab_id: 7,
-                capture: None,
                 dialogs: vec![],
             })
             .unwrap(),
@@ -287,7 +286,6 @@ async fn screenshot_forwards_ref_to_extension() {
                 height: 60,
                 format: "png".into(),
                 tab_id: 7,
-                capture: None,
                 dialogs: vec![],
             })
             .unwrap(),
@@ -480,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
                 tab_id: 13,
                 truncated: false,
                 dialogs: vec![],
+                hover_probe: None,
                 debug: None,
             })
             .unwrap(),
@@ -496,6 +495,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
             max_depth: None,
             max_tokens: None,
             debug_surfaces: false,
+            probe_hover: false,
         },
     )
     .await

crates/bsk-cli/tests/tools_m7_ipc.rs

@@ -1,4 +1,5 @@
-//! M7 integration: drive the navigation and interaction tool RPCs
+//! M7 integration: drive the 7 new tool RPCs (navigate /
+//! navigate_back / navigate_forward / reload / click / fill / press)
 //! through the IPC + per-session queue + fake extension and assert
 //! the wire shapes line up. The extension stub mirrors each method's
 //! `*Result` so we exercise the daemon's serialise → forward →
@@ -15,8 +16,7 @@ use bsk_protocol::tools::{
     ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
     NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
     NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
-    SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
+    SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
-    WaitUntil,
 };
 use bsk_protocol::{
     BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
@@ -354,9 +354,6 @@ async fn click_round_trips_ref_and_modifiers() {
                 used_selector: None,
                 x: 12.5,
                 y: 34.0,
-                capture_id: None,
-                image_x: None,
-                image_y: None,
                 dialogs: vec![],
             })
             .unwrap(),
@@ -376,9 +373,6 @@ async fn click_round_trips_ref_and_modifiers() {
             click_count: Some(1),
             modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
             timeout_ms: Some(5_000),
-            capture_id: None,
-            image_x: None,
-            image_y: None,
         },
     )
     .await
@@ -433,51 +427,6 @@ async fn fill_round_trips_clear_before_default() {
     handle.shutdown().await;
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn type_round_trips_dispatched_verification() {
-    let (handle, sock) = spawn_daemon().await;
-    let mut ws = connect_ext(handle.ws_addr()).await;
-    let _ = do_handshake(&mut ws).await;
-
-    run_extension(ws, |req| {
-        assert_eq!(req.method, Method::ToolType);
-        let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
-        assert_eq!(p.text, "测试");
-        assert!(!p.focused);
-        ResponseBody::Ok(
-            serde_json::to_value(TypeResult {
-                tab_id: 12,
-                used_ref: Some("e3".into()),
-                used_selector: None,
-                text_length: 2,
-                verification: TypeVerification::Dispatched,
-                dialogs: vec![],
-            })
-            .unwrap(),
-        )
-    });
-
-    let session_id = ipc_session_start(&sock).await;
-    let result: TypeResult = ipc_tool_call(
-        &sock,
-        Method::ToolType,
-        TypeParams {
-            session_id,
-            text: "测试".into(),
-            ref_: Some("@e3".into()),
-            selector: None,
-            focused: false,
-            tab_id: None,
-            timeout_ms: Some(5_000),
-        },
-    )
-    .await
-    .expect("type ok");
-    assert_eq!(result.text_length, 2);
-    assert_eq!(result.verification, TypeVerification::Dispatched);
-    handle.shutdown().await;
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn press_round_trips_compound_key() {
     let (handle, sock) = spawn_daemon().await;
@@ -580,7 +529,6 @@ async fn m7_tools_propagate_extension_errors() {
         | Method::ToolReload
         | Method::ToolClick
         | Method::ToolFill
-        | Method::ToolType
         | Method::ToolPress
         | Method::ToolSelect => ResponseBody::Err(RpcError {
             code: ErrorCode::CdpFailed,
@@ -662,9 +610,6 @@ async fn m7_tools_propagate_extension_errors() {
             click_count: None,
             modifiers: None,
             timeout_ms: Some(5_000),
-            capture_id: None,
-            image_x: None,
-            image_y: None,
         },
     )
     .await
@@ -688,23 +633,6 @@ async fn m7_tools_propagate_extension_errors() {
     .unwrap_err();
     assert_eq!(fill.code, ErrorCode::CdpFailed);
 
-    let type_result = ipc_tool_call::<_, Value>(
-        &sock,
-        Method::ToolType,
-        TypeParams {
-            session_id: session_id.clone(),
-            text: "hello".into(),
-            ref_: Some("@e1".into()),
-            selector: None,
-            focused: false,
-            tab_id: None,
-            timeout_ms: Some(5_000),
-        },
-    )
-    .await
-    .unwrap_err();
-    assert_eq!(type_result.code, ErrorCode::CdpFailed);
-
     let press = ipc_tool_call::<_, Value>(
         &sock,
         Method::ToolPress,

crates/bsk-protocol/schema/tool_click_params.json

@@ -16,13 +16,6 @@
         }
       ]
     },
-    "capture_id": {
-      "description": "Short-lived capture id returned by a visual Surface screenshot.",
-      "type": [
-        "string",
-        "null"
-      ]
-    },
     "click_count": {
       "description": "Number of consecutive mouse presses (double-click = 2).",
       "type": [
@@ -32,22 +25,6 @@
       "format": "uint32",
       "minimum": 1.0
     },
-    "image_x": {
-      "description": "X coordinate in the capture image's pixel coordinate space.",
-      "type": [
-        "number",
-        "null"
-      ],
-      "format": "double"
-    },
-    "image_y": {
-      "description": "Y coordinate in the capture image's pixel coordinate space.",
-      "type": [
-        "number",
-        "null"
-      ],
-      "format": "double"
-    },
     "modifiers": {
       "type": [
         "array",

crates/bsk-protocol/schema/tool_click_result.json

@@ -8,12 +8,6 @@
     "y"
   ],
   "properties": {
-    "capture_id": {
-      "type": [
-        "string",
-        "null"
-      ]
-    },
     "dialogs": {
       "description": "Native JS dialogs observed and auto-handled during this call.",
       "type": "array",
@@ -21,20 +15,6 @@
         "$ref": "#/definitions/JavaScriptDialogInfo"
       }
     },
-    "image_x": {
-      "type": [
-        "number",
-        "null"
-      ],
-      "format": "double"
-    },
-    "image_y": {
-      "type": [
-        "number",
-        "null"
-      ],
-      "format": "double"
-    },
     "tab_id": {
       "type": "integer",
       "format": "int64"

crates/bsk-protocol/schema/tool_observe_params.json

@@ -1,7 +1,7 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "ObserveParams",
-  "description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
+  "description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery, but only when `probe_hover` asks for them. It must not submit input, click, navigate, or otherwise commit page state.",
   "type": "object",
   "required": [
     "session_id"
@@ -30,6 +30,11 @@
       "format": "uint32",
       "minimum": 0.0
     },
+    "probe_hover": {
+      "description": "Opt in to active hover probing: the extension moves the real cursor over a bounded set of controls to discover hover-only menus and tooltips.\n\nOff by default. Probing costs seconds of wall clock, mutates the live page, and pays off only on pages whose content is genuinely hidden behind hover. Request it when a static observation looks like it is missing hover-revealed structure.",
+      "default": false,
+      "type": "boolean"
+    },
     "session_id": {
       "type": "string"
     },

crates/bsk-protocol/schema/tool_observe_result.json

@@ -24,6 +24,17 @@
         "$ref": "#/definitions/JavaScriptDialogInfo"
       }
     },
+    "hover_probe": {
+      "description": "Present only when `probe_hover` was requested and probing actually ran.",
+      "anyOf": [
+        {
+          "$ref": "#/definitions/HoverProbeReport"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
     "ref_count": {
       "description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
       "type": "integer",
@@ -46,6 +57,24 @@
     }
   },
   "definitions": {
+    "HoverProbeReport": {
+      "description": "Reports what active hover probing did during an observation, so the agent can judge how far the live page may have drifted from the returned text.",
+      "type": "object",
+      "required": [
+        "performed",
+        "revealed_content"
+      ],
+      "properties": {
+        "performed": {
+          "description": "The real cursor was moved over page content during this observation.",
+          "type": "boolean"
+        },
+        "revealed_content": {
+          "description": "Hovering surfaced content the static tree does not contain, so the page reacts to the cursor and may not have fully reverted.",
+          "type": "boolean"
+        }
+      }
+    },
     "JavaScriptDialogHandledAction": {
       "description": "How the extension resolved the dialog so CDP could continue.",
       "type": "string",

crates/bsk-protocol/schema/tool_screenshot_result.json

@@ -8,16 +8,6 @@
     "tab_id"
   ],
   "properties": {
-    "capture": {
-      "anyOf": [
-        {
-          "$ref": "#/definitions/SurfaceCaptureInfo"
-        },
-        {
-          "type": "null"
-        }
-      ]
-    },
     "dialogs": {
       "type": "array",
       "items": {
@@ -119,31 +109,6 @@
         "prompt",
         "beforeunload"
       ]
-    },
-    "SurfaceCaptureInfo": {
-      "type": "object",
-      "required": [
-        "coordinate_space",
-        "expires_at",
-        "id",
-        "surface_ref"
-      ],
-      "properties": {
-        "coordinate_space": {
-          "type": "string"
-        },
-        "expires_at": {
-          "type": "integer",
-          "format": "uint64",
-          "minimum": 0.0
-        },
-        "id": {
-          "type": "string"
-        },
-        "surface_ref": {
-          "type": "string"
-        }
-      }
     }
   }
 }

crates/bsk-protocol/schema/tool_snapshot_result.json

@@ -26,7 +26,7 @@
       "format": "int64"
     },
     "text": {
-      "description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
+      "description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
       "type": "string"
     },
     "truncated": {

crates/bsk-protocol/schema/tool_type_params.json

@@ -1,50 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "TypeParams",
-  "type": "object",
-  "required": [
-    "session_id",
-    "text"
-  ],
-  "properties": {
-    "focused": {
-      "description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
-      "type": "boolean"
-    },
-    "ref": {
-      "description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
-      "type": [
-        "string",
-        "null"
-      ]
-    },
-    "selector": {
-      "type": [
-        "string",
-        "null"
-      ]
-    },
-    "session_id": {
-      "type": "string"
-    },
-    "tab_id": {
-      "type": [
-        "integer",
-        "null"
-      ],
-      "format": "int64"
-    },
-    "text": {
-      "description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
-      "type": "string"
-    },
-    "timeout_ms": {
-      "type": [
-        "integer",
-        "null"
-      ],
-      "format": "uint32",
-      "minimum": 1.0
-    }
-  }
-}

crates/bsk-protocol/schema/tool_type_result.json

@@ -1,124 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "TypeResult",
-  "type": "object",
-  "required": [
-    "tab_id",
-    "text_length",
-    "verification"
-  ],
-  "properties": {
-    "dialogs": {
-      "type": "array",
-      "items": {
-        "$ref": "#/definitions/JavaScriptDialogInfo"
-      }
-    },
-    "tab_id": {
-      "type": "integer",
-      "format": "int64"
-    },
-    "text_length": {
-      "description": "UTF-16 code-unit length of text accepted for dispatch.",
-      "type": "integer",
-      "format": "uint32",
-      "minimum": 0.0
-    },
-    "used_ref": {
-      "type": [
-        "string",
-        "null"
-      ]
-    },
-    "used_selector": {
-      "type": [
-        "string",
-        "null"
-      ]
-    },
-    "verification": {
-      "$ref": "#/definitions/TypeVerification"
-    }
-  },
-  "definitions": {
-    "JavaScriptDialogHandledAction": {
-      "description": "How the extension resolved the dialog so CDP could continue.",
-      "type": "string",
-      "enum": [
-        "accepted",
-        "dismissed"
-      ]
-    },
-    "JavaScriptDialogInfo": {
-      "description": "One observed + handled JavaScript dialog during a tool call.",
-      "type": "object",
-      "required": [
-        "handled",
-        "message",
-        "sequence",
-        "tab_id",
-        "type"
-      ],
-      "properties": {
-        "default_prompt": {
-          "type": [
-            "string",
-            "null"
-          ]
-        },
-        "handled": {
-          "$ref": "#/definitions/JavaScriptDialogHandledAction"
-        },
-        "has_browser_handler": {
-          "type": [
-            "boolean",
-            "null"
-          ]
-        },
-        "message": {
-          "type": "string"
-        },
-        "sequence": {
-          "description": "Monotonic per-tab sequence for ordering within a session.",
-          "type": "integer",
-          "format": "uint64",
-          "minimum": 0.0
-        },
-        "tab_id": {
-          "type": "integer",
-          "format": "int64"
-        },
-        "type": {
-          "$ref": "#/definitions/JavaScriptDialogType"
-        },
-        "url": {
-          "type": [
-            "string",
-            "null"
-          ]
-        }
-      }
-    },
-    "JavaScriptDialogType": {
-      "description": "Native JS dialog kind reported by CDP.",
-      "type": "string",
-      "enum": [
-        "alert",
-        "confirm",
-        "prompt",
-        "beforeunload"
-      ]
-    },
-    "TypeVerification": {
-      "oneOf": [
-        {
-          "description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
-          "type": "string",
-          "enum": [
-            "dispatched"
-          ]
-        }
-      ]
-    }
-  }
-}

crates/bsk-protocol/src/bin/dump-schema.rs

@@ -81,8 +81,6 @@ fn main() {
     dump!(HoverResult, "tool_hover_result");
     dump!(FillParams, "tool_fill_params");
     dump!(FillResult, "tool_fill_result");
-    dump!(TypeParams, "tool_type_params");
-    dump!(TypeResult, "tool_type_result");
     dump!(PressParams, "tool_press_params");
     dump!(PressResult, "tool_press_result");
     dump!(SelectParams, "tool_select_params");

crates/bsk-protocol/src/method.rs

@@ -74,8 +74,6 @@ pub enum Method {
     ToolHover,
     #[serde(rename = "tool.fill")]
     ToolFill,
-    #[serde(rename = "tool.type")]
-    ToolType,
     #[serde(rename = "tool.press")]
     ToolPress,
     #[serde(rename = "tool.select")]
@@ -156,7 +154,6 @@ impl Method {
             | Method::ToolReload
             | Method::ToolClick
             | Method::ToolFill
-            | Method::ToolType
             | Method::ToolPress
             | Method::ToolSelect
             | Method::ToolEvaluate
@@ -249,13 +246,6 @@ mod tests {
         assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
     }
 
-    #[test]
-    fn type_method_round_trips() {
-        let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
-        assert_eq!(method, Method::ToolType);
-        assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
-    }
-
     #[test]
     fn cancel_params_and_result_round_trip() {
         let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
@@ -294,7 +284,6 @@ mod tests {
         assert!(Method::ToolReload.is_mutating());
         assert!(Method::ToolClick.is_mutating());
         assert!(Method::ToolFill.is_mutating());
-        assert!(Method::ToolType.is_mutating());
         assert!(Method::ToolPress.is_mutating());
         assert!(Method::ToolSelect.is_mutating());
         assert!(Method::ToolEvaluate.is_mutating());

crates/bsk-protocol/src/tools/interaction.rs

@@ -1,5 +1,5 @@
-//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
+//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
-//! `tool.press`, `tool.select`).
+//! `tool.select`).
 //!
 //! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
 //! normalised against the session's RefStore) **or** a CSS selector
@@ -67,15 +67,6 @@ pub struct ClickParams {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     #[schemars(range(min = 1))]
     pub timeout_ms: Option<u32>,
-    /// Short-lived capture id returned by a visual Surface screenshot.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub capture_id: Option<String>,
-    /// X coordinate in the capture image's pixel coordinate space.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub image_x: Option<f64>,
-    /// Y coordinate in the capture image's pixel coordinate space.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub image_y: Option<f64>,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -89,12 +80,6 @@ pub struct ClickResult {
     /// agents can correlate with a follow-up `tool.screenshot`.
     pub x: f64,
     pub y: f64,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub capture_id: Option<String>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub image_x: Option<f64>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub image_y: Option<f64>,
     /// Native JS dialogs observed and auto-handled during this call.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub dialogs: Vec<JavaScriptDialogInfo>,
@@ -188,60 +173,6 @@ pub struct FillResult {
     pub dialogs: Vec<JavaScriptDialogInfo>,
 }
 
-// ---------------------------------------------------------------------------
-// type
-// ---------------------------------------------------------------------------
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
-pub struct TypeParams {
-    pub session_id: String,
-    /// Text to dispatch to the live editing session. Unlike `fill`, `type`
-    /// does not require the focused DOM editable to retain this text.
-    pub text: String,
-    /// Target to focus before dispatching text. Exactly one targeting mode
-    /// must be selected: `ref` / `selector`, or `focused=true`.
-    #[serde(
-        rename = "ref",
-        alias = "ref_",
-        default,
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub ref_: Option<String>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub selector: Option<String>,
-    /// Explicitly dispatch to the unique focused text-entry target instead
-    /// of focusing a ref or selector.
-    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
-    pub focused: bool,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub tab_id: Option<i64>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    #[schemars(range(min = 1))]
-    pub timeout_ms: Option<u32>,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "snake_case")]
-pub enum TypeVerification {
-    /// Chromium accepted the complete keyboard/IME dispatch transaction.
-    /// The caller must observe the application output to verify business state.
-    Dispatched,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
-pub struct TypeResult {
-    pub tab_id: i64,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub used_ref: Option<String>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub used_selector: Option<String>,
-    /// UTF-16 code-unit length of text accepted for dispatch.
-    pub text_length: u32,
-    pub verification: TypeVerification,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    pub dialogs: Vec<JavaScriptDialogInfo>,
-}
-
 // ---------------------------------------------------------------------------
 // press
 // ---------------------------------------------------------------------------
@@ -357,9 +288,6 @@ mod tests {
             click_count: Some(1),
             modifiers: Some(vec![KeyModifier::Ctrl]),
             timeout_ms: Some(5_000),
-            capture_id: Some("sc_test".into()),
-            image_x: Some(160.0),
-            image_y: Some(90.0),
         };
         let v = serde_json::to_value(&p).unwrap();
         assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
@@ -405,50 +333,6 @@ mod tests {
         assert_eq!(v, json!("meta"));
     }
 
-    #[test]
-    fn type_payloads_distinguish_dispatch_from_value_verification() {
-        let params = TypeParams {
-            session_id: "abcd".into(),
-            text: "测试".into(),
-            ref_: Some("@e3".into()),
-            selector: None,
-            focused: false,
-            tab_id: Some(4),
-            timeout_ms: Some(5_000),
-        };
-        let encoded = serde_json::to_value(&params).unwrap();
-        assert_eq!(encoded["ref"], json!("@e3"));
-        assert_eq!(encoded["text"], json!("测试"));
-        assert!(encoded.get("focused").is_none());
-
-        let focused = TypeParams {
-            session_id: "abcd".into(),
-            text: "hello".into(),
-            ref_: None,
-            selector: None,
-            focused: true,
-            tab_id: None,
-            timeout_ms: None,
-        };
-        assert_eq!(
-            serde_json::to_value(focused).unwrap()["focused"],
-            json!(true)
-        );
-
-        let result = TypeResult {
-            tab_id: 4,
-            used_ref: Some("e3".into()),
-            used_selector: None,
-            text_length: 2,
-            verification: TypeVerification::Dispatched,
-            dialogs: vec![],
-        };
-        assert_eq!(
-            serde_json::to_value(result).unwrap()["verification"],
-            json!("dispatched")
-        );
-    }
-
     #[test]
     fn press_result_round_trips() {
         let r = PressResult {

crates/bsk-protocol/src/tools/observation.rs

@@ -33,8 +33,8 @@ pub struct SnapshotParams {
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct SnapshotResult {
     /// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
-    /// agent can copy them into subsequent `tool.click` / `tool.fill` /
+    /// agent can copy them into subsequent `tool.click` / `tool.fill`
-    /// `tool.type` targets.
+    /// selectors.
     pub text: String,
     /// Number of `@e<N>` refs registered for this session by this
     /// snapshot. Equivalent to the size of the new ref-store map.
@@ -58,8 +58,9 @@ pub struct SnapshotResult {
 /// for one tab plus a fresh `@e<N>` ref-store on the extension side.
 ///
 /// Unlike `tool.snapshot`, this path may run bounded perception probes
-/// such as hover-surface discovery. It must not submit input, click,
+/// such as hover-surface discovery, but only when `probe_hover` asks for
-/// navigate, or otherwise commit page state.
+/// them. It must not submit input, click, navigate, or otherwise commit
+/// page state.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct ObserveParams {
     pub session_id: String,
@@ -77,6 +78,16 @@ pub struct ObserveParams {
     /// Include implementation diagnostics for conditional surface probes.
     #[serde(default)]
     pub debug_surfaces: bool,
+    /// Opt in to active hover probing: the extension moves the real cursor
+    /// over a bounded set of controls to discover hover-only menus and
+    /// tooltips.
+    ///
+    /// Off by default. Probing costs seconds of wall clock, mutates the live
+    /// page, and pays off only on pages whose content is genuinely hidden
+    /// behind hover. Request it when a static observation looks like it is
+    /// missing hover-revealed structure.
+    #[serde(default)]
+    pub probe_hover: bool,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -102,6 +113,17 @@ pub struct ObserveDebug {
     pub surface_probes: Vec<SurfaceProbeDebug>,
 }
 
+/// Reports what active hover probing did during an observation, so the agent
+/// can judge how far the live page may have drifted from the returned text.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct HoverProbeReport {
+    /// The real cursor was moved over page content during this observation.
+    pub performed: bool,
+    /// Hovering surfaced content the static tree does not contain, so the page
+    /// reacts to the cursor and may not have fully reverted.
+    pub revealed_content: bool,
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct ObserveResult {
     /// Semantic VOM observation text. Refs are rendered as `@e<N>` so
@@ -118,6 +140,9 @@ pub struct ObserveResult {
     pub truncated: bool,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub dialogs: Vec<JavaScriptDialogInfo>,
+    /// Present only when `probe_hover` was requested and probing actually ran.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hover_probe: Option<HoverProbeReport>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub debug: Option<ObserveDebug>,
 }
@@ -208,20 +233,10 @@ pub struct ScreenshotResult {
     /// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
     pub format: String,
     pub tab_id: i64,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub capture: Option<SurfaceCaptureInfo>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub dialogs: Vec<JavaScriptDialogInfo>,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
-pub struct SurfaceCaptureInfo {
-    pub id: String,
-    pub surface_ref: String,
-    pub coordinate_space: String,
-    pub expires_at: u64,
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -303,12 +318,6 @@ mod tests {
             height: 600,
             format: "png".into(),
             tab_id: 7,
-            capture: Some(SurfaceCaptureInfo {
-                id: "sc_test".into(),
-                surface_ref: "@e3".into(),
-                coordinate_space: "capture-image-pixel".into(),
-                expires_at: 1_700_000_030_000,
-            }),
             dialogs: vec![],
         };
         let v = serde_json::to_value(&r).unwrap();
@@ -324,6 +333,7 @@ mod tests {
             tab_id: 42,
             truncated: false,
             dialogs: Vec::new(),
+            hover_probe: None,
             debug: None,
         };
         let v = serde_json::to_value(&r).unwrap();
@@ -331,4 +341,30 @@ mod tests {
         let round: ObserveResult = serde_json::from_value(v).unwrap();
         assert_eq!(round, r);
     }
+
+    #[test]
+    fn observe_params_default_to_no_hover_probing() {
+        let params: ObserveParams =
+            serde_json::from_value(serde_json::json!({ "session_id": "s1" })).unwrap();
+        assert!(!params.probe_hover);
+    }
+
+    #[test]
+    fn observe_result_round_trips_with_hover_probe_report() {
+        let r = ObserveResult {
+            text: "@vom 1\n".into(),
+            ref_count: 0,
+            tab_id: 42,
+            truncated: false,
+            dialogs: Vec::new(),
+            hover_probe: Some(HoverProbeReport {
+                performed: true,
+                revealed_content: true,
+            }),
+            debug: None,
+        };
+        let v = serde_json::to_value(&r).unwrap();
+        let round: ObserveResult = serde_json::from_value(v).unwrap();
+        assert_eq!(round, r);
+    }
 }

packages/dsh-plugin-browserskill/src/tools.ts

@@ -489,9 +489,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       ? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
         "carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
       : "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
-        "probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " +
+        "probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
-        "understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " +
-        "alongside them. Read-only: never submits input.";
     register(
       defineTool({
         name,
@@ -563,8 +561,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       name: "interact.click",
       description:
         "Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
-        "last browser_inspect observation, or a CSS selector. A visual Surface requires the " +
+        "last browser_inspect observation, or a CSS selector.",
-        "captureId and image coordinates returned by browser_inspect action=screenshot.",
       parameters: {
         target: {
           type: "string",
@@ -581,24 +578,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
           type: "integer",
           description: "Number of consecutive presses (double-click = 2).",
         },
-        modifiers: {
-          type: "array",
-          items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
-          description: "Keyboard modifiers held during the click.",
-        },
-        captureId: {
-          type: "string",
-          description:
-            "Short-lived Surface capture id returned by browser_inspect action=screenshot.",
-        },
-        imageX: {
-          type: "number",
-          description: "X coordinate in capture-image pixels; requires captureId and imageY.",
-        },
-        imageY: {
-          type: "number",
-          description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
-        },
       },
       output: {
         schema: {
@@ -609,7 +588,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             tabId: { type: "integer", required: true },
             x: { type: "number", required: true },
             y: { type: "number", required: true },
-            captureId: { type: "string" },
           },
         },
         render: (_args, value) => [
@@ -621,44 +599,17 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       },
       async execute(args, exec) {
         if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
-        const pointArgCount =
-          Number(args.captureId !== undefined) +
-          Number(args.imageX !== undefined) +
-          Number(args.imageY !== undefined);
-        if (pointArgCount !== 0 && pointArgCount !== 3) {
-          throw new Error("captureId, imageX, and imageY must be given together");
-        }
         const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
         const cmdArgs = ["click", "--session", sessionId];
         if (args.button !== undefined) cmdArgs.push("--button", args.button);
         if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
-        if (args.modifiers !== undefined && args.modifiers.length > 0) {
-          cmdArgs.push("--modifiers", args.modifiers.join(","));
-        }
-        if (args.captureId !== undefined) {
-          cmdArgs.push(
-            "--capture",
-            args.captureId,
-            "--image-x",
-            String(args.imageX),
-            "--image-y",
-            String(args.imageY),
-          );
-        }
         cmdArgs.push(args.target);
         const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
           tab_id: number;
           x: number;
           y: number;
-          capture_id?: string;
-        };
-        return {
-          session: sessionId,
-          tabId: reply.tab_id,
-          x: reply.x,
-          y: reply.y,
-          ...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
         };
+        return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
       },
       presentCall: (args) => ({
         card: "terminal",
@@ -801,17 +752,12 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       name: "inspect.screenshot",
       description:
         "Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
-        "This is the supported operation for visual surface refs emitted by inspect.observe. Only use " +
-        "it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
-        "or image-reading capability, tell the user to switch to a model with image understanding; do " +
-        "not guess content or coordinates. " +
         "Returns the image itself when the deployment supports image input, otherwise a file path.",
       parameters: {
         session: SESSION_PARAM,
         ref: {
           type: "string",
-          description:
+          description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
-            "Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
         },
       },
       output: {
@@ -825,8 +771,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             width: { type: "integer", required: true },
             height: { type: "integer", required: true },
             byteSize: { type: "integer", required: true },
-            captureId: { type: "string" },
-            captureExpiresAt: { type: "integer" },
             image: {
               type: "object",
               additionalProperties: false,
@@ -850,11 +794,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             value.image !== undefined
               ? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
               : `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
-          const captureText =
+          const blocks: ContentBlock[] = [{ type: "text", text }];
-            value.captureId !== undefined
-              ? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
-              : "";
-          const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
           if (value.image !== undefined) {
             blocks.push({
               type: "image",
@@ -893,7 +833,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             height: number;
             path: string;
             byte_size: number;
-            capture?: { id: string; expires_at: number };
           };
           writtenPath = reply.path;
           const data = await readFile(reply.path);
@@ -907,9 +846,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             width: reply.width,
             height: reply.height,
             byteSize: reply.byte_size,
-            ...(reply.capture !== undefined
-              ? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
-              : {}),
             ...(ref !== undefined
               ? {
                   image: {

packages/dsh-plugin-browserskill/tests/tools.test.ts

@@ -478,12 +478,12 @@ describe("interaction tools", () => {
     emulate: { tab_id: 7, cleared: false },
   };
 
-  it("interact.click maps target, button, click count, and modifiers", async () => {
+  it("interact.click maps target, button, and click count", async () => {
     const { tools, calls } = setup(responses);
     await startSession(tools);
     const click = tools.get("interact.click");
     const value = (await click?.execute(
-      { target: "@e1", button: "right", clickCount: 2, modifiers: ["ctrl", "shift"] },
+      { target: "@e1", button: "right", clickCount: 2 },
       makeExec(),
     )) as { session: string; x: number; y: number };
     expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
@@ -495,43 +495,6 @@ describe("interaction tools", () => {
       "right",
       "--click-count",
       "2",
-      "--modifiers",
-      "ctrl,shift",
-      "@e1",
-    ]);
-  });
-
-  it("interact.click forwards a complete Surface capture coordinate transaction", async () => {
-    const { tools, calls } = setup({
-      ...responses,
-      click: {
-        tab_id: 7,
-        used_ref: "e1",
-        x: 110,
-        y: 70,
-        capture_id: "sc_test",
-        image_x: 200,
-        image_y: 100,
-      },
-    });
-    await startSession(tools);
-    const click = tools.get("interact.click");
-    const value = await click?.execute(
-      { target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
-      makeExec(),
-    );
-
-    expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
-    expect(calls[1].args).toEqual([
-      "click",
-      "--session",
-      "s1",
-      "--capture",
-      "sc_test",
-      "--image-x",
-      "200",
-      "--image-y",
-      "100",
       "@e1",
     ]);
   });
@@ -960,32 +923,6 @@ describe("inspect.screenshot", () => {
     expect(rendered?.every((block) => block.type === "text")).toBe(true);
   });
 
-  it("returns Surface capture metadata alongside the screenshot", async () => {
-    const path = pngFile();
-    const { tools } = setup({
-      "session start": START_REPLY("s1"),
-      screenshot: {
-        tab_id: 7,
-        width: 800,
-        height: 600,
-        format: "png",
-        path,
-        byte_size: 8,
-        capture: { id: "sc_test", expires_at: 1700000030000 },
-      },
-    });
-    await startSession(tools);
-    const screenshot = tools.get("inspect.screenshot");
-    const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
-
-    expect(value).toMatchObject({
-      captureId: "sc_test",
-      captureExpiresAt: 1700000030000,
-    });
-    const rendered = screenshot?.output.render({ session: "s1" }, value as never);
-    expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
-  });
-
   it("commits the image through the attachment store on an image-capable route", async () => {
     const path = pngFile();
     const services = {

packages/vom/src/__tests__/render.test.ts

@@ -29,66 +29,6 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
 }
 
 describe("renderVom single-layer page", () => {
-  it("renders visual surfaces as screenshot-only refs", () => {
-    const input = scene([
-      node({ id: 1, role: "RootWebArea", frameId: "main" }),
-      node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
-    ]);
-    input.visualSurfaces = [
-      {
-        parentId: 2,
-        backendNodeId: 99,
-        frameId: "child",
-        renderingKind: "canvas",
-        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
-        label: "Data grid",
-        memberCount: 2,
-      },
-    ];
-    const out = renderVom(input);
-
-    expect(out.text).toContain(
-      '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
-    );
-    expect(out.refs).toContainEqual(
-      expect.objectContaining({
-        ref: "e1",
-        backendNodeId: 99,
-        kind: "surface",
-        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
-        capabilities: ["screenshot"],
-      }),
-    );
-  });
-
-  it("renders an unnamed visual surface with a stable generic label", () => {
-    const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
-    input.visualSurfaces = [
-      {
-        parentId: 1,
-        backendNodeId: 99,
-        frameId: "main",
-        renderingKind: "canvas",
-        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
-        memberCount: 1,
-      },
-    ];
-
-    const out = renderVom(input);
-
-    expect(out.text).toContain(
-      '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
-    );
-    expect(out.refs).toContainEqual(
-      expect.objectContaining({
-        ref: "e1",
-        name: "canvas visual surface",
-        kind: "surface",
-        capabilities: ["screenshot"],
-      }),
-    );
-  });
-
   it("does not derive handle context across frame scopes", () => {
     const out = renderVom(
       scene([

packages/vom/src/index.ts

@@ -15,8 +15,6 @@ export type {
   VomNode,
   VomOptions,
   VomRef,
-  VomRefCapability,
   VomResult,
   VomScene,
-  VomVisualSurface,
 } from "./types";

packages/vom/src/render.ts

@@ -9,7 +9,6 @@ import type {
   VomOptions,
   VomResult,
   VomScene,
-  VomVisualSurface,
 } from "./types";
 
 const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -738,53 +737,6 @@ interface RenderState {
   duplicateRefNames: Set<string>;
   surfaceMap: Map<number, CondSurface>;
   scopeMap: Map<number, ActiveScopeBlock>;
-  visualSurfaces: Map<number | null, VomVisualSurface[]>;
-}
-
-function groupedByParent<T extends { parentId: number | null }>(
-  items: T[],
-): Map<number | null, T[]> {
-  const grouped = new Map<number | null, T[]>();
-  for (const item of items) {
-    const siblings = grouped.get(item.parentId) ?? [];
-    siblings.push(item);
-    grouped.set(item.parentId, siblings);
-  }
-  return grouped;
-}
-
-function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
-  if (state.stopped) return;
-  for (const surface of state.visualSurfaces.get(parentId) ?? []) {
-    if (depth > state.maxDepth) {
-      state.truncated = true;
-      continue;
-    }
-    const ref = `e${state.nextRef}`;
-    const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
-    const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
-    const line = `${"  ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
-    const nextTokens = state.tokens + estimateTokens(line);
-    if (nextTokens > state.maxTokens) {
-      state.truncated = true;
-      state.stopped = true;
-      return;
-    }
-    state.lines.push(line);
-    state.tokens = nextTokens;
-    state.refs.push({
-      ref,
-      backendNodeId: surface.backendNodeId,
-      frameId: surface.frameId,
-      visibleRect: surface.visibleRect,
-      role: "surface",
-      name: label,
-      kind: "surface",
-      capabilities: ["screenshot"],
-      line: state.lines.length - 1,
-    });
-    state.nextRef += 1;
-  }
 }
 
 function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
@@ -860,8 +812,6 @@ function renderTree(
         ...(node.role ? { role: node.role } : {}),
         ...(name ? { name } : {}),
         ...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
-        kind: "dom",
-        capabilities: ["interact", "screenshot"],
         line: state.lines.length - 1,
       });
       state.nextRef += 1;
@@ -871,12 +821,10 @@ function renderTree(
     if (scope) emitActiveScopeBlock(scope, depth + 1, state);
 
     if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
-      emitVisualEntries(node.id, depth + 1, state);
       continue;
     }
     renderTree(children, node.id, depth + 1, state);
   }
-  emitVisualEntries(parentId, depth, state);
 }
 
 function renderNodes(
@@ -885,7 +833,6 @@ function renderNodes(
   initialLines: string[],
   surfaces: CondSurface[] = [],
   activeScopeBlocks: ActiveScopeBlock[] = [],
-  visualSurfaces: VomVisualSurface[] = [],
 ): RenderState {
   const children = buildChildren(nodes);
   const state: RenderState = {
@@ -904,7 +851,6 @@ function renderNodes(
     duplicateRefNames: duplicateReferenceNames(nodes),
     surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
     scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
-    visualSurfaces: groupedByParent(visualSurfaces),
   };
 
   renderTree(children, null, 1, state);
@@ -1110,16 +1056,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
     "@layers 2 focus=L1",
     `L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
   ];
-  const visibleSurface = (parentId: number | null): boolean =>
+  const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
-    parentId === null || included.has(parentId);
-  const state = renderNodes(
-    visibleNodes,
-    options,
-    header,
-    scene.surfaces,
-    scene.activeScopeBlocks,
-    scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
-  );
   state.lines.push(renderPageOcclusionLine(hiddenCount));
 
   return {
@@ -1147,7 +1084,6 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
     ],
     scene.surfaces,
     scene.activeScopeBlocks,
-    scene.visualSurfaces,
   );
 
   return {

packages/vom/src/types.ts

@@ -61,8 +61,6 @@ export interface VomScene {
   rootFrameId?: string;
   surfaces?: CondSurface[];
   activeScopeBlocks?: ActiveScopeBlock[];
-  /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
-  visualSurfaces?: VomVisualSurface[];
 }
 
 export interface VomOptions {
@@ -96,22 +94,6 @@ export interface VomRef {
   ref: string;
   backendNodeId: number;
   frameId?: string;
-  /** Observation-time top-viewport crop for screenshot-only surfaces. */
-  visibleRect?: Rect;
-  kind?: "dom" | "surface";
-  capabilities?: VomRefCapability[];
-}
-
-export type VomRefCapability = "interact" | "screenshot";
-
-export interface VomVisualSurface {
-  parentId: number | null;
-  backendNodeId: number;
-  frameId: string;
-  renderingKind: "canvas";
-  visibleRect: Rect;
-  label?: string;
-  memberCount: number;
 }
 
 export interface RenderedRef extends VomRef {

skill/SKILL.md

@@ -67,7 +67,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
 
 ```
 bsk navigate <url> --session <id>
-bsk observe --session <id>           → primary semantic VOM view; reveals hover/focus surfaces
+bsk observe --session <id>           → primary semantic VOM view
 bsk snapshot --session <id>          → static aria tree fallback when VOM is insufficient
 bsk hover @e3 --session <id>          → reveal hover-triggered menus before re-observing/clicking
 bsk click @e4 --session <id>          → or bsk fill, bsk select, bsk press
@@ -78,26 +78,19 @@ bsk observe --session <id>             → again after navigation / DOM change
 
 Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
 
-When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
+When VOM marks a control `[has-submenu]` or `[expanded]`, its menu items are not in the observation yet. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. `[hover first: …]` means the same thing and additionally lists what that trigger revealed.
 
-When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
+`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have good reason to expect is absent from the observation **and** no `[has-submenu]` marker points at a trigger — that combination is what a CSS-only hover menu looks like from here. Try it before falling back to `bsk get-html` or `bsk screenshot`. It hovers a bounded set of likely triggers to surface hover-only menus and tooltips; it costs a few seconds and touches the live page, so once you know which element hides the menu, `bsk hover <ref>` is cheaper and more precise.
-
-If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
-
-```bash
-bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
-```
-
-Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
 
 ## Observation priority
 
-Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
+Start with `bsk observe` to understand page structure, text, controls, and element refs. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
 
-1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
+1. `bsk observe` — primary semantic VOM observation
-2. `bsk snapshot` — strict static accessibility tree fallback
+2. `bsk observe --probe-hover` — re-run once when an expected control is missing and nothing marks a hover trigger
-3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
+3. `bsk snapshot` — strict static accessibility tree fallback
-4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
+4. `bsk get-html` — when hidden DOM, metadata, or markup details are required
+5. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
 
 Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
 
@@ -182,9 +175,9 @@ bsk emulate --session <id> --off
 | Command | Summary |
 |---------|---------|
 | `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
-| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
+| `bsk observe` | Semantic VOM observation; `--probe-hover` adds bounded active hover probing for conditional surfaces |
 | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
-| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
+| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
 
 ### Console & network debugging (read-only; require `--session`)
 
@@ -210,7 +203,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
 
 | Command | Summary |
 |---------|---------|
-| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
+| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
 | `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
 | `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
 | `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
@@ -329,7 +322,7 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
 3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
 4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
-5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
+5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
 6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
 
 ---