Merge pull request #68 from Tencent/feat/more-browser-actions

feat(extension): hover end to end capability support

apps/extension/src/tools/__tests__/dispatcher.test.ts

@@ -9,6 +9,8 @@ import type {
 } from "@/transport/types";
 import { ToolDispatcher } from "../dispatcher";
 
+type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
+
 function fakeTransport() {
   const handlers = new Set<FrameHandler>();
   const stateHandlers = new Set<ConnectionStateHandler>();
@@ -129,7 +131,7 @@ describe("ToolDispatcher", () => {
       setUserAgentOverride: vi.fn(async () => {}),
       setTouchEmulationEnabled: vi.fn(async () => {}),
     };
-    const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
+    const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
     dispatcher.start();
 
     deliver(makeRequest("tool.console", { session_id: "aa11" }));
@@ -172,7 +174,7 @@ describe("ToolDispatcher", () => {
       setUserAgentOverride: vi.fn(async () => {}),
       setTouchEmulationEnabled: vi.fn(async () => {}),
     };
-    const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
+    const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
     dispatcher.start();
 
     deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
@@ -311,6 +313,277 @@ describe("ToolDispatcher", () => {
     expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
   });
 
+  it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
+    vi.stubGlobal("chrome", {
+      tabs: {
+        sendMessage: vi.fn(async () => undefined),
+      },
+    });
+    const { transport } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 1),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    const order: string[] = [];
+    const cdp = {
+      send: vi.fn(async <T>() => {
+        order.push("hover");
+        return {} as T;
+      }),
+      detachSession: vi.fn(async () => {}),
+      ensureNetworkCapture: vi.fn(async () => {}),
+      networkEntriesSince: vi.fn(() => ({
+        tab_id: 7,
+        entries: [],
+        next_since: 0,
+        truncated: false,
+      })),
+      setDeviceMetricsOverride: vi.fn(async () => {}),
+      clearDeviceMetricsOverride: vi.fn(async () => {}),
+      setUserAgentOverride: vi.fn(async () => {}),
+      setTouchEmulationEnabled: vi.fn(async () => {}),
+    };
+    const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
+
+    (
+      dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
+    ).rememberHover("aa11", {
+      tab_id: 7,
+      x: 10,
+      y: 20,
+    });
+    const helpers = dispatcher as unknown as {
+      withHoverReassert: <T>(
+        params: { session_id: string; tab_id?: number },
+        work: () => Promise<T>,
+        options?: { releaseAfter?: boolean },
+      ) => Promise<T>;
+      setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
+    };
+
+    await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
+      order.push("observe");
+      return {};
+    });
+    expect(order).toEqual(["hover", "observe"]);
+    expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+
+    await helpers.setHoverBypass("aa11", 7, true);
+    await helpers.withHoverReassert(
+      { session_id: "aa11", tab_id: 7 },
+      async () => {
+        order.push("click");
+        return {};
+      },
+      { releaseAfter: true },
+    );
+
+    expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
+      type: "mouseMoved",
+      x: 10,
+      y: 20,
+    });
+    expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+  });
+
+  it("does not reassert or disable hover latches from another session", async () => {
+    vi.stubGlobal("chrome", {
+      tabs: {
+        sendMessage: vi.fn(async () => undefined),
+      },
+    });
+    const { transport } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 1),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    const cdp = {
+      send: vi.fn(async <T>() => ({}) as T),
+      detachSession: vi.fn(async () => {}),
+      ensureNetworkCapture: vi.fn(async () => {}),
+      networkEntriesSince: vi.fn(() => ({
+        tab_id: 7,
+        entries: [],
+        next_since: 0,
+        truncated: false,
+      })),
+      setDeviceMetricsOverride: vi.fn(async () => {}),
+      clearDeviceMetricsOverride: vi.fn(async () => {}),
+      setUserAgentOverride: vi.fn(async () => {}),
+      setTouchEmulationEnabled: vi.fn(async () => {}),
+    };
+    const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
+    const helpers = dispatcher as unknown as {
+      rememberHover: (sessionId: string, result: unknown) => unknown;
+      setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
+      withHoverReassert: <T>(
+        params: { session_id: string; tab_id?: number },
+        work: () => Promise<T>,
+        options?: { releaseAfter?: boolean },
+      ) => Promise<T>;
+      releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
+    };
+
+    helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
+    await helpers.setHoverBypass("aa11", 7, true);
+
+    await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
+      releaseAfter: true,
+    });
+
+    expect(cdp.send).not.toHaveBeenCalled();
+    expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+
+    await helpers.releaseHoverLatch("aa11", 7);
+    expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+  });
+
+  it("transfers hover overlay bypass ownership between sessions", async () => {
+    vi.stubGlobal("chrome", {
+      tabs: {
+        sendMessage: vi.fn(async () => undefined),
+      },
+    });
+    const { transport } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 1),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    const dispatcher = new ToolDispatcher({ transport, sessions });
+    const helpers = dispatcher as unknown as {
+      setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
+    };
+
+    await helpers.setHoverBypass("aa11", 7, true);
+    await helpers.setHoverBypass("bb22", 7, true);
+    await helpers.setHoverBypass("aa11", 7, false);
+    expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+
+    await helpers.setHoverBypass("bb22", 7, false);
+    expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+    expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
+  });
+
+  it("limits default-target hover reassertion to the active tab", async () => {
+    vi.stubGlobal("chrome", {
+      tabs: {
+        query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
+        get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
+        sendMessage: vi.fn(async () => undefined),
+      },
+    });
+    const { transport } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 4242),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    await sessions.start("aa11");
+    const cdp = {
+      send: vi.fn(async <T>() => ({}) as T),
+      detachSession: vi.fn(async () => {}),
+      ensureNetworkCapture: vi.fn(async () => {}),
+      networkEntriesSince: vi.fn(() => ({
+        tab_id: 9,
+        entries: [],
+        next_since: 0,
+        truncated: false,
+      })),
+      setDeviceMetricsOverride: vi.fn(async () => {}),
+      clearDeviceMetricsOverride: vi.fn(async () => {}),
+      setUserAgentOverride: vi.fn(async () => {}),
+      setTouchEmulationEnabled: vi.fn(async () => {}),
+    };
+    const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
+    const helpers = dispatcher as unknown as {
+      rememberHover: (sessionId: string, result: unknown) => unknown;
+      withHoverReassert: <T>(
+        params: { session_id: string; tab_id?: number },
+        work: () => Promise<T>,
+      ) => Promise<T>;
+    };
+
+    helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
+    helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
+
+    await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
+
+    expect(cdp.send).toHaveBeenCalledTimes(1);
+    expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
+      type: "mouseMoved",
+      x: 30,
+      y: 40,
+    });
+  });
+
+  it("releases the current session hover latch before navigation-style work", async () => {
+    vi.stubGlobal("chrome", {
+      tabs: {
+        sendMessage: vi.fn(async () => undefined),
+      },
+    });
+    const { transport } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 1),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    const dispatcher = new ToolDispatcher({ transport, sessions });
+    const helpers = dispatcher as unknown as {
+      rememberHover: (sessionId: string, result: unknown) => unknown;
+      setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
+      withHoverReleaseForRequest: <T>(
+        params: { session_id: string; tab_id?: number },
+        work: () => Promise<T>,
+      ) => Promise<T>;
+      withHoverReassert: <T>(
+        params: { session_id: string; tab_id?: number },
+        work: () => Promise<T>,
+      ) => Promise<T>;
+    };
+    helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
+    await helpers.setHoverBypass("aa11", 7, true);
+
+    await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
+    await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
+
+    expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
+      7,
+      expect.objectContaining({ enabled: false }),
+    );
+  });
+
   it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
     const { transport, deliver } = fakeTransport();
     const sessions = new SessionManager({

apps/extension/src/tools/__tests__/interaction.test.ts

@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
 import {
   handleClick,
   handleFill,
+  handleHover,
   handlePress,
   handleSelect,
   modifiersBitfield,
@@ -435,6 +436,50 @@ describe("handleClick", () => {
   });
 });
 
+describe("handleHover", () => {
+  it("keeps overlay bypass enabled after a successful hover when requested", async () => {
+    const bypassOverlay = vi.fn().mockResolvedValue(undefined);
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, { tabId: 4 });
+    const fake = makeFakeCdp({
+      "DOM.scrollIntoViewIfNeeded": () => ({}),
+      "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
+      "Runtime.evaluate": (params: unknown) => {
+        const expr = String((params as { expression?: string })?.expression ?? "");
+        if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
+          return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
+        }
+        if (expr.includes("hitIndex")) {
+          return {
+            result: {
+              value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
+            },
+          };
+        }
+        throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
+      },
+      "Input.dispatchMouseEvent": () => ({}),
+    });
+
+    const res = await handleHover(
+      sm,
+      { session_id: "aa11", ref: "@e3", settle_ms: 0 },
+      {
+        cdp: fake.cdp,
+        tabsApi: fake.tabsApi,
+        bypassOverlay,
+        keepOverlayBypassAfterHover: true,
+      },
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
+    expect(bypassOverlay).toHaveBeenCalledTimes(1);
+    expect(bypassOverlay).toHaveBeenCalledWith(4, true);
+  });
+});
+
 describe("handleFill", () => {
   it("returns not_found for unknown ref", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });

apps/extension/src/tools/__tests__/observation.test.ts

@@ -1523,7 +1523,7 @@ describe("buildVomScene", () => {
     expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
   });
 
-  it("maps hover surface probes to matching node labels", () => {
+  it("maps hover surface probes to matching backend node ids", () => {
     const axNodes: CdpAxNode[] = [
       {
         nodeId: "1",
@@ -1544,7 +1544,7 @@ describe("buildVomScene", () => {
       iframeNodes: new Map(),
       excludedBackendNodeIds: new Set(),
       surfaceProbes: [
-        { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
+        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
       ],
       nodes: [
         {
@@ -1573,7 +1573,260 @@ describe("buildVomScene", () => {
     expect(scene.surfaces).toEqual([
       { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
     ]);
-    expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
+    expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
+  });
+
+  it("maps hover surface probes to rendered descendants in the same trigger subtree", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "button" },
+        name: { type: "computedString", value: "image" },
+        backendDOMNodeId: 21,
+      },
+    ];
+    const scene = buildVomScene(axNodes, {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      surfaceProbes: [
+        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
+      ],
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "div",
+          attrs: { class: "tg-avatar" },
+          rect: { x: 900, y: 10, w: 30, h: 30 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 21,
+          parentBackendNodeId: 20,
+          tag: "div",
+          attrs: { class: "tg-avatar__inner" },
+          rect: { x: 902, y: 12, w: 26, h: 26 },
+          paintOrder: 2,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    });
+
+    expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
+  });
+
+  it("deduplicates hover surface probes by original trigger backend id", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "button" },
+        name: { type: "computedString", value: "image" },
+        backendDOMNodeId: 21,
+      },
+    ];
+    const scene = buildVomScene(axNodes, {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      surfaceProbes: [
+        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
+        { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
+        { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
+      ],
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "div",
+          attrs: { class: "tg-avatar" },
+          rect: { x: 900, y: 10, w: 30, h: 30 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 21,
+          parentBackendNodeId: 20,
+          tag: "div",
+          attrs: { class: "tg-avatar__inner" },
+          rect: { x: 902, y: 12, w: 26, h: 26 },
+          paintOrder: 2,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    });
+
+    expect(scene.surfaces).toEqual([
+      { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
+    ]);
+  });
+
+  it("maps hover surface probes to DOM-recovered custom controls", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "img" },
+        name: { type: "computedString", value: "image" },
+        backendDOMNodeId: 21,
+      },
+    ];
+    const scene = buildVomScene(axNodes, {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      surfaceProbes: [
+        {
+          triggerBackendNodeId: 20,
+          triggerPoint: { x: 915, y: 25 },
+          triggerAction: "hover",
+          subItems: ["My profile", "Sign out"],
+        },
+      ],
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+          cursor: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "div",
+          attrs: { class: "tg-avatar" },
+          rect: { x: 900, y: 10, w: 30, h: 30 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+          cursor: "pointer",
+        },
+        {
+          backendNodeId: 21,
+          parentBackendNodeId: 20,
+          tag: "div",
+          attrs: { class: "tg-avatar__inner" },
+          rect: { x: 902, y: 12, w: 26, h: 26 },
+          paintOrder: 2,
+          position: "static",
+          pointerEvents: "auto",
+          cursor: "pointer",
+        },
+      ],
+    });
+
+    expect(scene.surfaces).toEqual([
+      { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
+    ]);
+    expect(renderVom(scene).text).toContain(
+      '@e1 button "image" [hover first: My profile | Sign out]',
+    );
+  });
+
+  it("does not attach hover probes to distant geometry matches", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "link" },
+        name: { type: "computedString", value: "0" },
+        backendDOMNodeId: 42,
+      },
+    ];
+    const scene = buildVomScene(axNodes, {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      surfaceProbes: [
+        {
+          triggerBackendNodeId: 999,
+          triggerPoint: { x: 900, y: 20 },
+          triggerAction: "hover",
+          subItems: ["My profile"],
+        },
+      ],
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 42,
+          parentBackendNodeId: 10,
+          tag: "a",
+          attrs: {},
+          rect: { x: 860, y: 120, w: 40, h: 30 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    });
+
+    expect(scene.surfaces).toBeUndefined();
+    expect(renderVom(scene).text).not.toContain("[hover first:");
   });
 
   it("enriches names and active scope signals from AX properties", () => {
@@ -2046,7 +2299,7 @@ describe("handleSnapshot", () => {
     });
     const res = await handleObserve(
       sm,
-      { session_id: "aa11" },
+      { session_id: "aa11", debug_surfaces: true },
       {
         cdp: {
           send: send as unknown as <T = unknown>(
@@ -2067,6 +2320,7 @@ describe("handleSnapshot", () => {
     );
 
     if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(res.debug).toEqual({ surface_probes: [] });
     expect(send).toHaveBeenCalledWith(
       4,
       "Runtime.evaluate",