feat(vom): update vom algorithm, fix overlay problem with background state machine

2026-09-10 13:01:23 +08:00 · 2026-07-27 13:46:33 +08:00
parent e14ae9cea0
commit 4fb2e331a6
36 changed files with 2276 additions and 452 deletions
@@ -274,6 +274,35 @@ describe("ToolDispatcher", () => {
    expect(onSessionsChanged).toHaveBeenCalledTimes(2);
  });

+  it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
+    const { transport, sent, deliver } = fakeTransport();
+    const sessions = new SessionManager({
+      agentWindow: {
+        create: vi.fn(async () => 1),
+        remove: vi.fn(async () => {}),
+        ensureActiveTab: vi.fn(async () => {}),
+      },
+    });
+    const onBrowserControlResumed = vi.fn();
+    const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
+    dispatcher.start();
+
+    deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
+    await flushMicrotasks();
+    sent.length = 0;
+
+    deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
+    await flushMicrotasks();
+    expect(onBrowserControlResumed).not.toHaveBeenCalled();
+
+    deliver({
+      ...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
+      id: "r-control",
+    });
+    await flushMicrotasks();
+    expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
+  });
+
  it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
    const { transport, deliver } = fakeTransport();
    const sessions = new SessionManager({
@@ -7,6 +7,7 @@ import {
  buildVomScene,
  type CdpAxNode,
  handleGetHtml,
+  handleObserve,
  handleScreenshot,
  handleSnapshot,
  parsePngDimensions,
@@ -1412,7 +1413,7 @@ describe("buildVomScene", () => {
    expect(scene.surfaces).toEqual([
      { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
    ]);
-    expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
+    expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
  });

  it("enriches names and active scope signals from AX properties", () => {
@@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => {
    expect(ctx.refStore.resolve("e2")).toBeNull();
  });

+  it("keeps snapshots static without conditional surface probing", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Example" },
+      backendDOMNodeId: 100,
+      childIds: ["2"],
+    };
+    const button: CdpAxNode = {
+      nodeId: "2",
+      parentId: "1",
+      role: { type: "role", value: "button" },
+      name: { type: "computedString", value: "Products" },
+      backendDOMNodeId: 200,
+    };
+    const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
+    const i = (s: string) => strings.indexOf(s);
+    const send = vi.fn(async (_tabId: number, method: string) => {
+      if (method === "Accessibility.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
+      }
+      if (method === "DOMSnapshot.enable") return {};
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0],
+                nodeName: [i("body"), i("button")],
+                backendNodeId: [100, 200],
+                attributes: [[], []],
+              },
+              layout: {
+                nodeIndex: [0, 1],
+                styles: [
+                  [i("static"), i("auto"), i("auto")],
+                  [i("static"), i("auto"), i("auto")],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [20, 20, 120, 40],
+                ],
+                paintOrders: [0, 1],
+              },
+            },
+          ],
+        };
+      }
+      if (method === "Runtime.evaluate") return { result: { value: [] } };
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+    const res = await handleSnapshot(
+      sm,
+      { session_id: "aa11" },
+      {
+        cdp: {
+          send: send as unknown as <T = unknown>(
+            tabId: number,
+            method: string,
+            params?: object,
+          ) => Promise<T>,
+          trackSessionTab: vi.fn(),
+        },
+        tabsApi: {
+          get: vi.fn(
+            async (tabId: number) =>
+              ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+          ),
+          query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+        },
+      },
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(send).not.toHaveBeenCalledWith(
+      4,
+      "Runtime.evaluate",
+      expect.objectContaining({ returnByValue: true }),
+    );
+  });
+
+  it("enables conditional surface probing for semantic observe", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Example" },
+      backendDOMNodeId: 100,
+      childIds: ["2"],
+    };
+    const button: CdpAxNode = {
+      nodeId: "2",
+      parentId: "1",
+      role: { type: "role", value: "button" },
+      name: { type: "computedString", value: "Products" },
+      backendDOMNodeId: 200,
+    };
+    const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
+    const i = (s: string) => strings.indexOf(s);
+    const send = vi.fn(async (_tabId: number, method: string) => {
+      if (method === "Accessibility.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
+      }
+      if (method === "DOMSnapshot.enable") return {};
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0],
+                nodeName: [i("body"), i("button")],
+                backendNodeId: [100, 200],
+                attributes: [[], []],
+              },
+              layout: {
+                nodeIndex: [0, 1],
+                styles: [
+                  [i("static"), i("auto"), i("auto")],
+                  [i("static"), i("auto"), i("auto")],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [20, 20, 120, 40],
+                ],
+                paintOrders: [0, 1],
+              },
+            },
+          ],
+        };
+      }
+      if (method === "Runtime.evaluate") return { result: { value: [] } };
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+    const res = await handleObserve(
+      sm,
+      { session_id: "aa11" },
+      {
+        cdp: {
+          send: send as unknown as <T = unknown>(
+            tabId: number,
+            method: string,
+            params?: object,
+          ) => Promise<T>,
+          trackSessionTab: vi.fn(),
+        },
+        tabsApi: {
+          get: vi.fn(
+            async (tabId: number) =>
+              ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+          ),
+          query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+        },
+      },
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(send).toHaveBeenCalledWith(
+      4,
+      "Runtime.evaluate",
+      expect.objectContaining({ returnByValue: true }),
+    );
+  });
+
+  it("allows passive snapshots of explicit user-window tabs", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "User tab" },
+      backendDOMNodeId: 100,
+      childIds: ["2"],
+    };
+    const button: CdpAxNode = {
+      nodeId: "2",
+      parentId: "1",
+      role: { type: "role", value: "button" },
+      name: { type: "computedString", value: "Read" },
+      backendDOMNodeId: 200,
+    };
+    const deps = makeDeps([root, button]);
+    deps.tabsApi.get = vi.fn(
+      async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
+    );
+
+    const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps);
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(res.tab_id).toBe(9);
+    expect(res.text).toContain('@e1 button "Read"');
+    expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {});
+  });
+
+  it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const deps = makeDeps([]);
+    deps.tabsApi.get = vi.fn(
+      async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
+    );
+
+    const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps);
+
+    expect(res).toMatchObject({
+      code: "permission_denied",
+      data: { reason: "agent_window_scope" },
+    });
+    expect(deps.send).not.toHaveBeenCalled();
+  });
+
  it("resets the RefStore on every fresh snapshot", async () => {
    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
    const ctx = await sm.start("aa11");
@@ -1,6 +1,12 @@
 import { describe, expect, it, vi } from "vitest";
 import { SessionManager } from "@/session-manager/manager";
-import { lookupSession, parseBufferedReadBounds } from "../shared";
+import {
+  cdpBlockedUrlReason,
+  enforceCdpAccessibleTarget,
+  lookupSession,
+  parseBufferedReadBounds,
+  resolveCdpAccessibleTargetTab,
+} from "../shared";

 function fakeAgentWindow() {
  return {
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
    });
  });
 });
+
+describe("CDP target URL guard", () => {
+  it("allows ordinary pages and about:blank", () => {
+    expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
+    expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
+    expect(cdpBlockedUrlReason("about:blank")).toBeNull();
+  });
+
+  it("blocks browser and extension internal pages", () => {
+    expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
+    expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
+    expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
+  });
+
+  it("returns a structured permission error before page CDP access", () => {
+    expect(
+      enforceCdpAccessibleTarget(
+        {
+          tabId: 7,
+          windowId: 100,
+          active: true,
+          url: "chrome-extension://other-extension/page.html",
+        },
+        "snapshot",
+      ),
+    ).toMatchObject({
+      code: "permission_denied",
+      data: { reason: "restricted_tab_url" },
+      message: expect.stringContaining("snapshot cannot access tab 7"),
+    });
+  });
+
+  it("falls back from a restricted default active tab to an accessible agent tab", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
+    const ctx = await sm.start("aa11");
+    const tabsApi = {
+      get: vi.fn(),
+      query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
+        if (query.active) {
+          return [
+            {
+              id: 7,
+              windowId: ctx.agentWindowId,
+              active: true,
+              url: "chrome-extension://other-extension/page.html",
+            } as chrome.tabs.Tab,
+          ];
+        }
+        return [
+          {
+            id: 7,
+            windowId: ctx.agentWindowId,
+            active: true,
+            url: "chrome-extension://other-extension/page.html",
+          } as chrome.tabs.Tab,
+          {
+            id: 8,
+            windowId: ctx.agentWindowId,
+            active: false,
+            url: "https://example.test/",
+          } as chrome.tabs.Tab,
+        ];
+      }),
+    };
+
+    await expect(
+      resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
+    ).resolves.toMatchObject({
+      tabId: 8,
+      url: "https://example.test/",
+    });
+  });
+
+  it("does not replace an explicitly requested restricted tab", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
+    const ctx = await sm.start("aa11");
+    const tabsApi = {
+      get: vi.fn(
+        async () =>
+          ({
+            id: 7,
+            windowId: ctx.agentWindowId,
+            active: true,
+            url: "chrome-extension://other-extension/page.html",
+          }) as chrome.tabs.Tab,
+      ),
+      query: vi.fn(),
+    };
+
+    await expect(
+      resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
+    ).resolves.toMatchObject({
+      code: "permission_denied",
+      data: { reason: "restricted_tab_url" },
+    });
+    expect(tabsApi.query).not.toHaveBeenCalled();
+  });
+});