feat(vom): two channels to deal with canvas

fix(dsh-plugin): stabilize skill loading and browser sessions

apps/extension/src/session-manager/__tests__/ref-store.test.ts

@@ -13,6 +13,8 @@ describe("RefStore", () => {
       backendNodeId: 42,
       tabId: 7,
       generation: 0,
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
     expect(s.size()).toBe(1);
     expect(s.isEmpty()).toBe(false);
@@ -48,4 +50,31 @@ describe("RefStore", () => {
       cdpSessionId: "child-session",
     });
   });
+
+  it("preserves explicit capabilities for visual surface refs", () => {
+    const s = new RefStore();
+    s.set("e1", 42, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    expect(s.resolveEntry("e1")).toMatchObject({
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+  });
+
+  it("never grants ordinary interaction to a surface through defaults or caller input", () => {
+    const s = new RefStore();
+    s.set("e1", 42, { tabId: 7, kind: "surface" });
+    s.set("e2", 43, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["interact", "screenshot"],
+    });
+
+    expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
+    expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
+  });
 });

apps/extension/src/session-manager/ref-store.ts

@@ -1,3 +1,5 @@
+import type { Rect } from "@browser-skill/vom";
+
 /**
  * Per-session map from `@e<N>` snapshot refs to a CDP node address.
  *
@@ -12,12 +14,17 @@
  * inside the `SessionContext`.
  */
 export type BackendNodeId = number;
+export type RefCapability = "interact" | "screenshot";
+export type RefTargetKind = "dom" | "surface";
 
 export interface RefEntry {
   backendNodeId: BackendNodeId;
   tabId: number | null;
   frameId?: string;
   cdpSessionId?: string;
+  visibleRect?: Rect;
+  kind: RefTargetKind;
+  capabilities: RefCapability[];
   generation: number;
 }
 
@@ -28,8 +35,19 @@ export type RefInput =
       tabId: number;
       frameId?: string;
       cdpSessionId?: string;
+      visibleRect?: Rect;
+      kind?: RefTargetKind;
+      capabilities?: RefCapability[];
     };
 
+function capabilitiesFor(
+  kind: RefTargetKind,
+  capabilities: RefCapability[] | undefined,
+): RefCapability[] {
+  if (kind === "surface") return ["screenshot"];
+  return capabilities ?? ["interact", "screenshot"];
+}
+
 export class RefStore {
   private readonly map = new Map<string, RefEntry>();
   private generation = 0;
@@ -70,13 +88,20 @@ export class RefStore {
       tabId?: number;
       frameId?: string;
       cdpSessionId?: string;
+      visibleRect?: Rect;
+      kind?: RefTargetKind;
+      capabilities?: RefCapability[];
     } = {},
   ): void {
+    const kind = opts.kind ?? "dom";
     this.map.set(normaliseRef(ref), {
       backendNodeId: id,
       tabId: opts.tabId ?? null,
       ...(opts.frameId ? { frameId: opts.frameId } : {}),
       ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
+      ...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
+      kind,
+      capabilities: capabilitiesFor(kind, opts.capabilities),
       generation: this.generation,
     });
   }
@@ -94,14 +119,20 @@ export class RefStore {
       return {
         backendNodeId: input,
         tabId: null,
+        kind: "dom",
+        capabilities: ["interact", "screenshot"],
         generation: this.generation,
       };
     }
+    const kind = input.kind ?? "dom";
     return {
       backendNodeId: input.backendNodeId,
       tabId: input.tabId,
       ...(input.frameId ? { frameId: input.frameId } : {}),
       ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
+      ...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
+      kind,
+      capabilities: capabilitiesFor(kind, input.capabilities),
       generation: this.generation,
     };
   }

apps/extension/src/tools/__tests__/interaction.test.ts

@@ -123,6 +123,29 @@ describe("handleClick", () => {
     expect(fake.cdp.send).not.toHaveBeenCalled();
   });
 
+  it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+    const fake = makeFakeCdp({});
+
+    const res = await handleClick(
+      sm,
+      { session_id: "aa11", ref: "@e3" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expect(res).toMatchObject({
+      code: "permission_denied",
+      data: { reason: "ref_capability_denied", required_capability: "interact" },
+    });
+    expect(fake.cdp.send).not.toHaveBeenCalled();
+  });
+
   it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");

apps/extension/src/tools/__tests__/observation.test.ts

@@ -300,6 +300,76 @@ describe("handleScreenshot", () => {
     expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
   });
 
+  it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
+      }),
+      "DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
+    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
+      clip?: { width: number; height: number; scale: number };
+    };
+    expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
+    expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
+  });
+
+  it("crops a visual surface screenshot to its observation-time visible region", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 25, y: 30, w: 50, h: 40 },
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
+      }),
+      "DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
+      clip?: { x: number; y: number; width: number; height: number };
+    };
+    expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
+  });
+
   it("returns not_found for unknown ref", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");
@@ -1656,7 +1726,7 @@ describe("buildVomScene", () => {
 
     const scene = buildVomScene(axNodes, captured);
     expect(scene.nodes.find((n) => n.id === 20)).toEqual(
-      expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
+      expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
     );
     const rendered = renderVom(scene);
     expect(rendered.text).toContain('@e1 button "close"');
@@ -1732,7 +1802,7 @@ describe("buildVomScene", () => {
     };
 
     const scene = buildVomScene(axNodes, captured);
-    expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
+    expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
     expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
   });
 
@@ -1801,9 +1871,9 @@ describe("buildVomScene", () => {
 
     const scene = buildVomScene(axNodes, captured);
     expect(scene.nodes.find((n) => n.id === 20)).toEqual(
-      expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
+      expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
     );
-    expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
+    expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
     const rendered = renderVom(scene);
     expect(rendered.text).toContain('@e1 button "收藏"');
     expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
@@ -2131,7 +2201,7 @@ describe("buildVomScene", () => {
     });
 
     expect(scene.surfaces).toEqual([
-      { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
+      { triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
     ]);
     expect(renderVom(scene).text).toContain(
       '@e1 button "image" [hover first: My profile | Sign out]',
@@ -2693,6 +2763,85 @@ describe("handleSnapshot", () => {
     );
   });
 
+  it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Canvas app" },
+      backendDOMNodeId: 100,
+    };
+    const strings = ["body", "canvas", "static", "auto"];
+    const i = (value: string) => strings.indexOf(value);
+    const send = vi.fn(async (_tabId: number, method: string) => {
+      if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
+      }
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0],
+                nodeName: [i("body"), i("canvas")],
+                backendNodeId: [100, 200],
+                attributes: [[], []],
+              },
+              layout: {
+                nodeIndex: [0, 1],
+                styles: [
+                  [i("static"), i("auto"), i("auto")],
+                  [i("static"), i("auto"), i("auto")],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [20, 30, 800, 500],
+                ],
+                paintOrders: [0, 1],
+              },
+            },
+          ],
+        };
+      }
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+    const deps = {
+      cdp: {
+        send: send as unknown as <T = unknown>(
+          tabId: number,
+          method: string,
+          params?: object,
+        ) => Promise<T>,
+        trackSessionTab: vi.fn(),
+      },
+      tabsApi: {
+        get: vi.fn(
+          async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+        ),
+        query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+      },
+      conditionalSurfaceProbe: false,
+    };
+
+    const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
+    if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
+    expect(observed.text).toContain('@e1 surface "canvas visual surface"');
+    expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
+      backendNodeId: 200,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
+    if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
+    expect(snapshot.text).not.toContain("visual surface");
+    expect(snapshot.ref_count).toBe(0);
+  });
+
   it("allows passive snapshots of explicit user-window tabs", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");
@@ -3155,6 +3304,98 @@ describe("handleSnapshot", () => {
     });
   });
 
+  it("decorates live observations with active controlled content", async () => {
+    const strings = [
+      "html",
+      "body",
+      "button",
+      "div",
+      "#text",
+      "role",
+      "tab",
+      "aria-selected",
+      "true",
+      "aria-controls",
+      "reviews-panel",
+      "id",
+      "static",
+      "auto",
+      "visible",
+      "1",
+      "A detailed review",
+    ];
+    const index = (value: string) => strings.indexOf(value);
+    const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
+    const snapshot = {
+      strings,
+      documents: [
+        {
+          nodes: {
+            parentIndex: [-1, 0, 1, 1, 3],
+            nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
+            backendNodeId: [10, 11, 12, 13, 14],
+            attributes: [
+              [],
+              [],
+              [
+                index("role"),
+                index("tab"),
+                index("aria-selected"),
+                index("true"),
+                index("aria-controls"),
+                index("reviews-panel"),
+              ],
+              [index("id"), index("reviews-panel")],
+              [],
+            ],
+            nodeValue: [-1, -1, -1, -1, index("A detailed review")],
+          },
+          layout: {
+            nodeIndex: [0, 1, 2, 3, 4],
+            styles: [style, style, style, style, style],
+            bounds: [
+              [0, 0, 1000, 800],
+              [0, 0, 1000, 800],
+              [20, 20, 120, 40],
+              [20, 80, 500, 200],
+              [20, 80, 200, 20],
+            ],
+            paintOrders: [0, 0, 1, 1, 1],
+          },
+        },
+      ],
+    };
+    const ax: CdpAxNode[] = [
+      {
+        nodeId: "root",
+        backendDOMNodeId: 11,
+        role: { type: "role", value: "RootWebArea" },
+        childIds: ["reviews"],
+      },
+      {
+        nodeId: "reviews",
+        parentId: "root",
+        backendDOMNodeId: 12,
+        role: { type: "role", value: "tab" },
+        name: { type: "computedString", value: "Reviews" },
+        properties: [
+          { name: "selected", value: { value: true } },
+          { name: "controls", value: { value: "reviews-panel" } },
+        ],
+      },
+    ];
+
+    const result = await captureVomObservation(
+      makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
+      4,
+      "https://example.com",
+    );
+
+    expect(result.text).toContain('@e1 tab "Reviews"');
+    expect(result.text).toContain("[§ active: Reviews]");
+    expect(result.text).toContain("A detailed review");
+  });
+
   it("does not leak form secrets through the record-safe observation payload", async () => {
     const secrets = {
       email: "user@example.com",

apps/extension/src/tools/__tests__/snapshot-ref.test.ts

@@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => {
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
     expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
   });
 
@@ -53,6 +57,8 @@ describe("resolveSnapshotRef", () => {
       tabId: 4,
       frameId: "child-frame",
       cdpSessionId: "child-session",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
 
     const expected = {
@@ -60,6 +66,8 @@ describe("resolveSnapshotRef", () => {
       refKey: "e3",
       frameId: "child-frame",
       cdpSessionId: "child-session",
+      kind: "dom" as const,
+      capabilities: ["interact", "screenshot"] as const,
     };
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => {
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
+    });
+  });
+
+  it("rejects an operation outside the ref's declared capabilities", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
+      code: "permission_denied",
+      message: "ref @e3 does not support interact",
+      data: {
+        reason: "ref_capability_denied",
+        required_capability: "interact",
+        kind: "surface",
+      },
     });
   });
 });

apps/extension/src/tools/interaction.ts

@@ -159,7 +159,7 @@ async function resolveBackendNode(
     };
   }
   if (hasRef) {
-    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
+    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
     if (isRpcError(resolved)) return resolved;
     return {
       backendNodeId: resolved.backendNodeId,

apps/extension/src/tools/snapshot-ref.ts

@@ -2,8 +2,9 @@
 // compound CDP node identity via the session RefStore, and build stable
 // `ref_not_found` errors for hard-failure tool paths.
 
+import type { Rect } from "@browser-skill/vom";
 import type { SessionContext } from "@/session-manager/manager";
-import { normaliseRef } from "@/session-manager/ref-store";
+import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
 import type { RpcError } from "@/transport/types";
 import { rpcError } from "./errors";
 
@@ -12,6 +13,9 @@ export interface SnapshotRefLookup {
   refKey: string;
   frameId?: string;
   cdpSessionId?: string;
+  visibleRect?: Rect;
+  kind: RefTargetKind;
+  capabilities: RefCapability[];
 }
 
 function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
@@ -37,6 +41,9 @@ export function lookupSnapshotRef(
     refKey,
     ...(entry.frameId ? { frameId: entry.frameId } : {}),
     ...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
+    ...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
+    kind: entry.kind,
+    capabilities: entry.capabilities,
   };
 }
 
@@ -49,6 +56,7 @@ export function resolveSnapshotRef(
   ctx: SessionContext,
   ref: string,
   tabId: number,
+  requiredCapability?: RefCapability,
 ): SnapshotRefLookup | RpcError {
   const looked = lookupSnapshotRef(ctx, ref, tabId);
   if (looked === null) {
@@ -58,5 +66,13 @@ export function resolveSnapshotRef(
       `ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
     );
   }
+  if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
+    return rpcError(
+      "permission_denied",
+      "ref_capability_denied",
+      `ref ${ref} does not support ${requiredCapability}`,
+      { ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
+    );
+  }
   return looked;
 }

apps/extension/src/tools/vom/__tests__/capture.test.ts

@@ -36,7 +36,7 @@ function fakeSnapshotReply() {
         },
         layout: {
           nodeIndex: [1, 2, 3],
-          // styles columns follow requested computedStyles order: [position, pointer-events]
+          // Legacy fixture: omitted style columns fall back to visible defaults.
           styles: [
             [i("static"), i("auto")],
             [i("fixed"), i("auto")],
@@ -682,6 +682,84 @@ describe("captureViewModel", () => {
     expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
   });
 
+  it("captures painted visibility as DOM fallback evidence", async () => {
+    const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
+    const i = (value: string) => S.indexOf(value);
+    const snapshot = {
+      strings: S,
+      documents: [
+        {
+          nodes: {
+            parentIndex: [-1, 0, 1, 1, 1],
+            nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
+            backendNodeId: [10, 11, 12, 13, 14],
+            attributes: [[], [], [], [], []],
+          },
+          layout: {
+            nodeIndex: [1, 2, 3, 4],
+            styles: [
+              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
+              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
+              [i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
+              [i("static"), i("auto"), i("auto"), i("visible"), i("0")],
+            ],
+            bounds: [
+              [0, 0, 1000, 800],
+              [10, 10, 100, 30],
+              [10, 50, 100, 30],
+              [10, 90, 100, 30],
+            ],
+            paintOrders: [0, 1, 2, 3],
+          },
+        },
+      ],
+    };
+
+    const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
+
+    expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
+    expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
+    expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
+  });
+
+  it("propagates fully transparent ancestor suppression to painted descendants", async () => {
+    const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
+    const i = (value: string) => S.indexOf(value);
+    const snapshot = {
+      strings: S,
+      documents: [
+        {
+          nodes: {
+            parentIndex: [-1, 0, 1, 2],
+            nodeName: [i("html"), i("body"), i("div"), i("canvas")],
+            backendNodeId: [10, 11, 12, 13],
+            attributes: [[], [], [], []],
+          },
+          layout: {
+            nodeIndex: [1, 2, 3],
+            styles: [
+              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
+              [i("static"), i("auto"), i("auto"), i("visible"), i("0")],
+              [i("static"), i("auto"), i("auto"), i("visible"), i("1")],
+            ],
+            bounds: [
+              [0, 0, 1000, 800],
+              [10, 10, 200, 100],
+              [10, 10, 200, 100],
+            ],
+            paintOrders: [0, 1, 2],
+          },
+        },
+      ],
+    };
+
+    const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
+    const canvas = nodes.find((node) => node.backendNodeId === 13);
+
+    expect(canvas?.rendered).toBe(true);
+    expect(canvas?.visuallySuppressed).toBe(true);
+  });
+
   it("subtracts scroll offset to make rects viewport-relative", async () => {
     const cdp = {
       send: vi.fn(async (_t: number, method: string) => {

apps/extension/src/tools/vom/__tests__/frame-capture.test.ts

@@ -132,4 +132,67 @@ describe("captureFrameData", () => {
       ]),
     );
   });
+
+  it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
+    const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
+    const captured: CapturedViewModel = {
+      nodes: [owner],
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      frameNodes: new Map([["main", [owner]]]),
+      rootFrameId: "main",
+      excludedBackendNodeIds: new Set(),
+    };
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method) => {
+        if (method === "Accessibility.enable") return {};
+        if (method === "Accessibility.getFullAXTree") return { nodes: [] };
+        if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
+        if (method === "Page.getLayoutMetrics") {
+          return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
+        }
+        throw new Error(`unexpected root ${method}`);
+      }) as CdpRunner["send"],
+      sendToTarget: vi.fn(async (_target, method) => {
+        if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
+        if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
+        if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
+        if (method === "Accessibility.getFullAXTree") {
+          return {
+            nodes: [
+              {
+                nodeId: "button",
+                backendDOMNodeId: 101,
+                role: { type: "role", value: "button" },
+                name: { type: "computedString", value: "Continue" },
+              },
+            ],
+          };
+        }
+        throw new Error(`unexpected ${method}`);
+      }) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "main",
+        frames: [
+          { frameId: "main", target: { tabId: 4 } },
+          {
+            frameId: "child",
+            parentFrameId: "main",
+            ownerBackendNodeId: 10,
+            target: { tabId: 4, sessionId: "child-session" },
+          },
+        ],
+      })),
+    };
+
+    const documents = await captureFrameData(cdp, 4, captured);
+    const childDocument = documents.find((document) => document.frameId === "child");
+
+    expect(childDocument?.axNodes).toEqual([
+      expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
+    ]);
+    expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
+      expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
+    );
+  });
 });

apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts

@@ -0,0 +1,183 @@
+import { describe, expect, it, vi } from "vitest";
+import type { CdpTarget } from "@/browser-driver/frame-graph";
+import type { CdpRunner } from "../../shared";
+import type { CapturedNode } from "../capture";
+import type { FrameDocument, FrameOwnedAxNode } from "../frame-document";
+import { probeTooltipNames } from "../name-enrichment";
+import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph";
+import { stableIdentifierName } from "../semantic-graph/name-evidence";
+import { frameBackendKey } from "../semantic-graph/types";
+
+function document(
+  target: CdpTarget,
+  backendNodeId: number,
+  accessibleName?: string,
+  frameId = "child",
+  x = 10,
+): FrameDocument<FrameOwnedAxNode> {
+  const node: CapturedNode = {
+    backendNodeId,
+    parentBackendNodeId: null,
+    frameId,
+    tag: "button",
+    attrs: {
+      id: "toolInsertRecord",
+      "data-test-id": "toolInsertRecord",
+      ...(accessibleName ? { "aria-label": accessibleName } : {}),
+    },
+    rect: { x, y: 20, w: 40, h: 30 },
+    localRect: { x: 2, y: 3, w: 40, h: 30 },
+    paintOrder: 1,
+    position: "static",
+    pointerEvents: "auto",
+    cursor: "pointer",
+  };
+  return {
+    frameId,
+    parentFrameId: "main",
+    contextScopeId: "child",
+    target,
+    axNodes: [],
+    domNodes: [node],
+    url: "https://example.com/frame",
+  };
+}
+
+function semantics(documents: FrameDocument<FrameOwnedAxNode>[]) {
+  return resolveSemanticGraph(
+    buildSemanticGraph({
+      documents,
+      viewport: { width: 800, height: 600 },
+      rootFrameId: documents[0]?.frameId,
+    }),
+    { identifierFallback: false },
+  );
+}
+
+describe("tooltip name enrichment", () => {
+  it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => {
+    const target = { tabId: 7, sessionId: "oopif-session" };
+    let hovered = false;
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method, params) => {
+        if (method === "Input.dispatchMouseEvent") {
+          hovered = (params as { x?: number }).x === 30;
+        }
+        return {};
+      }) as CdpRunner["send"],
+      sendToTarget: vi.fn(async (_target, method) => {
+        if (method === "Page.createIsolatedWorld") return { executionContextId: 11 };
+        if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } };
+        if (method === "Runtime.callFunctionOn") {
+          return { result: { value: hovered ? ["插入行"] : [] } };
+        }
+        return {};
+      }) as CdpRunner["sendToTarget"],
+    };
+
+    const documents = [document(target, 501)];
+    const graph = semantics(documents);
+    const names = await probeTooltipNames(cdp, 7, documents, graph);
+
+    expect(names.get(frameBackendKey("child", 501))).toBe("插入行");
+    expect(cdp.sendToTarget).toHaveBeenCalledWith(
+      target,
+      "Page.createIsolatedWorld",
+      expect.objectContaining({ frameId: "child" }),
+    );
+
+    const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length;
+    const targetSendsBeforeCache = vi.mocked(
+      cdp.sendToTarget as NonNullable<CdpRunner["sendToTarget"]>,
+    ).mock.calls.length;
+    const cached = await probeTooltipNames(cdp, 7, documents, graph);
+    expect(cached.get(frameBackendKey("child", 501))).toBe("插入行");
+    expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache);
+    expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache);
+  });
+
+  it("does not probe controls that already have a higher-confidence name", async () => {
+    const cdp: CdpRunner = {
+      send: vi.fn(async () => ({})) as CdpRunner["send"],
+      sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
+    };
+
+    const documents = [document({ tabId: 8 }, 601, "Add a new record")];
+    const names = await probeTooltipNames(cdp, 8, documents, semantics(documents));
+
+    expect(names.size).toBe(0);
+    expect(cdp.send).not.toHaveBeenCalled();
+    expect(cdp.sendToTarget).not.toHaveBeenCalled();
+  });
+
+  it("does not collect supplemental names from form controls", async () => {
+    const cdp: CdpRunner = {
+      send: vi.fn(async () => ({})) as CdpRunner["send"],
+      sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
+    };
+    const inputDocument = document({ tabId: 8 }, 602);
+    inputDocument.domNodes[0] = {
+      ...inputDocument.domNodes[0],
+      tag: "input",
+      attrs: { type: "email" },
+      formValue: "user@example.com",
+      formDefaultValue: "",
+      formState: "filled",
+    };
+
+    const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
+
+    expect(names.size).toBe(0);
+    expect(cdp.send).not.toHaveBeenCalled();
+    expect(cdp.sendToTarget).not.toHaveBeenCalled();
+  });
+
+  it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
+    let hoveredX = -10;
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method, params) => {
+        if (method === "Input.dispatchMouseEvent") {
+          hoveredX = (params as { x: number }).x;
+        }
+        return {};
+      }) as CdpRunner["send"],
+      sendToTarget: vi.fn(async (target, method, params) => {
+        if (method === "Page.createIsolatedWorld") {
+          return { executionContextId: target.sessionId === "left-session" ? 21 : 22 };
+        }
+        if (method === "DOM.resolveNode") {
+          return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } };
+        }
+        if (method === "Runtime.callFunctionOn") {
+          const objectId = (params as { objectId: string }).objectId;
+          const value =
+            objectId === "left" && hoveredX === 30
+              ? ["Left action"]
+              : objectId === "right" && hoveredX === 130
+                ? ["Right action"]
+                : [];
+          return { result: { value } };
+        }
+        return {};
+      }) as CdpRunner["sendToTarget"],
+    };
+    const documents = [
+      document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10),
+      document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110),
+    ];
+
+    const names = await probeTooltipNames(cdp, 9, documents, semantics(documents));
+
+    expect(names.get(frameBackendKey("left", 701))).toBe("Left action");
+    expect(names.get(frameBackendKey("right", 701))).toBe("Right action");
+  });
+});
+
+describe("stable identifier evidence", () => {
+  it("accepts readable action identifiers and rejects framework-generated ids", () => {
+    expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record");
+    expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account");
+    expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined();
+    expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined();
+  });
+});

apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts

@@ -0,0 +1,256 @@
+import type { VomScene } from "@browser-skill/vom";
+import { describe, expect, it } from "vitest";
+import type { CapturedNode } from "../capture";
+import type { CapturedFrameDocument } from "../frame-capture";
+import {
+  clusterRenderedSurfaces,
+  discoverRenderedSurfaces,
+  projectRenderedSurfaces,
+} from "../rendered-surfaces";
+import type { SemanticAxNode } from "../semantic-graph";
+
+function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
+  return {
+    backendNodeId,
+    parentBackendNodeId: null,
+    frameId: "main",
+    ownerFrameBackendNodeId: null,
+    tag: "canvas",
+    attrs: {},
+    rect: { x: 10, y: 20, w: 800, h: 500 },
+    localRect: { x: 10, y: 20, w: 800, h: 500 },
+    paintOrder: 1,
+    position: "static",
+    pointerEvents: "auto",
+    rendered: true,
+    ...overrides,
+  };
+}
+
+function document(
+  domNodes: CapturedNode[],
+  axNodes: SemanticAxNode[] = [],
+): CapturedFrameDocument<SemanticAxNode> {
+  return {
+    frameId: domNodes[0]?.frameId ?? "main",
+    contextScopeId: domNodes[0]?.frameId ?? "main",
+    target: { tabId: 4 },
+    domNodes,
+    axNodes,
+  };
+}
+
+describe("rendered surface discovery", () => {
+  it("finds prominent rendered canvases and excludes hidden descendants", () => {
+    const visible = domNode(2, { parentBackendNodeId: 1 });
+    const hidden = domNode(3, { parentBackendNodeId: 4 });
+    const surfaces = discoverRenderedSurfaces(
+      [
+        document([
+          domNode(1, { tag: "main", rect: null, localRect: null }),
+          visible,
+          domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
+          hidden,
+        ]),
+      ],
+      { width: 1280, height: 720 },
+    );
+
+    expect(surfaces).toHaveLength(1);
+    expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
+  });
+
+  it("keeps tiny canvases as individually addressable exact discoveries", () => {
+    const surfaces = discoverRenderedSurfaces(
+      [document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
+      { width: 1280, height: 720 },
+    );
+    expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
+  });
+
+  it("keeps a surface when the canvas only has partial native accessibility content", () => {
+    const axNodes = [
+      {
+        nodeId: "canvas",
+        backendDOMNodeId: 2,
+        role: { type: "role", value: "presentation" },
+        childIds: ["button"],
+        frameId: "main",
+      },
+      {
+        nodeId: "button",
+        backendDOMNodeId: 3,
+        role: { type: "role", value: "button" },
+        frameId: "main",
+      },
+    ] as SemanticAxNode[];
+    expect(
+      discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
+    ).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
+  });
+
+  it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
+    const canvas = domNode(2, {
+      attrs: {
+        id: "must-not-be-a-label",
+        class: "must-not-be-a-label",
+        "data-testid": "must-not-be-a-label",
+      },
+    });
+
+    expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
+      expect.not.objectContaining({
+        label: expect.anything(),
+      }),
+    ]);
+  });
+
+  it("treats blank explicit Canvas names as unnamed", () => {
+    expect(
+      discoverRenderedSurfaces(
+        [document([domNode(2, { attrs: { "aria-label": "   ", title: "\n\t" } })])],
+        { width: 1280, height: 720 },
+      ),
+    ).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
+  });
+
+  it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
+    const owner = domNode(1, { tag: "div", rect: null, localRect: null });
+    const canvas = domNode(2, { parentBackendNodeId: 1 });
+
+    expect(
+      discoverRenderedSurfaces(
+        [document([owner, canvas])],
+        { width: 1280, height: 720 },
+        new Set([1]),
+      ),
+    ).toEqual([]);
+  });
+
+  it("excludes canvases suppressed by a fully transparent ancestor", () => {
+    const owner = domNode(1, {
+      tag: "div",
+      rect: { x: 0, y: 0, w: 900, h: 600 },
+      visuallySuppressed: true,
+    });
+    const canvas = domNode(2, { parentBackendNodeId: 1 });
+
+    expect(
+      discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
+    ).toEqual([]);
+  });
+
+  it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
+    const owner = domNode(1, {
+      tag: "div",
+      rect: { x: 10, y: 20, w: 100, h: 70 },
+      overflowX: "hidden",
+      overflowY: "hidden",
+    });
+    const partial = domNode(2, {
+      parentBackendNodeId: 1,
+      rect: { x: 55, y: 30, w: 140, h: 60 },
+    });
+    const fullyClipped = domNode(3, {
+      parentBackendNodeId: 1,
+      rect: { x: 140, y: 30, w: 80, h: 50 },
+    });
+
+    expect(
+      discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
+        width: 1280,
+        height: 720,
+      }),
+    ).toEqual([
+      expect.objectContaining({
+        backendNodeId: 2,
+        visibleRect: { x: 55, y: 30, w: 55, h: 60 },
+      }),
+    ]);
+  });
+});
+
+describe("rendered surface clustering and projection", () => {
+  it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
+    const body = domNode(1, { tag: "body", rect: null, localRect: null });
+    const surfaces = discoverRenderedSurfaces(
+      [
+        document([
+          body,
+          domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
+          domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
+        ]),
+      ],
+      { width: 1280, height: 720 },
+    );
+    const groups = clusterRenderedSurfaces(surfaces);
+    const scene: VomScene = {
+      viewport: { width: 1280, height: 720 },
+      nodes: [
+        {
+          id: 10,
+          parentId: null,
+          backendNodeId: 1,
+          frameId: "main",
+          tag: "body",
+          role: "RootWebArea",
+          rect: null,
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    };
+
+    expect(groups).toHaveLength(1);
+    expect(groups[0]?.members).toHaveLength(2);
+    expect(groups[0]?.label).toBeUndefined();
+    expect(
+      projectRenderedSurfaces(
+        scene,
+        [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
+        groups,
+      ),
+    ).toMatchObject({
+      visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
+    });
+  });
+
+  it("does not merge a contained canvas with a larger independent canvas", () => {
+    const groups = clusterRenderedSurfaces([
+      ...discoverRenderedSurfaces(
+        [
+          document([
+            domNode(1, { tag: "main", rect: null, localRect: null }),
+            domNode(2, { parentBackendNodeId: 1 }),
+            domNode(3, {
+              parentBackendNodeId: 1,
+              rect: { x: 30, y: 40, w: 200, h: 100 },
+            }),
+          ]),
+        ],
+        { width: 1280, height: 720 },
+      ),
+    ]);
+
+    expect(groups).toHaveLength(2);
+  });
+
+  it("does not merge coincident canvases owned by different containers", () => {
+    const groups = clusterRenderedSurfaces(
+      discoverRenderedSurfaces(
+        [
+          document([
+            domNode(1, { tag: "section", rect: null, localRect: null }),
+            domNode(2, { tag: "section", rect: null, localRect: null }),
+            domNode(3, { parentBackendNodeId: 1 }),
+            domNode(4, { parentBackendNodeId: 2 }),
+          ]),
+        ],
+        { width: 1280, height: 720 },
+      ),
+    );
+
+    expect(groups).toHaveLength(2);
+  });
+});

apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts

@@ -0,0 +1,782 @@
+import { isVomReferenceNode, renderVom } from "@browser-skill/vom";
+import { describe, expect, it } from "vitest";
+import type { CapturedNode } from "../capture";
+import type { FrameDocument } from "../frame-document";
+import { buildSemanticVomScene, type SemanticAxNode } from "../semantic-graph";
+
+function dom(
+  backendNodeId: number,
+  parentBackendNodeId: number | null,
+  tag: string,
+  attrs: Record<string, string> = {},
+  textContent?: string,
+): CapturedNode {
+  return {
+    backendNodeId,
+    parentBackendNodeId,
+    tag,
+    attrs,
+    rect: { x: 0, y: 0, w: 100, h: 30 },
+    paintOrder: backendNodeId,
+    position: "static",
+    pointerEvents: "auto",
+    ...(textContent ? { textContent } : {}),
+  };
+}
+
+function document(
+  frameId: string,
+  axNodes: SemanticAxNode[],
+  domNodes: CapturedNode[],
+  frame?: Pick<FrameDocument<SemanticAxNode>, "parentFrameId" | "ownerBackendNodeId">,
+): FrameDocument<SemanticAxNode> {
+  return {
+    frameId,
+    contextScopeId: frameId,
+    target: { tabId: 1 },
+    axNodes,
+    domNodes,
+    ...frame,
+  };
+}
+
+describe("semantic VOM graph", () => {
+  it("preserves backend-less AX structure without making it referenceable", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["toolbar"],
+            },
+            {
+              nodeId: "toolbar",
+              parentId: "root",
+              role: { type: "role", value: "toolbar" },
+              name: { type: "computedString", value: "Formatting" },
+              childIds: ["bold", "italic"],
+            },
+            {
+              nodeId: "bold",
+              parentId: "toolbar",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Bold" },
+            },
+            {
+              nodeId: "italic",
+              parentId: "toolbar",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Italic" },
+            },
+          ],
+          [dom(1, null, "body"), dom(2, 1, "button"), dom(3, 1, "button")],
+        ),
+      ],
+    });
+
+    const toolbar = scene.nodes.find((node) => node.role === "toolbar");
+    const buttons = scene.nodes.filter((node) => node.role === "button");
+    expect(toolbar).toEqual(expect.objectContaining({ name: "Formatting", referenceable: false }));
+    expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
+    expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
+    expect(renderVom(scene).refs.map((ref) => ref.backendNodeId)).toEqual([2, 3]);
+  });
+
+  it("resolves missing AX names through one DOM naming pipeline", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["labelled", "titled", "text"],
+            },
+            {
+              nodeId: "labelled",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "button" },
+            },
+            {
+              nodeId: "titled",
+              parentId: "root",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "button" },
+            },
+            {
+              nodeId: "text",
+              parentId: "root",
+              backendDOMNodeId: 4,
+              role: { type: "role", value: "button" },
+            },
+          ],
+          [
+            dom(1, null, "body"),
+            dom(2, 1, "button", { "aria-label": "Add row" }),
+            dom(3, 1, "button", { title: "Settings" }),
+            dom(4, 1, "button"),
+            dom(5, 4, "span", {}, "Save"),
+          ],
+        ),
+      ],
+    });
+
+    expect(scene.nodes.filter((node) => node.role === "button").map((node) => node.name)).toEqual([
+      "Add row",
+      "Settings",
+      "Save",
+    ]);
+  });
+
+  it("aggregates AX text at the semantic text frontier", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["covered", "inline-only", "repeated", "labelled"],
+            },
+            {
+              nodeId: "covered",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "paragraph" },
+              childIds: ["covered-static"],
+            },
+            {
+              nodeId: "covered-static",
+              parentId: "covered",
+              role: { type: "role", value: "StaticText" },
+              name: { type: "computedString", value: "Covered text" },
+              childIds: ["covered-inline"],
+            },
+            {
+              nodeId: "covered-inline",
+              parentId: "covered-static",
+              role: { type: "role", value: "InlineTextBox" },
+              name: { type: "computedString", value: "Covered text" },
+            },
+            {
+              nodeId: "inline-only",
+              parentId: "root",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "paragraph" },
+              childIds: ["inline-leaf"],
+            },
+            {
+              nodeId: "inline-leaf",
+              parentId: "inline-only",
+              role: { type: "role", value: "InlineTextBox" },
+              name: { type: "computedString", value: "Inline fallback" },
+            },
+            {
+              nodeId: "repeated",
+              parentId: "root",
+              backendDOMNodeId: 4,
+              role: { type: "role", value: "paragraph" },
+              childIds: ["first-go", "second-go"],
+            },
+            {
+              nodeId: "first-go",
+              parentId: "repeated",
+              role: { type: "role", value: "StaticText" },
+              name: { type: "computedString", value: "Go" },
+            },
+            {
+              nodeId: "second-go",
+              parentId: "repeated",
+              role: { type: "role", value: "StaticText" },
+              name: { type: "computedString", value: "Go" },
+            },
+            {
+              nodeId: "labelled",
+              parentId: "root",
+              backendDOMNodeId: 5,
+              role: { type: "role", value: "button" },
+              childIds: ["visible-label"],
+            },
+            {
+              nodeId: "visible-label",
+              parentId: "labelled",
+              role: { type: "role", value: "StaticText" },
+              name: { type: "computedString", value: "Visible text" },
+            },
+          ],
+          [
+            dom(1, null, "body"),
+            dom(2, 1, "p"),
+            dom(3, 1, "p"),
+            dom(4, 1, "p"),
+            dom(5, 1, "button", { "aria-label": "Explicit action" }),
+          ],
+        ),
+      ],
+    });
+
+    expect(scene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Covered text");
+    expect(scene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Inline fallback");
+    expect(scene.nodes.find((node) => node.backendNodeId === 4)?.name).toBe("Go Go");
+    expect(scene.nodes.find((node) => node.backendNodeId === 5)?.name).toBe("Explicit action");
+    expect(renderVom(scene).text).not.toContain("Covered text Covered text");
+  });
+
+  it("uses stable identifiers only as a final name fallback", () => {
+    const base = document(
+      "main",
+      [
+        {
+          nodeId: "root",
+          backendDOMNodeId: 1,
+          role: { type: "role", value: "RootWebArea" },
+          childIds: ["fallback", "labelled", "container"],
+        },
+        {
+          nodeId: "fallback",
+          parentId: "root",
+          backendDOMNodeId: 2,
+          role: { type: "role", value: "button" },
+        },
+        {
+          nodeId: "labelled",
+          parentId: "root",
+          backendDOMNodeId: 3,
+          role: { type: "role", value: "button" },
+          name: { type: "computedString", value: "Localized name" },
+        },
+        {
+          nodeId: "container",
+          parentId: "root",
+          backendDOMNodeId: 4,
+          role: { type: "role", value: "generic" },
+        },
+      ],
+      [
+        dom(1, null, "body"),
+        dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }),
+        dom(3, 1, "button", { id: "toolIgnoredFallback" }),
+        dom(4, 1, "div", { id: "toolbarContainer" }),
+      ],
+    );
+    const staticScene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [base],
+    });
+    const enrichedScene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [base],
+      supplementalNames: new Map([["main\u00002", "插入行"]]),
+    });
+
+    expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record");
+    expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name");
+    expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined();
+    expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行");
+    expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe(
+      "Localized name",
+    );
+  });
+
+  it("isolates equal backend node ids in sibling frames", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 1000, height: 800 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["owner-a", "owner-b"],
+            },
+            {
+              nodeId: "owner-a",
+              parentId: "root",
+              backendDOMNodeId: 10,
+              role: { type: "role", value: "Iframe" },
+            },
+            {
+              nodeId: "owner-b",
+              parentId: "root",
+              backendDOMNodeId: 20,
+              role: { type: "role", value: "Iframe" },
+            },
+          ],
+          [dom(1, null, "body"), dom(10, 1, "iframe"), dom(20, 1, "iframe")],
+        ),
+        document(
+          "frame-a",
+          [
+            {
+              nodeId: "button-a",
+              backendDOMNodeId: 5,
+              role: { type: "role", value: "button" },
+              childIds: ["text-a"],
+            },
+            {
+              nodeId: "text-a",
+              parentId: "button-a",
+              role: { type: "role", value: "StaticText" },
+              name: { type: "computedString", value: "Accept" },
+              childIds: ["inline-a"],
+            },
+            {
+              nodeId: "inline-a",
+              parentId: "text-a",
+              role: { type: "role", value: "InlineTextBox" },
+              name: { type: "computedString", value: "Accept" },
+            },
+          ],
+          [dom(5, null, "button")],
+          { parentFrameId: "main", ownerBackendNodeId: 10 },
+        ),
+        document(
+          "frame-b",
+          [
+            {
+              nodeId: "button-b",
+              backendDOMNodeId: 5,
+              role: { type: "role", value: "button" },
+              childIds: ["text-b"],
+            },
+            {
+              nodeId: "text-b",
+              parentId: "button-b",
+              role: { type: "role", value: "StaticText" },
+              name: { type: "computedString", value: "Decline" },
+              childIds: ["inline-b"],
+            },
+            {
+              nodeId: "inline-b",
+              parentId: "text-b",
+              role: { type: "role", value: "InlineTextBox" },
+              name: { type: "computedString", value: "Decline" },
+            },
+          ],
+          [dom(5, null, "button")],
+          { parentFrameId: "main", ownerBackendNodeId: 20 },
+        ),
+      ],
+    });
+
+    const frameButtons = scene.nodes.filter((node) => node.backendNodeId === 5);
+    const owners = new Map(
+      scene.nodes
+        .filter((node) => node.backendNodeId === 10 || node.backendNodeId === 20)
+        .map((node) => [node.backendNodeId, node.id]),
+    );
+    expect(frameButtons.map((node) => node.frameId)).toEqual(["frame-a", "frame-b"]);
+    expect(frameButtons.map((node) => node.name)).toEqual(["Accept", "Decline"]);
+    expect(frameButtons.map((node) => node.parentId)).toEqual([owners.get(10), owners.get(20)]);
+    expect(renderVom(scene).refs.map((ref) => [ref.frameId, ref.backendNodeId])).toEqual([
+      ["frame-a", 5],
+      ["frame-b", 5],
+    ]);
+  });
+
+  it("requires render evidence before promoting DOM fallback semantics", () => {
+    const hiddenOwner = {
+      ...dom(10, 1, "iframe", { hidden: "" }),
+      rect: null,
+      localRect: null,
+      rendered: false,
+    };
+    const hiddenButton = {
+      ...dom(2, 1, "button", { "aria-label": "Hidden action" }),
+      rendered: false,
+    };
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+            },
+          ],
+          [dom(1, null, "body"), hiddenButton, hiddenOwner],
+        ),
+        document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
+          parentFrameId: "main",
+          ownerBackendNodeId: 10,
+        }),
+      ],
+    });
+
+    expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
+    expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
+    expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
+    expect(renderVom(scene).refs).toEqual([]);
+  });
+
+  it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [],
+          [dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
+        ),
+      ],
+    });
+
+    expect(renderVom(scene).refs).toEqual([
+      expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
+    ]);
+  });
+
+  it("lets explicit DOM structure override ignored AX metadata", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["toolbar"],
+            },
+            {
+              nodeId: "toolbar",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              ignored: true,
+              role: { type: "role", value: "generic" },
+              childIds: ["wrapper"],
+            },
+            {
+              nodeId: "wrapper",
+              parentId: "toolbar",
+              backendDOMNodeId: 5,
+              role: { type: "role", value: "generic" },
+              childIds: ["bold", "italic"],
+            },
+            {
+              nodeId: "bold",
+              parentId: "wrapper",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Bold" },
+            },
+            {
+              nodeId: "italic",
+              parentId: "wrapper",
+              backendDOMNodeId: 4,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Italic" },
+            },
+          ],
+          [
+            dom(1, null, "body"),
+            dom(2, 1, "div", { role: "toolbar" }),
+            dom(5, 2, "div"),
+            dom(3, 5, "button"),
+            dom(4, 5, "button"),
+          ],
+        ),
+      ],
+    });
+
+    const toolbar = scene.nodes.find((node) => node.backendNodeId === 2);
+    const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
+    expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" }));
+    expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
+    expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
+    expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
+  });
+
+  it("infers an unnamed group only from independent interaction branches", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["wrapper"],
+            },
+            {
+              nodeId: "wrapper",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "generic" },
+              childIds: ["first", "second"],
+            },
+            {
+              nodeId: "first",
+              parentId: "wrapper",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "First" },
+            },
+            {
+              nodeId: "second",
+              parentId: "wrapper",
+              backendDOMNodeId: 4,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Second" },
+            },
+          ],
+          [dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")],
+        ),
+      ],
+    });
+
+    const group = scene.nodes.find((node) => node.backendNodeId === 2);
+    const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
+    expect(group).toEqual(expect.objectContaining({ role: "group" }));
+    expect(group?.name).toBeUndefined();
+    expect(group && isVomReferenceNode(group)).toBe(false);
+    expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
+
+    const rendered = renderVom(scene);
+    const lines = rendered.text.split("\n");
+    const groupLine = lines.findIndex((line) => line.trim() === "group");
+    expect(groupLine).toBeGreaterThanOrEqual(0);
+    for (const ref of rendered.refs) {
+      expect(ref.line).toBeGreaterThan(groupLine);
+      expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
+    }
+  });
+
+  it("never promotes a form value into its semantic name through supplemental evidence", () => {
+    const secret = "user@example.com";
+    const input = {
+      ...dom(2, 1, "input", { type: "email" }),
+      formValue: secret,
+      formDefaultValue: "",
+      formState: "filled" as const,
+    };
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      supplementalNames: new Map([["main\u00002", secret]]),
+      identifierFallback: false,
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["email"],
+            },
+            {
+              nodeId: "email",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "textbox" },
+              value: { value: secret },
+            },
+          ],
+          [dom(1, null, "body"), input],
+        ),
+      ],
+    });
+
+    const field = scene.nodes.find((node) => node.backendNodeId === 2);
+    expect(field?.name).toBeUndefined();
+    const rendered = renderVom(scene, { redactValues: true });
+    expect(rendered.text).toContain("•••");
+    expect(JSON.stringify(rendered)).not.toContain(secret);
+  });
+
+  it("keeps uncertain single-branch wrappers flat", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["wrapper"],
+            },
+            {
+              nodeId: "wrapper",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "generic" },
+              childIds: ["button"],
+            },
+            {
+              nodeId: "button",
+              parentId: "wrapper",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Only action" },
+            },
+          ],
+          [dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")],
+        ),
+      ],
+    });
+
+    expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
+    expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe(
+      scene.nodes.find((node) => node.backendNodeId === 1)?.id,
+    );
+  });
+
+  it("does not infer across mixed content boundaries", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["wrapper"],
+            },
+            {
+              nodeId: "wrapper",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "generic" },
+              childIds: ["first", "second", "paragraph"],
+            },
+            {
+              nodeId: "first",
+              parentId: "wrapper",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "First" },
+            },
+            {
+              nodeId: "second",
+              parentId: "wrapper",
+              backendDOMNodeId: 4,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Second" },
+            },
+            {
+              nodeId: "paragraph",
+              parentId: "wrapper",
+              backendDOMNodeId: 5,
+              role: { type: "role", value: "paragraph" },
+            },
+          ],
+          [
+            dom(1, null, "body"),
+            dom(2, 1, "div"),
+            dom(3, 2, "button"),
+            dom(4, 2, "button"),
+            dom(5, 2, "p", {}, "Description"),
+          ],
+        ),
+      ],
+    });
+
+    expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
+  });
+
+  it("keeps only the innermost reliable inferred group", () => {
+    const scene = buildSemanticVomScene({
+      viewport: { width: 800, height: 600 },
+      rootFrameId: "main",
+      documents: [
+        document(
+          "main",
+          [
+            {
+              nodeId: "root",
+              backendDOMNodeId: 1,
+              role: { type: "role", value: "RootWebArea" },
+              childIds: ["outer"],
+            },
+            {
+              nodeId: "outer",
+              parentId: "root",
+              backendDOMNodeId: 2,
+              role: { type: "role", value: "generic" },
+              childIds: ["inner", "third"],
+            },
+            {
+              nodeId: "inner",
+              parentId: "outer",
+              backendDOMNodeId: 3,
+              role: { type: "role", value: "generic" },
+              childIds: ["first", "second"],
+            },
+            {
+              nodeId: "first",
+              parentId: "inner",
+              backendDOMNodeId: 4,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "First" },
+            },
+            {
+              nodeId: "second",
+              parentId: "inner",
+              backendDOMNodeId: 5,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Second" },
+            },
+            {
+              nodeId: "third",
+              parentId: "outer",
+              backendDOMNodeId: 6,
+              role: { type: "role", value: "button" },
+              name: { type: "computedString", value: "Third" },
+            },
+          ],
+          [
+            dom(1, null, "body"),
+            dom(2, 1, "div"),
+            dom(3, 2, "div"),
+            dom(4, 3, "button"),
+            dom(5, 3, "button"),
+            dom(6, 2, "button"),
+          ],
+        ),
+      ],
+    });
+
+    const groups = scene.nodes.filter((node) => node.role === "group");
+    expect(groups.map((node) => node.backendNodeId)).toEqual([3]);
+    expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id);
+  });
+});

apps/extension/src/tools/vom/capture.ts

@@ -1,16 +1,25 @@
 // CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
 // → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
 // raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
-// shared string table); we request exactly three computed styles so the
+// shared string table); requested computed styles are decoded through
-// `styles` columns are [position, pointer-events, cursor] in that order.
+// STYLE_COL so capture and visibility policy share one explicit contract.
 
 import type { Rect, Viewport } from "@browser-skill/vom";
 import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
 import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
 import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
 import type { CdpRunner } from "../shared";
+import { clearHover, waitForHover } from "./hover-perception";
 
-const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
+const REQUESTED_STYLES = [
+  "position",
+  "pointer-events",
+  "cursor",
+  "visibility",
+  "opacity",
+  "overflow-x",
+  "overflow-y",
+] as const;
 const STYLE_COL = Object.fromEntries(
   REQUESTED_STYLES.map((name, index) => [name, index]),
 ) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -38,6 +47,16 @@ export interface CapturedNode {
    * `textContent`: the live parser always sets it, hand-built fixtures may not.
    */
   cursor?: string;
+  overflowX?: string;
+  overflowY?: string;
+  /** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
+  visuallySuppressed?: boolean;
+  /**
+   * Whether the live DOM snapshot provides a painted, non-hidden box for this
+   * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
+   * nodes remain authoritative even when they are outside the viewport.
+   */
+  rendered?: boolean;
   textContent?: string;
   formState?: "empty" | "filled" | "default";
   formValue?: string;
@@ -460,36 +479,6 @@ function hoverStateExpression(): string {
   })()`;
 }
 
-async function wait(ms: number, signal?: AbortSignal): Promise<void> {
-  throwIfAborted(signal);
-  await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      signal?.removeEventListener("abort", onAbort);
-      resolve();
-    }, ms);
-    const onAbort = () => {
-      clearTimeout(timer);
-      signal?.removeEventListener("abort", onAbort);
-      reject(captureAbortError());
-    };
-    signal?.addEventListener("abort", onAbort, { once: true });
-  });
-}
-
-async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
-  await cdp
-    .send(tabId, "Input.dispatchMouseEvent", {
-      type: "mouseMoved",
-      x: -10,
-      y: -10,
-    })
-    .catch(() =>
-      cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
-        // best effort
-      }),
-    );
-}
-
 function capturedText(node: CapturedNode): string | undefined {
   const value =
     node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
@@ -669,7 +658,7 @@ async function probeHoverSurfaces(
         try {
           await clearHover(cdp, tabId);
           throwIfAborted(options.signal);
-          await wait(HOVER_SETTLE_MS, options.signal);
+          await waitForHover(HOVER_SETTLE_MS, options.signal);
           const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
             expression: hoverStateExpression(),
             returnByValue: true,
@@ -684,7 +673,7 @@ async function probeHoverSurfaces(
             y: candidate.y,
           });
           throwIfAborted(options.signal);
-          await wait(HOVER_SETTLE_MS, options.signal);
+          await waitForHover(HOVER_SETTLE_MS, options.signal);
           const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
             expression: hoverStateExpression(),
             returnByValue: true,
@@ -817,6 +806,10 @@ function parseDocumentNodes(
   const posCol = STYLE_COL.position;
   const peCol = STYLE_COL["pointer-events"];
   const cursorCol = STYLE_COL.cursor;
+  const visibilityCol = STYLE_COL.visibility;
+  const opacityCol = STYLE_COL.opacity;
+  const overflowXCol = STYLE_COL["overflow-x"];
+  const overflowYCol = STYLE_COL["overflow-y"];
   const inputValues = sparseIndexMap(dn.inputValue);
   const textValues = sparseIndexMap(dn.textValue);
   const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -842,6 +835,7 @@ function parseDocumentNodes(
 
   // Build CapturedNodes.
   const nodes: CapturedNode[] = [];
+  const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
   for (let n = 0; n < count; n++) {
     if (excluded[n]) continue;
     const backendNodeId = dn.backendNodeId[n];
@@ -861,6 +855,11 @@ function parseDocumentNodes(
     let position = "static";
     let pointerEvents = "auto";
     let cursor = "auto";
+    let overflowX = "visible";
+    let overflowY = "visible";
+    let rendered = false;
+    let visuallySuppressed =
+      parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
     const li = layoutByNode.get(n);
     if (li !== undefined) {
       const b = dl?.bounds?.[li];
@@ -881,7 +880,21 @@ function parseDocumentNodes(
       position = str(strings, styleRow[posCol]) || "static";
       pointerEvents = str(strings, styleRow[peCol]) || "auto";
       cursor = str(strings, styleRow[cursorCol]) || "auto";
+      const visibility = str(strings, styleRow[visibilityCol]) || "visible";
+      const opacity = str(strings, styleRow[opacityCol]) || "1";
+      overflowX = str(strings, styleRow[overflowXCol]) || "visible";
+      overflowY = str(strings, styleRow[overflowYCol]) || "visible";
+      visuallySuppressed ||=
+        visibility === "hidden" ||
+        visibility === "collapse" ||
+        (Number.parseFloat(opacity) || 0) <= 0;
+      rendered =
+        localRect !== null &&
+        visibility !== "hidden" &&
+        visibility !== "collapse" &&
+        (Number.parseFloat(opacity) || 0) > 0;
     }
+    visuallySuppressedByNodeIndex[n] = visuallySuppressed;
 
     // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
     // geometry and are never queried by callers.
@@ -916,6 +929,10 @@ function parseDocumentNodes(
       position,
       pointerEvents,
       cursor,
+      overflowX,
+      overflowY,
+      visuallySuppressed,
+      rendered,
       textContent,
       ...(formValue !== undefined ? { formValue } : {}),
       ...(tag === "input" || tag === "textarea"
@@ -1020,7 +1037,13 @@ export async function captureViewModel(
   options: CaptureViewModelOptions = {},
 ): Promise<CapturedViewModel> {
   throwIfAborted(options.signal);
-  const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
+  let metrics: LayoutMetricsReply = {};
+  try {
+    metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
+  } catch (error) {
+    if (isAbortError(error)) throw error;
+    console.debug("[bsk capture] layout metrics unavailable", error);
+  }
   throwIfAborted(options.signal);
   const dpr = devicePixelRatio(metrics);
   const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};

apps/extension/src/tools/vom/frame-capture.ts

@@ -33,25 +33,19 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
 
 function transformFrameNodes(
   nodes: CapturedNode[],
-  projection: GeometryProjection,
+  projection: GeometryProjection | null,
 ): CapturedNode[] {
   return nodes.map((node) => ({
     ...node,
-    rect: (() => {
+    rect: projection
-      const bounds = projectRectToViewport(node.rect, projection);
+      ? (() => {
-      return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
+          const bounds = projectRectToViewport(node.rect, projection);
-    })(),
+          return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
+        })()
+      : null,
   }));
 }
 
-function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
-  if (frame.ownerBackendNodeId === undefined) return undefined;
-  const parentNodes = frame.parentFrameId
-    ? captured.frameNodes?.get(frame.parentFrameId)
-    : captured.nodes;
-  return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
-}
-
 async function captureMissingFrameDocuments(
   cdp: CdpRunner,
   tabId: number,
@@ -65,14 +59,20 @@ async function captureMissingFrameDocuments(
 
   for (const frame of graph.frames) {
     if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
-    const owner = frameOwnerNode(captured, frame);
-    if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
     try {
       const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
         signal,
       });
-      const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
+      let projection: GeometryProjection | null = null;
-      if (!projection) continue;
+      try {
+        projection = await resolveFrameProjection(cdp, graph, frame.frameId);
+      } catch (err) {
+        if (isAbortError(err)) throw err;
+        console.debug("[bsk observation] child frame geometry projection failed", {
+          frameId: frame.frameId,
+          err,
+        });
+      }
       const rootFrameId = child.rootFrameId ?? frame.frameId;
       const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
       if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
@@ -85,11 +85,13 @@ async function captureMissingFrameDocuments(
       for (const [childFrameId, nodes] of childFrames) {
         captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
       }
-      captured.iframeNodes.set(
+      if (frame.ownerBackendNodeId !== undefined) {
-        frame.ownerBackendNodeId,
+        captured.iframeNodes.set(
-        captured.frameNodes.get(frame.frameId) ?? [],
+          frame.ownerBackendNodeId,
-      );
+          captured.frameNodes.get(frame.frameId) ?? [],
-      captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
+        );
+        captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
+      }
       if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
       for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
         captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);

apps/extension/src/tools/vom/hover-perception.ts

@@ -0,0 +1,37 @@
+import type { CdpRunner } from "../shared";
+
+function abortError(): Error {
+  const error = new Error("hover perception aborted");
+  error.name = "AbortError";
+  return error;
+}
+
+export async function waitForHover(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) throw abortError();
+  await new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener("abort", onAbort);
+      reject(abortError());
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
+  await cdp
+    .send(tabId, "Input.dispatchMouseEvent", {
+      type: "mouseMoved",
+      x: -10,
+      y: -10,
+    })
+    .catch(() =>
+      cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
+        // Best effort cleanup.
+      }),
+    );
+}

apps/extension/src/tools/vom/name-enrichment.ts

@@ -0,0 +1,296 @@
+import { isVomReferenceNode } from "@browser-skill/vom";
+import { type CdpRunner, sendToCdpTarget } from "../shared";
+import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
+import { clearHover, waitForHover } from "./hover-perception";
+import { stableIdentifierName } from "./semantic-graph/name-evidence";
+import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
+
+const MAX_TOOLTIP_CANDIDATES = 8;
+const MAX_TOOLTIP_PROBE_MS = 2_400;
+const TOOLTIP_SETTLE_MS = 250;
+const CACHE_TTL_MS = 5 * 60_000;
+const MAX_CACHE_ENTRIES = 500;
+const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
+
+interface TooltipCacheEntry {
+  name: string;
+  expiresAt: number;
+}
+
+interface TooltipCandidate<T extends FrameOwnedAxNode> {
+  document: FrameDocument<T>;
+  backendNodeId: number;
+  x: number;
+  y: number;
+  stableIdentifier: boolean;
+}
+
+export interface TooltipNameProbeOptions {
+  signal?: AbortSignal;
+  hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
+}
+
+const tooltipCache = new Map<string, TooltipCacheEntry>();
+
+function cacheKey<T extends FrameOwnedAxNode>(
+  document: FrameDocument<T>,
+  backendNodeId: number,
+): string {
+  return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
+}
+
+function cachedName<T extends FrameOwnedAxNode>(
+  document: FrameDocument<T>,
+  backendNodeId: number,
+): string | undefined {
+  const key = cacheKey(document, backendNodeId);
+  const entry = tooltipCache.get(key);
+  if (!entry) return undefined;
+  if (entry.expiresAt <= Date.now()) {
+    tooltipCache.delete(key);
+    return undefined;
+  }
+  return entry.name;
+}
+
+function storeName<T extends FrameOwnedAxNode>(
+  document: FrameDocument<T>,
+  backendNodeId: number,
+  name: string,
+): void {
+  if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
+    const oldest = tooltipCache.keys().next().value;
+    if (oldest !== undefined) tooltipCache.delete(oldest);
+  }
+  tooltipCache.set(cacheKey(document, backendNodeId), {
+    name,
+    expiresAt: Date.now() + CACHE_TTL_MS,
+  });
+}
+
+function tooltipCollectorFunction(): string {
+  return `function() {
+    const ownRoot = this && typeof this.getRootNode === "function" ? this.getRootNode() : null;
+    const roots = ownRoot && ownRoot !== document ? [ownRoot, document] : [document];
+    const items = [];
+    const seen = new Set();
+    for (const root of roots) {
+      for (const el of Array.from(root.querySelectorAll("[role='tooltip']"))) {
+        const style = getComputedStyle(el);
+        const rect = el.getBoundingClientRect();
+        if (rect.width <= 0 || rect.height <= 0) continue;
+        if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
+        const text = String(el.getAttribute("aria-label") || el.getAttribute("title") || el.textContent || "")
+          .replace(/\\s+/g, " ")
+          .trim();
+        if (!text || text.length > 160 || seen.has(text)) continue;
+        seen.add(text);
+        items.push(text);
+      }
+    }
+    return items.slice(0, 20);
+  }`;
+}
+
+function cleanTooltipDiff(before: string[], after: string[]): string | undefined {
+  const existing = new Set(before.map((value) => value.toLowerCase()));
+  const added = [...new Set(after.map((value) => value.replace(/\s+/g, " ").trim()))].filter(
+    (value) => value && !existing.has(value.toLowerCase()),
+  );
+  return added.length === 1 ? added[0] : undefined;
+}
+
+function candidates<T extends FrameOwnedAxNode>(
+  documents: FrameDocument<T>[],
+  graph: ResolvedSemanticGraph,
+): TooltipCandidate<T>[] {
+  const documentByFrame = new Map(documents.map((document) => [document.frameId, document]));
+  const byFrame = new Map<string, TooltipCandidate<T>[]>();
+  for (const node of graph.nodes.values()) {
+    if (
+      node.vom.name ||
+      FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") ||
+      !node.referenceable ||
+      node.backendNodeId === undefined ||
+      !isVomReferenceNode({
+        ...node.vom,
+        id: 0,
+        parentId: null,
+        backendNodeId: node.backendNodeId,
+        frameId: node.frameId,
+        referenceable: true,
+      })
+    ) {
+      continue;
+    }
+    const document = documentByFrame.get(node.frameId);
+    const rect = node.dom?.rect;
+    if (!document || !rect || rect.w <= 0 || rect.h <= 0) continue;
+    if (
+      rect.x + rect.w <= 0 ||
+      rect.y + rect.h <= 0 ||
+      rect.x >= graph.viewport.width ||
+      rect.y >= graph.viewport.height
+    ) {
+      continue;
+    }
+    if (
+      node.vom.disabled ||
+      node.vom.inert ||
+      node.vom.pointerEvents === "none" ||
+      node.vom.sensitive
+    ) {
+      continue;
+    }
+    const frameCandidates = byFrame.get(node.frameId) ?? [];
+    frameCandidates.push({
+      document,
+      backendNodeId: node.backendNodeId,
+      x: (Math.max(0, rect.x) + Math.min(graph.viewport.width, rect.x + rect.w)) / 2,
+      y: (Math.max(0, rect.y) + Math.min(graph.viewport.height, rect.y + rect.h)) / 2,
+      stableIdentifier: stableIdentifierName(node.dom?.attrs ?? {}) !== undefined,
+    });
+    byFrame.set(node.frameId, frameCandidates);
+  }
+
+  const out: TooltipCandidate<T>[] = [];
+  for (const stableIdentifier of [true, false]) {
+    const queues = documents.map((document) =>
+      (byFrame.get(document.frameId) ?? []).filter(
+        (candidate) => candidate.stableIdentifier === stableIdentifier,
+      ),
+    );
+    for (let index = 0; queues.some((queue) => index < queue.length); index += 1) {
+      for (const queue of queues) {
+        const candidate = queue[index];
+        if (candidate) out.push(candidate);
+      }
+    }
+  }
+  return out;
+}
+
+async function executionContextId<T extends FrameOwnedAxNode>(
+  cdp: CdpRunner,
+  document: FrameDocument<T>,
+): Promise<number | undefined> {
+  const result = await sendToCdpTarget<{ executionContextId?: number }>(
+    cdp,
+    document.target,
+    "Page.createIsolatedWorld",
+    {
+      frameId: document.frameId,
+      worldName: "bsk-vom-tooltip",
+      grantUniveralAccess: false,
+    },
+  );
+  return result.executionContextId;
+}
+
+async function resolvedObjectId<T extends FrameOwnedAxNode>(
+  cdp: CdpRunner,
+  document: FrameDocument<T>,
+  contextId: number,
+  backendNodeId: number,
+): Promise<string | undefined> {
+  const result = await sendToCdpTarget<{
+    object?: { objectId?: string };
+  }>(cdp, document.target, "DOM.resolveNode", {
+    backendNodeId,
+    executionContextId: contextId,
+  });
+  return result.object?.objectId;
+}
+
+async function tooltipTexts<T extends FrameOwnedAxNode>(
+  cdp: CdpRunner,
+  document: FrameDocument<T>,
+  objectId: string,
+): Promise<string[]> {
+  const result = await sendToCdpTarget<{
+    result?: { value?: string[] };
+  }>(cdp, document.target, "Runtime.callFunctionOn", {
+    functionDeclaration: tooltipCollectorFunction(),
+    objectId,
+    returnByValue: true,
+  });
+  return result.result?.value ?? [];
+}
+
+async function releaseObject<T extends FrameOwnedAxNode>(
+  cdp: CdpRunner,
+  document: FrameDocument<T>,
+  objectId: string,
+): Promise<void> {
+  await sendToCdpTarget(cdp, document.target, "Runtime.releaseObject", { objectId }).catch(
+    () => undefined,
+  );
+}
+
+export async function probeTooltipNames<T extends FrameOwnedAxNode>(
+  cdp: CdpRunner,
+  tabId: number,
+  documents: FrameDocument<T>[],
+  graph: ResolvedSemanticGraph,
+  options: TooltipNameProbeOptions = {},
+): Promise<Map<string, string>> {
+  const names = new Map<string, string>();
+  const pending: TooltipCandidate<T>[] = [];
+  for (const candidate of candidates(documents, graph)) {
+    const cached = cachedName(candidate.document, candidate.backendNodeId);
+    if (cached) {
+      names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
+    } else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
+      pending.push(candidate);
+    }
+  }
+  if (pending.length === 0) return names;
+
+  const started = Date.now();
+  const contexts = new Map<string, number>();
+  await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
+  try {
+    for (const candidate of pending) {
+      if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
+      if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
+      let objectId: string | undefined;
+      try {
+        let contextId = contexts.get(candidate.document.frameId);
+        if (contextId === undefined) {
+          contextId = await executionContextId(cdp, candidate.document);
+          if (contextId === undefined) continue;
+          contexts.set(candidate.document.frameId, contextId);
+        }
+        objectId = await resolvedObjectId(
+          cdp,
+          candidate.document,
+          contextId,
+          candidate.backendNodeId,
+        );
+        if (!objectId) continue;
+        await clearHover(cdp, tabId);
+        const before = await tooltipTexts(cdp, candidate.document, objectId);
+        await cdp.send(tabId, "Input.dispatchMouseEvent", {
+          type: "mouseMoved",
+          x: candidate.x,
+          y: candidate.y,
+        });
+        await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
+        const after = await tooltipTexts(cdp, candidate.document, objectId);
+        const name = cleanTooltipDiff(before, after);
+        if (!name) continue;
+        names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
+        storeName(candidate.document, candidate.backendNodeId, name);
+      } catch (error) {
+        if (error instanceof Error && error.name === "AbortError") throw error;
+      } finally {
+        if (objectId) await releaseObject(cdp, candidate.document, objectId);
+        await clearHover(cdp, tabId);
+      }
+    }
+  } finally {
+    await clearHover(cdp, tabId);
+    await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
+  }
+  return names;
+}

apps/extension/src/tools/vom/rendered-surfaces/cluster.ts

@@ -0,0 +1,65 @@
+import type { Rect } from "@browser-skill/vom";
+import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
+
+const STACK_IOU_RATIO = 0.9;
+const STACK_SIZE_RATIO = 0.9;
+
+function area(rect: Rect): number {
+  return Math.max(0, rect.w) * Math.max(0, rect.h);
+}
+
+function intersectionArea(a: Rect, b: Rect): number {
+  const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
+  const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
+  return width * height;
+}
+
+function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
+  if (a.frameId !== b.frameId) return false;
+  if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
+  const aArea = area(a.visibleRect);
+  const bArea = area(b.visibleRect);
+  const intersection = intersectionArea(a.visibleRect, b.visibleRect);
+  const union = aArea + bArea - intersection;
+  const widthRatio =
+    Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
+  const heightRatio =
+    Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
+  return (
+    union > 0 &&
+    intersection / union >= STACK_IOU_RATIO &&
+    widthRatio >= STACK_SIZE_RATIO &&
+    heightRatio >= STACK_SIZE_RATIO
+  );
+}
+
+function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
+  const representative = [...members].sort(
+    (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
+  )[0];
+  return {
+    frameId: representative.frameId,
+    members,
+    representative,
+    label: members.map((surface) => surface.label).find((label) => label !== undefined),
+  };
+}
+
+export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
+  const groups: RenderedSurface[][] = [];
+  for (const surface of surfaces) {
+    const matching = groups.find((group) =>
+      group.some((member) => sameVisualStack(member, surface)),
+    );
+    if (matching) matching.push(surface);
+    else groups.push([surface]);
+  }
+  return groups
+    .map(toGroup)
+    .sort(
+      (a, b) =>
+        a.frameId.localeCompare(b.frameId) ||
+        a.representative.visibleRect.y - b.representative.visibleRect.y ||
+        a.representative.visibleRect.x - b.representative.visibleRect.x,
+    );
+}

apps/extension/src/tools/vom/rendered-surfaces/discovery.ts

@@ -0,0 +1,113 @@
+import type { Rect, Viewport } from "@browser-skill/vom";
+import type { CapturedNode } from "../capture";
+import type { CapturedFrameDocument } from "../frame-capture";
+import type { SemanticAxNode } from "../semantic-graph";
+import type { RenderedSurface } from "./types";
+
+function clean(value: string | undefined): string | undefined {
+  const normalized = value?.replace(/\s+/g, " ").trim();
+  return normalized || undefined;
+}
+
+function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
+  return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
+}
+
+function intersectRect(a: Rect, b: Rect): Rect | null {
+  const x = Math.max(a.x, b.x);
+  const y = Math.max(a.y, b.y);
+  const right = Math.min(a.x + a.w, b.x + b.w);
+  const bottom = Math.min(a.y + a.h, b.y + b.h);
+  if (right <= x || bottom <= y) return null;
+  return { x, y, w: right - x, h: bottom - y };
+}
+
+const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
+
+function clipToOverflowAncestors(
+  node: CapturedNode,
+  rect: Rect,
+  nodesByBackend: ReadonlyMap<number, CapturedNode>,
+): Rect | null {
+  let clipped: Rect | null = rect;
+  let parentBackendNodeId = node.parentBackendNodeId;
+  let guard = 0;
+  while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
+    const parent = nodesByBackend.get(parentBackendNodeId);
+    if (!parent) break;
+    if (parent.rect) {
+      const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
+      const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
+      if (clipX || clipY) {
+        const clipRect = {
+          x: clipX ? parent.rect.x : clipped.x,
+          y: clipY ? parent.rect.y : clipped.y,
+          w: clipX ? parent.rect.w : clipped.w,
+          h: clipY ? parent.rect.h : clipped.h,
+        };
+        clipped = intersectRect(clipped, clipRect);
+      }
+    }
+    parentBackendNodeId = parent.parentBackendNodeId;
+    guard += 1;
+  }
+  return clipped;
+}
+
+function excludedByDomPolicy(
+  node: CapturedNode,
+  nodesByBackend: Map<number, CapturedNode>,
+  excludedBackendNodeIds: ReadonlySet<number>,
+): boolean {
+  let current: CapturedNode | undefined = node;
+  let guard = 0;
+  while (current && guard <= nodesByBackend.size) {
+    if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
+    if (current.visuallySuppressed === true) return true;
+    const attrs = current.attrs;
+    if (
+      Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
+      Object.prototype.hasOwnProperty.call(attrs, "inert") ||
+      (attrs["aria-hidden"] ?? "").toLowerCase() === "true"
+    ) {
+      return true;
+    }
+    current =
+      current.parentBackendNodeId === null
+        ? undefined
+        : nodesByBackend.get(current.parentBackendNodeId);
+    guard += 1;
+  }
+  return false;
+}
+
+export function discoverRenderedSurfaces(
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+  viewport: Viewport,
+  excludedBackendNodeIds: ReadonlySet<number> = new Set(),
+): RenderedSurface[] {
+  const surfaces: RenderedSurface[] = [];
+  for (const document of documents) {
+    const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
+    for (const node of document.domNodes) {
+      if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
+      if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
+      const viewportRect = intersectViewport(node.rect, viewport);
+      const visibleRect = viewportRect
+        ? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
+        : null;
+      if (!visibleRect) continue;
+      const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
+      surfaces.push({
+        renderingKind: "canvas",
+        frameId: document.frameId,
+        backendNodeId: node.backendNodeId,
+        parentBackendNodeId: node.parentBackendNodeId,
+        visibleRect,
+        paintOrder: node.paintOrder,
+        ...(label ? { label } : {}),
+      });
+    }
+  }
+  return surfaces;
+}

apps/extension/src/tools/vom/rendered-surfaces/index.ts

@@ -0,0 +1,4 @@
+export { clusterRenderedSurfaces } from "./cluster";
+export { discoverRenderedSurfaces } from "./discovery";
+export { projectRenderedSurfaces } from "./project";
+export type { RenderedSurface, RenderedSurfaceGroup } from "./types";

apps/extension/src/tools/vom/rendered-surfaces/project.ts

@@ -0,0 +1,100 @@
+import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
+import type { CapturedFrameDocument } from "../frame-capture";
+import type { SemanticAxNode } from "../semantic-graph";
+import type { RenderedSurfaceGroup } from "./types";
+
+function frameBackendKey(frameId: string, backendNodeId: number): string {
+  return `${frameId}\u0000${backendNodeId}`;
+}
+
+interface ProjectionIndex {
+  sceneByBackend: ReadonlyMap<string, number>;
+  documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
+  domByFrame: ReadonlyMap<
+    string,
+    ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
+  >;
+  rootSceneIdByFrame: ReadonlyMap<string, number>;
+}
+
+function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
+  const document = index.documentByFrame.get(group.frameId);
+  const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
+  let parentBackendNodeId = group.representative.parentBackendNodeId;
+  let guard = 0;
+  while (parentBackendNodeId !== null && guard <= domByBackend.size) {
+    const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
+    if (sceneId !== undefined) return sceneId;
+    parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
+    guard += 1;
+  }
+
+  if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
+    const owner = index.sceneByBackend.get(
+      frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
+    );
+    if (owner !== undefined) return owner;
+  }
+
+  return index.rootSceneIdByFrame.get(group.frameId) ?? null;
+}
+
+function buildProjectionIndex(
+  scene: VomScene,
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+): ProjectionIndex {
+  const rootSceneIdByFrame = new Map<string, number>();
+  for (const node of scene.nodes) {
+    if (
+      node.frameId &&
+      !rootSceneIdByFrame.has(node.frameId) &&
+      ["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
+    ) {
+      rootSceneIdByFrame.set(node.frameId, node.id);
+    }
+  }
+  return {
+    sceneByBackend: new Map(
+      scene.nodes.flatMap((node) =>
+        node.backendNodeId === undefined
+          ? []
+          : [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
+      ),
+    ),
+    documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
+    domByFrame: new Map(
+      documents.map((document) => [
+        document.frameId,
+        new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
+      ]),
+    ),
+    rootSceneIdByFrame,
+  };
+}
+
+export function projectRenderedSurfaces(
+  scene: VomScene,
+  documents: CapturedFrameDocument<SemanticAxNode>[],
+  groups: RenderedSurfaceGroup[],
+): VomScene {
+  const index = buildProjectionIndex(scene, documents);
+  const visualSurfaces: VomVisualSurface[] = [];
+
+  for (const group of groups) {
+    const parentId = nearestSceneParent(index, group);
+    visualSurfaces.push({
+      parentId,
+      backendNodeId: group.representative.backendNodeId,
+      frameId: group.frameId,
+      renderingKind: "canvas",
+      visibleRect: group.representative.visibleRect,
+      ...(group.label ? { label: group.label } : {}),
+      memberCount: group.members.length,
+    });
+  }
+
+  return {
+    ...scene,
+    ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
+  };
+}

apps/extension/src/tools/vom/rendered-surfaces/types.ts

@@ -0,0 +1,18 @@
+import type { Rect } from "@browser-skill/vom";
+
+export interface RenderedSurface {
+  renderingKind: "canvas";
+  frameId: string;
+  backendNodeId: number;
+  parentBackendNodeId: number | null;
+  visibleRect: Rect;
+  paintOrder: number;
+  label?: string;
+}
+
+export interface RenderedSurfaceGroup {
+  frameId: string;
+  members: RenderedSurface[];
+  representative: RenderedSurface;
+  label?: string;
+}

apps/extension/src/tools/vom/semantic-graph/build.ts

@@ -0,0 +1,154 @@
+import type { Viewport } from "@browser-skill/vom";
+import type { FrameDocument } from "../frame-document";
+import {
+  frameAxKey,
+  frameBackendKey,
+  type SemanticAxNode,
+  type SemanticFrame,
+  type SemanticGraph,
+  type SemanticGraphNode,
+  type SemanticNodeId,
+} from "./types";
+
+export interface BuildSemanticGraphInput {
+  documents: FrameDocument<SemanticAxNode>[];
+  viewport: Viewport;
+  rootFrameId?: string;
+  excludedBackendNodeIds?: ReadonlySet<number>;
+}
+
+function domNodeId(frameId: string, backendNodeId: number): SemanticNodeId {
+  return JSON.stringify([frameId, "dom", backendNodeId]);
+}
+
+function axNodeId(frameId: string, nodeId: string): SemanticNodeId {
+  return JSON.stringify([frameId, "ax", nodeId]);
+}
+
+export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGraph {
+  const rootFrameId =
+    input.rootFrameId ??
+    input.documents.find((document) => !document.parentFrameId)?.frameId ??
+    input.documents[0]?.frameId ??
+    "root";
+  const frames = new Map<string, SemanticFrame>();
+  const nodes = new Map<SemanticNodeId, SemanticGraphNode>();
+  const nodeIdsByFrame = new Map<string, SemanticNodeId[]>();
+  const nodeByFrameBackend = new Map<string, SemanticNodeId>();
+  const nodeByFrameAx = new Map<string, SemanticNodeId>();
+
+  for (const [frameOrder, document] of input.documents.entries()) {
+    frames.set(document.frameId, {
+      frameId: document.frameId,
+      ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
+      ...(document.ownerBackendNodeId !== undefined
+        ? { ownerBackendNodeId: document.ownerBackendNodeId }
+        : {}),
+      contextScopeId: document.contextScopeId,
+      target: document.target,
+      ...(document.url ? { url: document.url } : {}),
+      order: frameOrder,
+    });
+
+    const frameNodeIds: SemanticNodeId[] = [];
+    nodeIdsByFrame.set(document.frameId, frameNodeIds);
+    for (const [index, dom] of document.domNodes.entries()) {
+      const id = domNodeId(document.frameId, dom.backendNodeId);
+      const node: SemanticGraphNode = {
+        id,
+        frameId: document.frameId,
+        backendNodeId: dom.backendNodeId,
+        dom,
+        sourceOrder: index,
+        excluded:
+          document.frameId === rootFrameId &&
+          input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
+      };
+      nodes.set(id, node);
+      frameNodeIds.push(id);
+      nodeByFrameBackend.set(frameBackendKey(document.frameId, dom.backendNodeId), id);
+    }
+
+    for (const [index, ax] of document.axNodes.entries()) {
+      const joinedId =
+        typeof ax.backendDOMNodeId === "number"
+          ? nodeByFrameBackend.get(frameBackendKey(document.frameId, ax.backendDOMNodeId))
+          : undefined;
+      const id = joinedId ?? axNodeId(document.frameId, ax.nodeId);
+      const existing = nodes.get(id);
+      if (existing) {
+        existing.ax = ax;
+        existing.axNodeId = ax.nodeId;
+      } else {
+        nodes.set(id, {
+          id,
+          frameId: document.frameId,
+          ...(typeof ax.backendDOMNodeId === "number"
+            ? { backendNodeId: ax.backendDOMNodeId }
+            : {}),
+          axNodeId: ax.nodeId,
+          ax,
+          sourceOrder: document.domNodes.length + index,
+          excluded:
+            document.frameId === rootFrameId &&
+            typeof ax.backendDOMNodeId === "number" &&
+            input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
+        });
+        frameNodeIds.push(id);
+      }
+      nodeByFrameAx.set(frameAxKey(document.frameId, ax.nodeId), id);
+    }
+  }
+
+  for (const document of input.documents) {
+    for (const dom of document.domNodes) {
+      const id = nodeByFrameBackend.get(frameBackendKey(document.frameId, dom.backendNodeId));
+      const node = id ? nodes.get(id) : undefined;
+      if (!node || dom.parentBackendNodeId === null) continue;
+      node.domParentId = nodeByFrameBackend.get(
+        frameBackendKey(document.frameId, dom.parentBackendNodeId),
+      );
+    }
+    for (const ax of document.axNodes) {
+      const id = nodeByFrameAx.get(frameAxKey(document.frameId, ax.nodeId));
+      const node = id ? nodes.get(id) : undefined;
+      if (!node || !ax.parentId) continue;
+      node.axParentId = nodeByFrameAx.get(frameAxKey(document.frameId, ax.parentId));
+    }
+  }
+
+  const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
+  const excludedQueue: SemanticNodeId[] = [];
+  for (const node of nodes.values()) {
+    if (node.axParentId) {
+      const children = axChildren.get(node.axParentId) ?? [];
+      children.push(node.id);
+      axChildren.set(node.axParentId, children);
+    }
+    if (node.excluded) excludedQueue.push(node.id);
+  }
+  for (let index = 0; index < excludedQueue.length; index += 1) {
+    for (const childId of axChildren.get(excludedQueue[index]) ?? []) {
+      const child = nodes.get(childId);
+      if (!child || child.excluded) continue;
+      child.excluded = true;
+      excludedQueue.push(childId);
+    }
+  }
+
+  for (const frame of frames.values()) {
+    if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) continue;
+    frame.ownerNodeId = nodeByFrameBackend.get(
+      frameBackendKey(frame.parentFrameId, frame.ownerBackendNodeId),
+    );
+  }
+
+  return {
+    viewport: input.viewport,
+    rootFrameId,
+    frames,
+    nodes,
+    nodeIdsByFrame,
+    nodeByFrameBackend,
+  };
+}

apps/extension/src/tools/vom/semantic-graph/index.ts

@@ -0,0 +1,59 @@
+import type { Viewport, VomScene } from "@browser-skill/vom";
+import type { FrameDocument } from "../frame-document";
+import { buildSemanticGraph as buildGraph } from "./build";
+import { projectSemanticGraph as projectGraph } from "./project";
+import { resolveSemanticGraph as resolveGraph } from "./resolve";
+import { normalizeSemanticStructure as normalizeStructure } from "./structure";
+import type {
+  ResolvedSemanticGraph,
+  SemanticAxNode,
+  SemanticGraph,
+  StructuredSemanticGraph,
+} from "./types";
+
+export type { SemanticAxNode } from "./types";
+
+export interface SemanticVomInput {
+  documents: FrameDocument<SemanticAxNode>[];
+  viewport: Viewport;
+  rootFrameId?: string;
+  excludedBackendNodeIds?: ReadonlySet<number>;
+}
+
+export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph {
+  return buildGraph(input);
+}
+
+export function resolveSemanticGraph(
+  graph: SemanticGraph,
+  options: {
+    supplementalNames?: ReadonlyMap<string, string>;
+    identifierFallback?: boolean;
+  } = {},
+): ResolvedSemanticGraph {
+  return resolveGraph(graph, options);
+}
+
+export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
+  return normalizeStructure(graph);
+}
+
+export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
+  return projectGraph(graph);
+}
+
+export function buildSemanticVomScene(
+  input: SemanticVomInput & {
+    supplementalNames?: ReadonlyMap<string, string>;
+    identifierFallback?: boolean;
+  },
+): VomScene {
+  return projectSemanticGraph(
+    normalizeSemanticStructure(
+      resolveSemanticGraph(buildSemanticGraph(input), {
+        supplementalNames: input.supplementalNames,
+        identifierFallback: input.identifierFallback,
+      }),
+    ),
+  );
+}

apps/extension/src/tools/vom/semantic-graph/name-evidence.ts

@@ -0,0 +1,34 @@
+const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const;
+const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]);
+const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]);
+
+function identifierTokens(value: string): string[] {
+  return value
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .split(/[^a-zA-Z0-9]+/)
+    .map((token) => token.toLowerCase())
+    .filter(Boolean);
+}
+
+function readableIdentifier(value: string | undefined): string | undefined {
+  const raw = value?.trim();
+  if (!raw || raw.length < 3 || raw.length > 80) return undefined;
+  if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined;
+
+  const tokens = identifierTokens(raw);
+  if (GENERATED_PREFIXES.has(tokens[0])) return undefined;
+  while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift();
+  while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
+  if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined;
+  return tokens
+    .map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token))
+    .join(" ");
+}
+
+export function stableIdentifierName(attrs: Readonly<Record<string, string>>): string | undefined {
+  for (const name of IDENTIFIER_ATTRS) {
+    const label = readableIdentifier(attrs[name]);
+    if (label) return label;
+  }
+  return undefined;
+}

apps/extension/src/tools/vom/semantic-graph/project.ts

@@ -0,0 +1,85 @@
+import type { VomNode, VomScene } from "@browser-skill/vom";
+import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
+
+function retained(node: StructuredSemanticNode): boolean {
+  return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
+}
+
+function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
+  const connected = new Set<string>([graph.rootFrameId]);
+  const childrenByParent = new Map<string, string[]>();
+  for (const frame of graph.frames.values()) {
+    if (!frame.parentFrameId) continue;
+    const children = childrenByParent.get(frame.parentFrameId) ?? [];
+    children.push(frame.frameId);
+    childrenByParent.set(frame.parentFrameId, children);
+  }
+  const queue = [graph.rootFrameId];
+  for (let index = 0; index < queue.length; index += 1) {
+    for (const frameId of childrenByParent.get(queue[index]) ?? []) {
+      if (connected.has(frameId)) continue;
+      const frame = graph.frames.get(frameId);
+      if (!frame?.ownerNodeId) continue;
+      const owner = graph.nodes.get(frame.ownerNodeId);
+      if (!owner || !retained(owner)) continue;
+      connected.add(frame.frameId);
+      queue.push(frame.frameId);
+    }
+  }
+  return connected;
+}
+
+function domAncestors(
+  graph: StructuredSemanticGraph,
+  node: StructuredSemanticNode,
+  numericId: ReadonlyMap<SemanticNodeId, number>,
+): number[] {
+  const ancestors: number[] = [];
+  const seen = new Set<SemanticNodeId>();
+  let current = node.domParentId;
+  while (current && !seen.has(current)) {
+    seen.add(current);
+    const id = numericId.get(current);
+    if (id !== undefined) ancestors.push(id);
+    current = graph.nodes.get(current)?.domParentId;
+  }
+  return ancestors;
+}
+
+export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
+  const connectedFrames = connectedFrameIds(graph);
+  const orderedNodes = [...graph.frames.values()]
+    .filter((frame) => connectedFrames.has(frame.frameId))
+    .sort((a, b) => a.order - b.order)
+    .flatMap((frame) =>
+      (graph.nodeIdsByFrame.get(frame.frameId) ?? [])
+        .map((id) => graph.nodes.get(id))
+        .filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
+        .sort((a, b) => a.sourceOrder - b.sourceOrder),
+    );
+
+  const numericId = new Map<SemanticNodeId, number>();
+  orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
+  const nodes: VomNode[] = orderedNodes.map((node) => ({
+    ...node.vom,
+    id: numericId.get(node.id) as number,
+    parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
+    ...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
+    frameId: node.frameId,
+    contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
+    referenceable: node.referenceable,
+    ...(node.dom
+      ? {
+          domParentId:
+            node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
+          domAncestorIds: domAncestors(graph, node, numericId),
+        }
+      : {}),
+  }));
+
+  return {
+    viewport: graph.viewport,
+    nodes,
+    rootFrameId: graph.rootFrameId,
+  };
+}

apps/extension/src/tools/vom/semantic-graph/resolve.ts

@@ -0,0 +1,521 @@
+import type { VomNode } from "@browser-skill/vom";
+import { stableIdentifierName } from "./name-evidence";
+import type {
+  ResolvedSemanticGraph,
+  ResolvedSemanticNode,
+  SemanticAxNode,
+  SemanticGraph,
+  SemanticGraphNode,
+  SemanticNodeId,
+} from "./types";
+import { frameBackendKey } from "./types";
+
+const FORM_TAGS = new Set(["input", "textarea", "select"]);
+const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
+const TEXT_AX_ROLES = new Set(["statictext", "inlinetextbox", "labeltext", "text"]);
+const INTERACTIVE_ROLES = new Set([
+  "button",
+  "checkbox",
+  "combobox",
+  "link",
+  "listbox",
+  "menuitem",
+  "menuitemcheckbox",
+  "menuitemradio",
+  "option",
+  "radio",
+  "searchbox",
+  "slider",
+  "spinbutton",
+  "switch",
+  "tab",
+  "textbox",
+  "treeitem",
+]);
+const SENSITIVE_INPUT_TYPES = new Set([
+  "password",
+  "credit-card",
+  "one-time-code",
+  "current-password",
+  "new-password",
+]);
+
+function clean(value: string | undefined): string | undefined {
+  const normalized = value?.replace(/\s+/g, " ").trim();
+  return normalized || undefined;
+}
+
+function axValue(field?: { value?: string | number | boolean }): string | undefined {
+  return field?.value === undefined ? undefined : clean(String(field.value));
+}
+
+function axProperty(node: SemanticAxNode | undefined, name: string): string | undefined {
+  return axValue(node?.properties?.find((property) => property.name === name)?.value);
+}
+
+function normalizeRole(value: string | undefined): string | undefined {
+  const role = clean(value)?.toLowerCase();
+  return role || undefined;
+}
+
+function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
+  if (tag === "button") return "button";
+  if (tag === "textarea") return "textbox";
+  if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
+  if (tag === "a" && attrs.href) return "link";
+  if (tag === "iframe") return "Iframe";
+  if (tag === "dialog") return "dialog";
+  if (tag === "img") return "img";
+  if (tag === "table") return "table";
+  if (tag === "tr") return "row";
+  if (tag === "th") return "columnheader";
+  if (tag === "td") return "cell";
+  if (tag === "ul" || tag === "ol") return "list";
+  if (tag === "li") return "listitem";
+  if (/^h[1-6]$/.test(tag)) return "heading";
+  if (tag === "nav") return "navigation";
+  if (tag === "main") return "main";
+  if (tag === "form") return "form";
+  if (tag === "input") {
+    const type = (attrs.type ?? "text").toLowerCase();
+    if (type === "hidden") return undefined;
+    if (type === "checkbox") return "checkbox";
+    if (type === "radio") return "radio";
+    if (type === "range") return "slider";
+    if (type === "number") return "spinbutton";
+    if (type === "search") return "searchbox";
+    if (["button", "submit", "reset", "image"].includes(type)) return "button";
+    return "textbox";
+  }
+  return undefined;
+}
+
+function domRole(tag: string, attrs: Record<string, string>): string | undefined {
+  return normalizeRole(attrs.role) ?? nativeRole(tag, attrs);
+}
+
+function resolvedRole(node: SemanticGraphNode): {
+  value?: string;
+  source: ResolvedSemanticNode["roleSource"];
+} {
+  const rawAxRole = clean(axValue(node.ax?.role));
+  const axRole = normalizeRole(rawAxRole);
+  const attrs = node.dom?.attrs ?? {};
+  const explicitDomRole = normalizeRole(attrs.role);
+  const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs);
+  const usableAxRole = node.ax?.ignored !== true;
+  if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) {
+    return { value: rawAxRole, source: "ax" };
+  }
+  if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" };
+  if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" };
+  if (rawAxRole) {
+    return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" };
+  }
+  return { source: "none" };
+}
+
+function sensitive(node: SemanticGraphNode): boolean {
+  const attrs = node.dom?.attrs ?? {};
+  const type = (attrs.type ?? "").toLowerCase();
+  const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
+  return (
+    type === "password" ||
+    autocomplete.startsWith("cc-") ||
+    SENSITIVE_INPUT_TYPES.has(autocomplete) ||
+    SENSITIVE_INPUT_TYPES.has(axProperty(node.ax, "inputType") ?? "")
+  );
+}
+
+function inputState(
+  node: SemanticGraphNode,
+  role: string | undefined,
+  value: string | undefined,
+  isSensitive: boolean,
+): VomNode["inputState"] {
+  if (node.dom?.formState) return node.dom.formState;
+  const tag = node.dom?.tag.toLowerCase() ?? "";
+  if (!FORM_TAGS.has(tag) && !["textbox", "searchbox"].includes(role ?? "")) return undefined;
+  if (isSensitive) return value ? "filled" : "empty";
+  if (node.dom?.formValue !== undefined) {
+    if (node.dom.formValue === "") return "empty";
+    return node.dom.formValue === (node.dom.formDefaultValue ?? "") ? "default" : "filled";
+  }
+  return value ? "filled" : "empty";
+}
+
+function externalHrefHost(
+  href: string | undefined,
+  pageUrl: string | undefined,
+): string | undefined {
+  if (!href || !pageUrl) return undefined;
+  try {
+    const page = new URL(pageUrl);
+    const target = new URL(href, page);
+    return target.origin === page.origin ? undefined : target.hostname;
+  } catch {
+    return undefined;
+  }
+}
+
+interface ResolveIndexes {
+  domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
+  axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
+  domIdIndex: Map<string, SemanticNodeId>;
+  labelForIndex: Map<string, SemanticNodeId[]>;
+  domText: (nodeId: SemanticNodeId) => string | undefined;
+  axText: (nodeId: SemanticNodeId) => string | undefined;
+  hasNativeDescendant: (nodeId: SemanticNodeId) => boolean;
+}
+
+function buildIndexes(graph: SemanticGraph): ResolveIndexes {
+  const domChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
+  const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
+  const domIdIndex = new Map<string, SemanticNodeId>();
+  const labelForIndex = new Map<string, SemanticNodeId[]>();
+  for (const node of graph.nodes.values()) {
+    if (node.domParentId) {
+      const children = domChildren.get(node.domParentId) ?? [];
+      children.push(node.id);
+      domChildren.set(node.domParentId, children);
+    }
+    if (node.axParentId) {
+      const children = axChildren.get(node.axParentId) ?? [];
+      children.push(node.id);
+      axChildren.set(node.axParentId, children);
+    }
+    const id = clean(node.dom?.attrs.id);
+    if (id) domIdIndex.set(`${node.frameId}\u0000${id}`, node.id);
+    const targetId = clean(node.dom?.attrs.for);
+    if (node.dom?.tag.toLowerCase() === "label" && targetId) {
+      const key = `${node.frameId}\u0000${targetId}`;
+      const labels = labelForIndex.get(key) ?? [];
+      labels.push(node.id);
+      labelForIndex.set(key, labels);
+    }
+  }
+
+  for (const [parentId, children] of axChildren) {
+    const order = graph.nodes.get(parentId)?.ax?.childIds ?? [];
+    const orderByAxId = new Map(order.map((nodeId, index) => [nodeId, index]));
+    children.sort((a, b) => {
+      const aIndex = orderByAxId.get(graph.nodes.get(a)?.axNodeId ?? "") ?? -1;
+      const bIndex = orderByAxId.get(graph.nodes.get(b)?.axNodeId ?? "") ?? -1;
+      return (
+        (aIndex < 0 ? Number.MAX_SAFE_INTEGER : aIndex) -
+        (bIndex < 0 ? Number.MAX_SAFE_INTEGER : bIndex)
+      );
+    });
+  }
+
+  const textMemo = new Map<SemanticNodeId, string | undefined>();
+  const resolvingText = new Set<SemanticNodeId>();
+  const domText = (nodeId: SemanticNodeId): string | undefined => {
+    if (textMemo.has(nodeId)) return textMemo.get(nodeId);
+    if (resolvingText.has(nodeId)) return undefined;
+    resolvingText.add(nodeId);
+    const node = graph.nodes.get(nodeId);
+    const parts: string[] = [];
+    const own = clean(node?.dom?.textContent);
+    if (own) parts.push(own);
+    for (const childId of domChildren.get(nodeId) ?? []) {
+      const child = graph.nodes.get(childId);
+      const childRole = child
+        ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
+        : undefined;
+      if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
+      const childText = domText(childId);
+      if (childText) parts.push(childText);
+      if (parts.join(" ").length >= 240) break;
+    }
+    resolvingText.delete(nodeId);
+    const result = clean(parts.join(" ").slice(0, 240));
+    textMemo.set(nodeId, result);
+    return result;
+  };
+
+  const axTextMemo = new Map<SemanticNodeId, string | undefined>();
+  const resolvingAxText = new Set<SemanticNodeId>();
+  const axText = (nodeId: SemanticNodeId): string | undefined => {
+    if (axTextMemo.has(nodeId)) return axTextMemo.get(nodeId);
+    if (resolvingAxText.has(nodeId)) return undefined;
+    resolvingAxText.add(nodeId);
+    const complete = (value: string | undefined): string | undefined => {
+      resolvingAxText.delete(nodeId);
+      axTextMemo.set(nodeId, value);
+      return value;
+    };
+    const node = graph.nodes.get(nodeId);
+    const role = normalizeRole(axValue(node?.ax?.role)) ?? "";
+    if (TEXT_AX_ROLES.has(role)) {
+      const own = clean(axValue(node?.ax?.name) ?? axValue(node?.ax?.value));
+      if (own) return complete(own);
+    }
+    const parts: string[] = [];
+    for (const childId of axChildren.get(nodeId) ?? []) {
+      const child = graph.nodes.get(childId);
+      const childRole = normalizeRole(axValue(child?.ax?.role)) ?? "";
+      if (INTERACTIVE_ROLES.has(childRole)) continue;
+      const childText = axText(childId);
+      if (childText) parts.push(childText);
+      if (parts.join(" ").length >= 240) break;
+    }
+    return complete(clean(parts.join(" ").slice(0, 240)));
+  };
+
+  const nativeMemo = new Map<SemanticNodeId, boolean>();
+  const hasNativeDescendant = (nodeId: SemanticNodeId): boolean => {
+    const cached = nativeMemo.get(nodeId);
+    if (cached !== undefined) return cached;
+    nativeMemo.set(nodeId, false);
+    for (const childId of domChildren.get(nodeId) ?? []) {
+      const child = graph.nodes.get(childId);
+      const childRole =
+        (child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ??
+        (child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined);
+      if (
+        (child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
+        INTERACTIVE_ROLES.has(childRole ?? "") ||
+        hasNativeDescendant(childId)
+      ) {
+        nativeMemo.set(nodeId, true);
+        return true;
+      }
+    }
+    return false;
+  };
+
+  return {
+    domChildren,
+    axChildren,
+    domIdIndex,
+    labelForIndex,
+    domText,
+    axText,
+    hasNativeDescendant,
+  };
+}
+
+function ariaLabelledText(node: SemanticGraphNode, indexes: ResolveIndexes): string | undefined {
+  const ids = (node.dom?.attrs["aria-labelledby"] ?? "").split(/\s+/).filter(Boolean);
+  const parts = ids.flatMap((id) => {
+    const target = indexes.domIdIndex.get(`${node.frameId}\u0000${id}`);
+    return target ? (indexes.domText(target) ?? []) : [];
+  });
+  return clean(parts.join(" "));
+}
+
+function nativeLabelText(
+  node: SemanticGraphNode,
+  graph: SemanticGraph,
+  indexes: ResolveIndexes,
+): string | undefined {
+  const id = clean(node.dom?.attrs.id);
+  if (id) {
+    const labels = indexes.labelForIndex.get(`${node.frameId}\u0000${id}`) ?? [];
+    const explicit = clean(
+      labels
+        .map((labelId) => indexes.domText(labelId))
+        .filter(Boolean)
+        .join(" "),
+    );
+    if (explicit) return explicit;
+  }
+  let parentId = node.domParentId;
+  let guard = 0;
+  while (parentId && guard <= graph.nodes.size) {
+    const parent = graph.nodes.get(parentId);
+    if (!parent) break;
+    if (parent.dom?.tag.toLowerCase() === "label") return indexes.domText(parentId);
+    parentId = parent.domParentId;
+    guard += 1;
+  }
+  return undefined;
+}
+
+function precedingAxText(
+  node: SemanticGraphNode,
+  graph: SemanticGraph,
+  indexes: ResolveIndexes,
+): string | undefined {
+  if (!node.axParentId) return undefined;
+  const siblings = indexes.axChildren.get(node.axParentId) ?? [];
+  const index = siblings.indexOf(node.id);
+  for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
+    const sibling = graph.nodes.get(siblings[index - offset]);
+    const role = normalizeRole(axValue(sibling?.ax?.role)) ?? "";
+    if (INTERACTIVE_ROLES.has(role)) break;
+    if (!TEXT_AX_ROLES.has(role) && role !== "generic") continue;
+    const text = clean(axValue(sibling?.ax?.name) ?? axValue(sibling?.ax?.value))?.replace(
+      /[：:]\s*$/,
+      "",
+    );
+    if (text && text.length <= 80) return text;
+  }
+  return undefined;
+}
+
+function precedingDomText(
+  node: SemanticGraphNode,
+  graph: SemanticGraph,
+  indexes: ResolveIndexes,
+): string | undefined {
+  if (!node.domParentId) return undefined;
+  const siblings = indexes.domChildren.get(node.domParentId) ?? [];
+  const index = siblings.indexOf(node.id);
+  for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
+    const sibling = graph.nodes.get(siblings[index - offset]);
+    const siblingRole = sibling
+      ? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
+      : undefined;
+    if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
+    const text = indexes.domText(siblings[index - offset])?.replace(/[：:]\s*$/, "");
+    if (text && text.length <= 80) return text;
+  }
+  return undefined;
+}
+
+function resolvedName(
+  node: SemanticGraphNode,
+  role: string | undefined,
+  graph: SemanticGraph,
+  indexes: ResolveIndexes,
+  supplementalNames: ReadonlyMap<string, string>,
+  identifierFallback: boolean,
+): string | undefined {
+  const attrs = node.dom?.attrs ?? {};
+  const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
+  const descendantText = indexes.domText(node.id);
+  const interactive =
+    INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
+    NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
+  const candidates = [
+    node.ax?.ignored === true ? undefined : axValue(node.ax?.name),
+    ariaLabelledText(node, indexes),
+    clean(attrs["aria-label"]),
+    nativeLabelText(node, graph, indexes),
+    clean(attrs.title),
+    clean(attrs.alt),
+    node.ax?.ignored === true ? undefined : indexes.axText(node.id),
+    descendantText,
+    formControl ? precedingAxText(node, graph, indexes) : undefined,
+    formControl ? precedingDomText(node, graph, indexes) : undefined,
+    formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
+    formControl || !interactive || node.backendNodeId === undefined
+      ? undefined
+      : clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
+    identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
+  ];
+  let name = candidates.find((candidate) => candidate !== undefined);
+  const hasPopup = axProperty(node.ax, "hasPopup");
+  if (name && hasPopup && hasPopup !== "false") {
+    name =
+      axProperty(node.ax, "expanded") === "true" ? `${name} [expanded]` : `${name} [has-submenu]`;
+  }
+  if (!name && role?.toLowerCase() === "iframe") return clean(attrs.id);
+  return name;
+}
+
+function insideNative(node: SemanticGraphNode, graph: SemanticGraph): boolean {
+  let parentId = node.domParentId;
+  let guard = 0;
+  while (parentId && guard <= graph.nodes.size) {
+    const parent = graph.nodes.get(parentId);
+    if (!parent) break;
+    if (parent.dom && NATIVE_CONTROL_TAGS.has(parent.dom.tag.toLowerCase())) return true;
+    parentId = parent.domParentId;
+    guard += 1;
+  }
+  return false;
+}
+
+function nearbyText(
+  node: SemanticGraphNode,
+  graph: SemanticGraph,
+  indexes: ResolveIndexes,
+): string | undefined {
+  if (!node.domParentId) return undefined;
+  const siblings = indexes.domChildren.get(node.domParentId) ?? [];
+  const index = siblings.indexOf(node.id);
+  if (index < 0) return undefined;
+  const texts = siblings
+    .slice(Math.max(0, index - 3), index + 4)
+    .filter((id) => id !== node.id)
+    .flatMap((id) => indexes.domText(id) ?? [])
+    .slice(0, 3);
+  return clean(texts.join(" "));
+}
+
+export function resolveSemanticGraph(
+  graph: SemanticGraph,
+  options: {
+    supplementalNames?: ReadonlyMap<string, string>;
+    identifierFallback?: boolean;
+  } = {},
+): ResolvedSemanticGraph {
+  const indexes = buildIndexes(graph);
+  const supplementalNames = options.supplementalNames ?? new Map<string, string>();
+  const identifierFallback = options.identifierFallback ?? true;
+  const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
+  for (const node of graph.nodes.values()) {
+    const roleResolution = resolvedRole(node);
+    const role = roleResolution.value;
+    const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback);
+    const isSensitive = sensitive(node);
+    const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
+    const value = isSensitive ? undefined : sourceValue;
+    const attrs = { ...(node.dom?.attrs ?? {}) };
+    if (isSensitive) delete attrs.value;
+    const frame = graph.frames.get(node.frameId);
+    const tag = node.dom?.tag.toLowerCase() ?? "";
+    const vom: ResolvedSemanticNode["vom"] = {
+      tag,
+      rect: node.dom?.rect ?? null,
+      paintOrder: node.dom?.paintOrder ?? 0,
+      position: node.dom?.position || "static",
+      pointerEvents: node.dom?.pointerEvents || "auto",
+      ...(node.dom?.cursor ? { cursor: node.dom.cursor } : {}),
+      ...(role ? { role } : {}),
+      ...(name ? { name } : {}),
+      ...(value !== undefined ? { value } : {}),
+      ...(clean(node.dom?.formPlaceholder ?? attrs.placeholder)
+        ? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) }
+        : {}),
+      attrs,
+      ...(clean(node.dom?.textContent) ? { text: clean(node.dom?.textContent) } : {}),
+      ...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}),
+      inputState: inputState(node, role, sourceValue, isSensitive),
+      sensitive: isSensitive,
+      modal:
+        ["dialog", "alertdialog"].includes(role?.toLowerCase() ?? "") ||
+        tag === "dialog" ||
+        (attrs["aria-modal"] ?? "").toLowerCase() === "true",
+      disabled:
+        Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
+        (attrs["aria-disabled"] ?? "").toLowerCase() === "true",
+      inert: Object.prototype.hasOwnProperty.call(attrs, "inert"),
+      hasNativeDescendant: indexes.hasNativeDescendant(node.id),
+      insideNative: insideNative(node, graph),
+      ...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
+    };
+    const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected");
+    const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded");
+    const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls");
+    if (selected === "true" && !vom.attrs?.["aria-selected"]) {
+      vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
+    }
+    if (expanded === "true" && !vom.attrs?.["aria-expanded"]) {
+      vom.attrs = { ...(vom.attrs ?? {}), "aria-expanded": "true" };
+    }
+    if (controls && !vom.attrs?.["aria-controls"]) {
+      vom.attrs = { ...(vom.attrs ?? {}), "aria-controls": controls };
+    }
+    nodes.set(node.id, {
+      ...node,
+      vom,
+      referenceable: node.backendNodeId !== undefined,
+      roleSource: roleResolution.source,
+    });
+  }
+  return { ...graph, nodes };
+}

apps/extension/src/tools/vom/semantic-graph/structure.ts

@@ -0,0 +1,456 @@
+import {
+  applyVomInteractionRecovery,
+  isVomReferenceNode,
+  isVomStructuralRole,
+  type VomNode,
+} from "@browser-skill/vom";
+import type {
+  ResolvedSemanticGraph,
+  ResolvedSemanticNode,
+  SemanticNodeId,
+  StructureDecision,
+  StructuredSemanticGraph,
+  StructuredSemanticNode,
+} from "./types";
+
+const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
+const ROOT_ROLES = new Set(["rootwebarea", "webarea"]);
+const CONTENT_BOUNDARY_ROLES = new Set([
+  "article",
+  "dialog",
+  "document",
+  "feed",
+  "form",
+  "grid",
+  "heading",
+  "iframe",
+  "list",
+  "main",
+  "navigation",
+  "paragraph",
+  "region",
+  "section",
+  "table",
+  "tree",
+]);
+
+function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
+  const attrs = node.vom.attrs ?? {};
+  return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
+    attrs[name]?.trim(),
+  );
+}
+
+function explicitStructural(node: ResolvedSemanticNode): boolean {
+  return (
+    isVomStructuralRole(node.vom.role) &&
+    node.roleSource !== "none" &&
+    node.roleSource !== "ax-ignored"
+  );
+}
+
+function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean {
+  return node.ax !== undefined && node.ax.ignored !== true;
+}
+
+function isRenderedDomFallback(node: ResolvedSemanticNode): boolean {
+  if (!node.dom) return false;
+  const attrs = node.dom.attrs;
+  if (
+    Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
+    Object.prototype.hasOwnProperty.call(attrs, "inert") ||
+    (attrs["aria-hidden"] ?? "").toLowerCase() === "true"
+  ) {
+    return false;
+  }
+  return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null);
+}
+
+function initialParentId(
+  graph: ResolvedSemanticGraph,
+  node: ResolvedSemanticNode,
+): SemanticNodeId | undefined {
+  let parentId = node.axParentId ?? node.domParentId;
+  if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
+    const axParent = graph.nodes.get(node.axParentId);
+    const axRole = axParent?.vom.role?.toLowerCase() ?? "";
+    const meaningfulAxParent =
+      axParent !== undefined &&
+      !ROOT_ROLES.has(axRole) &&
+      (explicitStructural(axParent) ||
+        !TRANSPARENT_ROLES.has(axRole) ||
+        Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent)));
+    parentId = meaningfulAxParent ? node.axParentId : node.domParentId;
+  }
+  if (parentId) return parentId;
+  if (node.frameId === graph.rootFrameId) return undefined;
+  return graph.frames.get(node.frameId)?.ownerNodeId;
+}
+
+function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] {
+  return [...graph.frames.values()]
+    .sort((a, b) => a.order - b.order)
+    .flatMap((frame) =>
+      (graph.nodeIdsByFrame.get(frame.frameId) ?? [])
+        .map((id) => graph.nodes.get(id))
+        .filter((node): node is ResolvedSemanticNode => node !== undefined)
+        .sort((a, b) => a.sourceOrder - b.sourceOrder),
+    );
+}
+
+function recoveredNodes(
+  graph: ResolvedSemanticGraph,
+  nodes: ResolvedSemanticNode[],
+): Map<SemanticNodeId, VomNode> {
+  const numericId = new Map(nodes.map((node, index) => [node.id, index + 1]));
+  const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id]));
+  const provisional = nodes.map((node) => ({
+    ...node.vom,
+    id: numericId.get(node.id) as number,
+    parentId: (() => {
+      const parent = initialParentId(graph, node);
+      return parent ? (numericId.get(parent) ?? null) : null;
+    })(),
+    ...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
+    frameId: node.frameId,
+    contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
+    referenceable: node.referenceable,
+  }));
+  return new Map(
+    applyVomInteractionRecovery(provisional).flatMap((node) => {
+      const id = semanticId.get(node.id);
+      return id ? [[id, node] as const] : [];
+    }),
+  );
+}
+
+function redundantSourceNodes(
+  graph: ResolvedSemanticGraph,
+  nodes: ResolvedSemanticNode[],
+  recovered: ReadonlyMap<SemanticNodeId, VomNode>,
+): Set<SemanticNodeId> {
+  const byKey = new Map<string, ResolvedSemanticNode[]>();
+  for (const node of nodes) {
+    const output = recovered.get(node.id) as VomNode;
+    const name = output.name?.trim().toLowerCase() ?? "";
+    if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue;
+    const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`;
+    const matches = byKey.get(key) ?? [];
+    matches.push(node);
+    byKey.set(key, matches);
+  }
+  const redundant = new Set<SemanticNodeId>();
+  for (const matches of byKey.values()) {
+    const axOnly = matches.filter((node) => node.ax && !node.dom);
+    const domOnly = matches.filter((node) => node.dom && !node.ax);
+    if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
+      redundant.add(domOnly[0].id);
+    }
+  }
+  return redundant;
+}
+
+function baseDecision(
+  graph: ResolvedSemanticGraph,
+  node: ResolvedSemanticNode,
+  output: VomNode,
+  frameOwnerIds: ReadonlySet<SemanticNodeId>,
+  redundant: ReadonlySet<SemanticNodeId>,
+): StructureDecision {
+  if (node.excluded) return { kind: "excluded", reason: "excluded" };
+  if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
+  if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) {
+    return { kind: "excluded", reason: "non-rendered-dom-fallback" };
+  }
+  if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
+  if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
+    return { kind: "transparent", reason: "child-document-root" };
+  }
+  if (node.ax?.ignored === true && !node.dom) {
+    return { kind: "transparent", reason: "ax-ignored" };
+  }
+  const role = output.role?.toLowerCase() ?? "";
+  if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" };
+  if (
+    output.name ||
+    output.text ||
+    output.value ||
+    output.placeholder ||
+    hasMeaningfulRelations(node) ||
+    (output.rect && ["fixed", "sticky"].includes(output.position))
+  ) {
+    return { kind: "keep", reason: "semantic" };
+  }
+  return { kind: "transparent", reason: "wrapper" };
+}
+
+function buildChildren(
+  nodes: ResolvedSemanticNode[],
+  relation: "axParentId" | "domParentId",
+): Map<SemanticNodeId, SemanticNodeId[]> {
+  const children = new Map<SemanticNodeId, SemanticNodeId[]>();
+  for (const node of nodes) {
+    const parentId = node[relation];
+    if (!parentId) continue;
+    const siblings = children.get(parentId) ?? [];
+    siblings.push(node.id);
+    children.set(parentId, siblings);
+  }
+  return children;
+}
+
+function explicitDomAncestors(
+  graph: ResolvedSemanticGraph,
+  nodes: ResolvedSemanticNode[],
+  domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
+): Set<SemanticNodeId> {
+  const inside = new Set<SemanticNodeId>();
+  const roots = nodes.filter((node) => !node.domParentId);
+  const queue = roots.map((node) => node.id);
+  for (let index = 0; index < queue.length; index += 1) {
+    const parentId = queue[index];
+    const parent = graph.nodes.get(parentId);
+    const parentIsBoundary =
+      parent !== undefined &&
+      explicitStructural(parent) &&
+      !ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? "");
+    for (const childId of domChildren.get(parentId) ?? []) {
+      if (inside.has(parentId) || parentIsBoundary) inside.add(childId);
+      queue.push(childId);
+    }
+  }
+  return inside;
+}
+
+function reliableGroupCandidates(
+  graph: ResolvedSemanticGraph,
+  nodes: ResolvedSemanticNode[],
+  recovered: ReadonlyMap<SemanticNodeId, VomNode>,
+  decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
+  domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
+): Set<SemanticNodeId> {
+  const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren);
+  const remainingChildren = new Map(
+    nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]),
+  );
+  const summaries = new Map<SemanticNodeId, { blocked: boolean; interactions: number }>();
+  const queue = nodes
+    .filter((node) => (remainingChildren.get(node.id) ?? 0) === 0)
+    .map((node) => node.id);
+  for (let index = 0; index < queue.length; index += 1) {
+    const nodeId = queue[index];
+    const node = graph.nodes.get(nodeId);
+    const output = recovered.get(nodeId);
+    if (!node || !output) continue;
+    const role = output.role?.toLowerCase() ?? "";
+    const excluded = decisions.get(nodeId)?.kind === "excluded";
+    const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role);
+    const summary = excluded
+      ? { blocked: true, interactions: 0 }
+      : isVomReferenceNode(output)
+        ? { blocked: false, interactions: 1 }
+        : blocked
+          ? { blocked: true, interactions: 0 }
+          : (domChildren.get(nodeId) ?? []).reduce(
+              (combined, childId) => {
+                const child = summaries.get(childId);
+                return {
+                  blocked: combined.blocked || !child || child.blocked,
+                  interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)),
+                };
+              },
+              { blocked: false, interactions: 0 },
+            );
+    summaries.set(nodeId, summary);
+    const parentId = node.domParentId;
+    if (!parentId) continue;
+    const remaining = (remainingChildren.get(parentId) ?? 1) - 1;
+    remainingChildren.set(parentId, remaining);
+    if (remaining === 0) queue.push(parentId);
+  }
+
+  const candidates = new Set<SemanticNodeId>();
+  for (const node of nodes) {
+    const output = recovered.get(node.id) as VomNode;
+    const role = output.role?.toLowerCase() ?? "";
+    if (
+      !node.dom ||
+      node.dom.tag.toLowerCase() !== "div" ||
+      !TRANSPARENT_ROLES.has(role) ||
+      insideExplicitStructure.has(node.id) ||
+      decisions.get(node.id)?.kind === "excluded" ||
+      node.dom.attrs.hidden !== undefined ||
+      (node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" ||
+      node.dom.attrs.inert !== undefined
+    ) {
+      continue;
+    }
+
+    const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id));
+    const blocked = branches.some((summary) => !summary || summary.blocked);
+    const contributingBranches = branches.filter(
+      (summary) => (summary?.interactions ?? 0) > 0,
+    ).length;
+    const interactions = branches.reduce(
+      (count, summary) => Math.min(2, count + (summary?.interactions ?? 0)),
+      0,
+    );
+    if (!blocked && contributingBranches >= 2 && interactions >= 2) {
+      candidates.add(node.id);
+    }
+  }
+  return candidates;
+}
+
+function leafCandidates(
+  nodes: ResolvedSemanticNode[],
+  candidates: ReadonlySet<SemanticNodeId>,
+  domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
+): Set<SemanticNodeId> {
+  const parentById = new Map(
+    nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])),
+  );
+  const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]));
+  const containsCandidate = new Set<SemanticNodeId>();
+  const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id);
+  const selected = new Set<SemanticNodeId>();
+  for (let index = 0; index < queue.length; index += 1) {
+    const nodeId = queue[index];
+    if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId);
+    const parentId = parentById.get(nodeId);
+    if (!parentId) continue;
+    if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId);
+    const remaining = (childCount.get(parentId) ?? 1) - 1;
+    childCount.set(parentId, remaining);
+    if (remaining === 0) queue.push(parentId);
+  }
+  return selected;
+}
+
+function nearestKept(
+  graph: ResolvedSemanticGraph,
+  startId: SemanticNodeId | undefined,
+  relation: "axParentId" | "domParentId",
+  kept: ReadonlySet<SemanticNodeId>,
+  memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
+): SemanticNodeId | undefined {
+  let currentId = startId;
+  const path: SemanticNodeId[] = [];
+  const seen = new Set<SemanticNodeId>();
+  let result: SemanticNodeId | undefined;
+  while (currentId && !seen.has(currentId)) {
+    if (kept.has(currentId)) {
+      result = currentId;
+      break;
+    }
+    if (memo.has(currentId)) {
+      result = memo.get(currentId);
+      break;
+    }
+    seen.add(currentId);
+    path.push(currentId);
+    currentId = graph.nodes.get(currentId)?.[relation];
+  }
+  for (const id of path) memo.set(id, result);
+  return result;
+}
+
+function finalParentId(
+  graph: ResolvedSemanticGraph,
+  node: ResolvedSemanticNode,
+  kept: ReadonlySet<SemanticNodeId>,
+  decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
+  axParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
+  domParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
+): SemanticNodeId | undefined {
+  const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo);
+  const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo);
+  const axNode = axParent ? graph.nodes.get(axParent) : undefined;
+  const domNode = domParent ? graph.nodes.get(domParent) : undefined;
+  const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? "");
+  const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false;
+  const domIsStructural =
+    domNode !== undefined &&
+    (explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group");
+  if (axIsExplicit) return axParent;
+  if (domIsStructural) return domParent;
+  if (axParent && !axIsRoot) return axParent;
+  if (domParent) return domParent;
+  if (axParent) return axParent;
+  if (node.frameId === graph.rootFrameId) return undefined;
+  return graph.frames.get(node.frameId)?.ownerNodeId;
+}
+
+function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] {
+  const {
+    id: _id,
+    parentId: _parentId,
+    backendNodeId: _backendNodeId,
+    frameId: _frameId,
+    contextScopeId: _contextScopeId,
+    referenceable: _referenceable,
+    ...vom
+  } = node;
+  if (!inferredGroup) return vom;
+  const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom;
+  return { ...groupVom, role: "group" };
+}
+
+export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
+  const nodes = orderedNodes(graph);
+  const recovered = recoveredNodes(graph, nodes);
+  const redundant = redundantSourceNodes(graph, nodes, recovered);
+  const frameOwnerIds = new Set(
+    [...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
+  );
+  const decisions = new Map<SemanticNodeId, StructureDecision>();
+  for (const node of nodes) {
+    decisions.set(
+      node.id,
+      baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant),
+    );
+  }
+
+  const domChildren = buildChildren(nodes, "domParentId");
+  const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren);
+  for (const nodeId of leafCandidates(nodes, candidates, domChildren)) {
+    decisions.set(nodeId, {
+      kind: "inferred-group",
+      reason: "independent-interaction-branches",
+    });
+  }
+
+  const kept = new Set(
+    nodes
+      .filter((node) => {
+        const kind = decisions.get(node.id)?.kind;
+        return kind === "keep" || kind === "inferred-group";
+      })
+      .map((node) => node.id),
+  );
+  const axParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
+  const domParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
+  const structuredNodes = new Map<SemanticNodeId, StructuredSemanticNode>();
+  for (const node of nodes) {
+    const structure = decisions.get(node.id) as StructureDecision;
+    const inferredGroup = structure.kind === "inferred-group";
+    structuredNodes.set(node.id, {
+      ...node,
+      vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup),
+      structure,
+      ...(kept.has(node.id)
+        ? {
+            semanticParentId: finalParentId(
+              graph,
+              node,
+              kept,
+              decisions,
+              axParentMemo,
+              domParentMemo,
+            ),
+          }
+        : {}),
+    });
+  }
+  return { ...graph, nodes: structuredNodes };
+}

apps/extension/src/tools/vom/semantic-graph/types.ts

@@ -0,0 +1,88 @@
+import type { Viewport, VomNode } from "@browser-skill/vom";
+import type { CdpTarget } from "@/browser-driver/frame-graph";
+import type { CapturedNode } from "../capture";
+
+export interface SemanticAxNode {
+  nodeId: string;
+  frameId?: string;
+  parentId?: string;
+  backendDOMNodeId?: number;
+  ignored?: boolean;
+  role?: { type: string; value?: string };
+  name?: { type: string; value?: string };
+  description?: { value?: string };
+  value?: { value?: string | number | boolean };
+  properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
+  childIds?: string[];
+}
+
+export type SemanticNodeId = string;
+
+export interface SemanticFrame {
+  frameId: string;
+  parentFrameId?: string;
+  ownerBackendNodeId?: number;
+  ownerNodeId?: SemanticNodeId;
+  contextScopeId: string;
+  target: CdpTarget;
+  url?: string;
+  order: number;
+}
+
+export interface SemanticGraphNode {
+  id: SemanticNodeId;
+  frameId: string;
+  backendNodeId?: number;
+  axNodeId?: string;
+  ax?: SemanticAxNode;
+  dom?: CapturedNode;
+  axParentId?: SemanticNodeId;
+  domParentId?: SemanticNodeId;
+  sourceOrder: number;
+  excluded: boolean;
+}
+
+export interface SemanticGraph {
+  viewport: Viewport;
+  rootFrameId: string;
+  frames: Map<string, SemanticFrame>;
+  nodes: Map<SemanticNodeId, SemanticGraphNode>;
+  nodeIdsByFrame: Map<string, SemanticNodeId[]>;
+  nodeByFrameBackend: Map<string, SemanticNodeId>;
+}
+
+export interface ResolvedSemanticNode extends SemanticGraphNode {
+  vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
+  referenceable: boolean;
+  roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none";
+}
+
+export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
+  nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
+}
+
+export type StructureDecision =
+  | { kind: "keep"; reason: "semantic" | "frame-owner" }
+  | { kind: "inferred-group"; reason: "independent-interaction-branches" }
+  | { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
+  | {
+      kind: "excluded";
+      reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback";
+    };
+
+export interface StructuredSemanticNode extends ResolvedSemanticNode {
+  structure: StructureDecision;
+  semanticParentId?: SemanticNodeId;
+}
+
+export interface StructuredSemanticGraph extends Omit<ResolvedSemanticGraph, "nodes"> {
+  nodes: Map<SemanticNodeId, StructuredSemanticNode>;
+}
+
+export function frameBackendKey(frameId: string, backendNodeId: number): string {
+  return `${frameId}\u0000${backendNodeId}`;
+}
+
+export function frameAxKey(frameId: string, axNodeId: string): string {
+  return `${frameId}\u0000${axNodeId}`;
+}

apps/extension/src/transport/types.ts

@@ -23,6 +23,7 @@ export type ErrorCode =
 export type RpcErrorReason =
   | "agent_window_scope"
   | "element_not_visible"
+  | "ref_capability_denied"
   | "ref_not_found"
   | "selector_not_found"
   | "target_not_fillable"

apps/extension/test-fixtures/rendered-surfaces/README.md

@@ -0,0 +1,105 @@
+# Rendered Surface browser fixtures
+
+These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
+
+## Serve
+
+From the repository root:
+
+```bash
+python3 -m http.server 4173 --bind 127.0.0.1 \
+  --directory apps/extension/test-fixtures/rendered-surfaces
+```
+
+Use an Agent Window large enough to keep the primary matrix visible:
+
+```bash
+bsk session start --width 1440 --height 1100
+bsk navigate http://127.0.0.1:4173/index.html --session <id>
+bsk observe --session <id>
+```
+
+Always stop the session after the run.
+
+## Pages
+
+### `index.html`
+
+Expected individual Surface labels:
+
+- `fixture-basic-large`
+- `fixture-tiny-8px`
+- `fixture-pointer-events-none`
+- `fixture-partial-ax`
+- `fixture-contained-outer`
+- `fixture-contained-inner`
+- `fixture-different-parent-a`
+- `fixture-different-parent-b`
+
+Expected one grouped Surface:
+
+- `fixture-layered-group`, with `layers=2`
+
+Expected absent:
+
+- `fixture-display-none`
+- `fixture-visibility-hidden`
+- `fixture-opacity-zero`
+- `fixture-parent-opacity-zero`
+- `fixture-hidden-attribute`
+- `fixture-inert-parent`
+- `fixture-aria-hidden-parent`
+- `fixture-zero-size`
+- `fixture-offscreen-right`
+- `fixture-fully-overflow-clipped`
+
+`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
+
+### `unlabeled.html`
+
+This page contains eight Canvas elements but must render seven generic Surface markers because
+the two coincident layers form one group.
+
+- Every marker uses the stable fallback label `canvas visual surface`.
+- Exactly one marker has `layers=2`.
+- The two contained canvases remain separate markers.
+- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
+- Blank `aria-label` and `title` values behave as no label.
+- Fallback AX content does not suppress the corresponding Surface.
+
+### `frames.html`
+
+Expected Surface labels:
+
+- `fixture-same-origin-frame`
+- `fixture-cross-origin-frame`
+- `fixture-nested-frame`
+
+The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
+
+### `viewport.html`
+
+Run with a `900x700` Agent Window.
+
+- `fixture-partial-viewport` is partly visible and must be represented.
+- `fixture-below-fold` is absent initially.
+- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
+
+### `dynamic.html`
+
+Use the normal DOM buttons and re-observe after each action:
+
+- Initial state: `fixture-dynamic` absent.
+- After **Insert canvas**: present.
+- After **Resize to zero**: absent.
+- After **Restore size**: present with a fresh ref.
+- After **Remove canvas**: absent.
+
+## Invariants
+
+- `snapshot` must not emit any of these Surface markers.
+- `observe` must emit only currently visible Canvas surfaces.
+- Existing DOM/AX controls remain readable alongside Surface markers.
+- Missing labels never prevent discovery; naming and discovery are independent.
+- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
+- No fixture URL, id, label, or layout constant may appear in production discovery code.

apps/extension/test-fixtures/rendered-surfaces/dynamic.html

@@ -0,0 +1,46 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface dynamic matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 16px; }
+      .controls { display: flex; gap: 8px; margin-bottom: 16px; }
+      button { padding: 8px 12px; }
+      canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
+    </style>
+  </head>
+  <body>
+    <h1>Dynamic Canvas lifecycle</h1>
+    <div class="controls">
+      <button id="insert" type="button">Insert canvas</button>
+      <button id="zero" type="button">Resize to zero</button>
+      <button id="restore" type="button">Restore size</button>
+      <button id="remove" type="button">Remove canvas</button>
+    </div>
+    <div id="mount"></div>
+    <script>
+      const mount = document.querySelector("#mount");
+      const getCanvas = () => mount.querySelector("canvas");
+      document.querySelector("#insert").addEventListener("click", () => {
+        if (getCanvas()) return;
+        const canvas = document.createElement("canvas");
+        canvas.width = 260;
+        canvas.height = 120;
+        canvas.setAttribute("aria-label", "fixture-dynamic");
+        mount.append(canvas);
+      });
+      document.querySelector("#zero").addEventListener("click", () => {
+        const canvas = getCanvas();
+        if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
+      });
+      document.querySelector("#restore").addEventListener("click", () => {
+        const canvas = getCanvas();
+        if (canvas) canvas.removeAttribute("style");
+      });
+      document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
+    </script>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/frame.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 12px; background: #fdfefe; }
+      canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
+      iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
+    </style>
+  </head>
+  <body>
+    <canvas id="frame-canvas" width="220" height="90"></canvas>
+    <script>
+      const params = new URLSearchParams(location.search);
+      const canvas = document.querySelector("#frame-canvas");
+      if (params.get("canvas") === "0") canvas.remove();
+      else canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
+      if (params.get("nested") === "1") {
+        const iframe = document.createElement("iframe");
+        iframe.title = "nested fixture frame";
+        iframe.src = "frame.html?label=fixture-nested-frame";
+        document.body.append(iframe);
+      }
+    </script>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/frames.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface frame matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 16px; }
+      .frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
+      iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
+    </style>
+  </head>
+  <body>
+    <h1>Frame matrix</h1>
+    <div class="frames">
+      <iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame"></iframe>
+      <iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame"></iframe>
+      <iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
+    </div>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/index.html

@@ -0,0 +1,118 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface discovery matrix</title>
+    <style>
+      * { box-sizing: border-box; }
+      html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
+      body { padding: 16px; background: #f4f6f8; color: #17202a; }
+      h1 { margin: 0 0 12px; font-size: 20px; }
+      .matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
+      .case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
+      .case h2 { margin: 0 0 8px; font-size: 13px; }
+      canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
+      .stack { position: relative; width: 180px; height: 90px; }
+      .stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
+      .stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
+      .contained { position: relative; width: 190px; height: 100px; }
+      .contained > canvas:first-child { width: 190px; height: 100px; }
+      .contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
+      .same-place { position: relative; width: 180px; height: 90px; }
+      .same-place > div { position: absolute; inset: 0; }
+      .same-place canvas { width: 180px; height: 90px; }
+      .same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
+      .clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
+      .partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
+      .fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
+      .tiny { width: 8px; height: 8px; border-width: 1px; }
+    </style>
+  </head>
+  <body>
+    <h1>Rendered Surface discovery matrix</h1>
+    <main class="matrix">
+      <section class="case">
+        <h2>Visible named canvas</h2>
+        <canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Tiny canvas remains addressable</h2>
+        <canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Visible but pointer-events none</h2>
+        <canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Partial AX/fallback content</h2>
+        <canvas aria-label="fixture-partial-ax" width="180" height="80">
+          <button type="button">Fallback button must not suppress the surface</button>
+        </canvas>
+      </section>
+
+      <section class="case">
+        <h2>Two real rendering layers</h2>
+        <div class="stack">
+          <canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
+          <canvas width="180" height="90"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Contained canvases are independent</h2>
+        <div class="contained">
+          <canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
+          <canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Same geometry, different parents</h2>
+        <div class="same-place">
+          <div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
+          <div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Partially overflow-clipped</h2>
+        <div class="clip-box">
+          <canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Fully overflow-clipped</h2>
+        <div class="clip-box">
+          <canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>CSS-hidden variants</h2>
+        <canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
+        <canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
+        <canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Hidden ancestor variants</h2>
+        <div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
+        <div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
+        <div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
+        <div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
+      </section>
+
+      <section class="case">
+        <h2>Zero-size and offscreen</h2>
+        <canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
+        <canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
+      </section>
+    </main>
+  </body>
+</html>
+

apps/extension/test-fixtures/rendered-surfaces/unlabeled.html

@@ -0,0 +1,68 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Unlabeled Rendered Surface matrix</title>
+    <style>
+      * { box-sizing: border-box; }
+      html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
+      body { padding: 16px; background: #f4f6f8; color: #17202a; }
+      h1 { margin: 0 0 12px; font-size: 20px; }
+      .matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
+      .case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
+      .case h2 { margin: 0 0 8px; font-size: 13px; }
+      .case p { margin: 0 0 8px; }
+      canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
+      .stack { position: relative; width: 180px; height: 90px; }
+      .stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
+      .stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
+      .contained { position: relative; width: 190px; height: 100px; }
+      .contained > canvas:first-child { width: 190px; height: 100px; }
+      .contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
+    </style>
+  </head>
+  <body>
+    <h1>Unlabeled Rendered Surface matrix</h1>
+    <main class="matrix">
+      <section class="case">
+        <h2>Only engineering attributes</h2>
+        <canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Blank explicit names</h2>
+        <canvas aria-label="   " title="   " width="180" height="80"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Nearby prose is not a Canvas name</h2>
+        <p>must-not-be-inferred-as-a-surface-label</p>
+        <canvas width="180" height="80"></canvas>
+      </section>
+
+      <section class="case">
+        <h2>Fallback AX content does not name or suppress the Canvas</h2>
+        <canvas width="180" height="80">
+          <button type="button">Fallback control remains separate</button>
+        </canvas>
+      </section>
+
+      <section class="case">
+        <h2>Unnamed rendering layers still cluster</h2>
+        <div class="stack">
+          <canvas width="180" height="90"></canvas>
+          <canvas width="180" height="90"></canvas>
+        </div>
+      </section>
+
+      <section class="case">
+        <h2>Unnamed contained canvases stay independent</h2>
+        <div class="contained">
+          <canvas width="190" height="100"></canvas>
+          <canvas width="70" height="40"></canvas>
+        </div>
+      </section>
+    </main>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/viewport.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface viewport matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { min-height: 1800px; padding: 16px; }
+      canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
+      #partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
+      #below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
+    </style>
+  </head>
+  <body>
+    <h1>Viewport matrix</h1>
+    <p>The orange canvas on the left is intentionally clipped by the viewport.</p>
+    <canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
+    <canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
+  </body>
+</html>
+

crates/bsk-cli/skill/SKILL.md

@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
 
 When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
 
+When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
+
 ## Observation priority
 
 Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:

crates/bsk-cli/src/cli/render_error.rs

@@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode;
 pub mod reason {
     pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
     pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
+    pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
     pub const REF_NOT_FOUND: &str = "ref_not_found";
     pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
     pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
@@ -228,6 +229,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
             ),
             exit_code: base.exit_code,
         },
+        (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
+            summary: "snapshot ref does not support this operation",
+            hint: Some(
+                "visual surface refs are screenshot-only; inspect `bsk screenshot --ref <ref> --session <id>` with image-understanding capability",
+            ),
+            exit_code: base.exit_code,
+        },
         (ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
             summary: "target element is not a <select>",
             hint: Some(
@@ -377,6 +385,18 @@ mod tests {
         assert!(!info.summary.contains("sandbox"));
     }
 
+    #[test]
+    fn ref_capability_denied_explains_visual_surface_refs() {
+        let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
+        let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
+        assert_eq!(info.summary, "snapshot ref does not support this operation");
+        assert!(
+            info.hint.unwrap().contains("screenshot-only"),
+            "expected visual-surface-specific hint"
+        );
+        assert_eq!(info.exit_code, 1);
+    }
+
     #[test]
     fn session_busy_overrides_timeout_copy() {
         let data = serde_json::json!({ "reason": reason::SESSION_BUSY });

packages/dsh-plugin-browserskill/package.json

@@ -107,6 +107,8 @@
     "@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
     "@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
     "@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-scope": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-skill": "^0.1.0-rc.6",
     "@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
     "@deepseek-ai/schemastery": "^3.18.1",
     "@tailwindcss/cli": "^4.3.3",

packages/dsh-plugin-browserskill/src/index.ts

@@ -19,7 +19,7 @@ import { registerObservationRoutes } from "./observation-http";
 import { KeyedExecutor } from "./queue";
 import { type BskRunner, createBskRunner } from "./runner";
 import { SessionRegistry } from "./sessions";
-import { registerBskSkill } from "./skill";
+import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
 import { type PluginConfig, registerTools } from "./tools";
 
 export const name = "dsh-plugin-browserskill";
@@ -95,6 +95,11 @@ export function apply(
   // ONLY model-visible advertisement — the tool suite reveals itself on a
   // successful skill invocation (or a session whose history already has one).
   const unregisterSkill = registerBskSkill(ctx);
+  // A same-name CLI skill left in ~/.agents is discovered in the nearer preset
+  // layer and shadows the global registration above. Re-register through every
+  // exact agent context at startup so DSH always loads the browser_* protocol
+  // instructions; the shared CLI skill remains untouched for other agents.
+  const disarmAgentSkill = armAgentScopedBskSkill(ctx);
   const removeSuite = resolved.lazyTools
     ? armLazyTools(ctx, () =>
         registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
@@ -135,17 +140,15 @@ export function apply(
   ctx.effect(() => {
     return () => {
       removeSuite();
+      disarmAgentSkill();
       unregisterSkill();
       removeRoutes();
       disarmArchiveCleanup();
-      observation.dispose();
       runner.killAll();
       const stops = registry
         .ownedIds()
-        .map((sessionId) =>
+        .map((sessionId) => observation.stopSession(sessionId).catch(() => false));
-          runner.run(["session", "stop", sessionId], { timeoutMs: 15_000 }).catch(() => {}),
+      return Promise.all(stops).then(() => observation.dispose());
-        );
-      return Promise.all(stops).then(() => {});
     };
   });
 }

packages/dsh-plugin-browserskill/src/observation.ts

@@ -18,7 +18,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import type { Context } from "@deepseek-ai/cordis";
 import type { KeyedExecutor } from "./queue";
-import type { BskRunner } from "./runner";
+import { BskError, type BskRunner, parseBskJson, runWithSessionBusyRetry } from "./runner";
 import type { SessionRegistry } from "./sessions";
 
 /** One owned session's live observation record (wire-stable shape). */
@@ -90,6 +90,10 @@ export class ObservationService {
   private readonly listeners = new Set<(event: ObservationEvent) => void>();
   private readonly captureTimers = new Map<string, unknown>();
   private readonly captureInFlight = new Set<string>();
+  /** Captures intentionally cancelled to make way for foreground work. */
+  private readonly capturePreempted = new Set<string>();
+  /** Number of queued/running model-facing calls that currently outrank thumbnails. */
+  private readonly foregroundDepth = new Map<string, number>();
   private readonly captureFailures = new Map<string, number>();
   private readonly lastActivity = new Map<string, number>();
   /** Ephemeral frame id → PNG bytes. Only the live ring members are present. */
@@ -175,6 +179,8 @@ export class ObservationService {
   /** Drop a session (called from browser_session_stop). */
   removeSession(sessionId: string): void {
     this.cancelCapture(sessionId);
+    this.foregroundDepth.delete(sessionId);
+    this.capturePreempted.delete(sessionId);
     this.captureFailures.delete(sessionId);
     this.lastActivity.delete(sessionId);
     this.dropSessionFrames(sessionId);
@@ -186,6 +192,37 @@ export class ObservationService {
     }
   }
 
+  /**
+   * Give model-facing work priority over the best-effort thumbnail lane.
+   * The first lease cancels a pending timer and gracefully interrupts an
+   * active bsk screenshot; the last release schedules one fresh frame.
+   */
+  acquireForeground(sessionId: string): () => void {
+    if (!this.deps.options.enabled || this.disposed || !this.observations.has(sessionId)) {
+      return () => {};
+    }
+    const depth = this.foregroundDepth.get(sessionId) ?? 0;
+    this.foregroundDepth.set(sessionId, depth + 1);
+    if (depth === 0) {
+      this.cancelCapture(sessionId);
+      if (this.deps.runner.killFor(`observation:${sessionId}`) > 0) {
+        this.capturePreempted.add(sessionId);
+      }
+    }
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      const remaining = (this.foregroundDepth.get(sessionId) ?? 1) - 1;
+      if (remaining > 0) {
+        this.foregroundDepth.set(sessionId, remaining);
+        return;
+      }
+      this.foregroundDepth.delete(sessionId);
+      if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId, 0);
+    };
+  }
+
   /** Mark an action starting on a session (tool entry instrumentation). */
   beginAction(sessionId: string, action: string): void {
     if (!this.deps.options.enabled || this.disposed) return;
@@ -213,7 +250,7 @@ export class ObservationService {
     if (error !== undefined) next.lastError = error;
     else delete next.lastError;
     this.put(next);
-    this.scheduleCapture(sessionId, 0);
+    if (!this.foregroundDepth.has(sessionId)) this.scheduleCapture(sessionId, 0);
   }
 
   /** Record the settled page URL (navigate success / start with url). */
@@ -242,29 +279,48 @@ export class ObservationService {
    * button — same end state as `browser_session_stop`). Never waits behind a
    * hung in-flight command: tool children are killed first so the session's
    * keyed queue drains immediately, and no further captures queue up. A
-   * session the daemon already forgot (dead strip entries) stops
+   * session the daemon already forgot stops idempotently — the goal state
-   * idempotently — the goal state (entry gone) is identical.
+   * (entry gone) is identical.
-   * @returns whether the session ended up stopped/removed.
+   * @returns false only for a foreign session; bsk failures reject so callers
+   * can preserve the structured error instead of silently leaving a ghost.
    */
-  async stopSession(sessionId: string): Promise<boolean> {
+  async stopSession(sessionId: string, signal?: AbortSignal): Promise<boolean> {
     if (!this.deps.registry.isOwned(sessionId)) return false;
-    this.cancelCapture(sessionId);
+    const releaseForeground = this.acquireForeground(sessionId);
-    this.deps.runner.killFor(sessionId);
+    let actionError: string | undefined;
-    const result = await this.deps.queue.run(sessionId, () =>
+    this.beginAction(sessionId, "stopping");
-      this.deps.runner.run(["session", "stop", sessionId], { timeoutMs: 30_000 }),
+    try {
-    );
+      this.deps.runner.killFor(sessionId);
-    if (result.code !== 0) {
+      const result = await this.deps.queue.run(
-      let code: string | undefined;
+        sessionId,
+        () =>
+          runWithSessionBusyRetry(
+            () =>
+              this.deps.runner.run(["session", "stop", sessionId], {
+                signal,
+                timeoutMs: 30_000,
+                tag: sessionId,
+              }),
+            signal,
+          ),
+        signal,
+      );
+      if (result.aborted) throw abortError();
       try {
-        code = (JSON.parse(result.stdout) as { code?: string }).code;
+        parseBskJson(result, "session stop");
-      } catch {
+      } catch (error) {
-        code = undefined;
+        if (!isSessionNotFoundError(error)) throw error;
       }
-      if (code !== "session_not_found") return false;
+      this.deps.registry.remove(sessionId);
+      this.removeSession(sessionId);
+      return true;
+    } catch (error) {
+      actionError = error instanceof Error ? error.message.split("\n")[0] : String(error);
+      throw error;
+    } finally {
+      this.endAction(sessionId, actionError);
+      releaseForeground();
     }
-    this.deps.registry.remove(sessionId);
-    this.removeSession(sessionId);
-    return true;
   }
 
   /**
@@ -287,6 +343,8 @@ export class ObservationService {
     for (const sessionId of [...this.captureTimers.keys()]) this.cancelCapture(sessionId);
     this.captureTimers.clear();
     this.captureInFlight.clear();
+    this.capturePreempted.clear();
+    this.foregroundDepth.clear();
     this.captureFailures.clear();
     this.lastActivity.clear();
     this.frames.clear();
@@ -313,6 +371,7 @@ export class ObservationService {
   /** Schedule the next capture for a session; `delayMs` 0 means "as soon as the event loop allows". */
   private scheduleCapture(sessionId: string, delayMs?: number): void {
     if (!this.deps.options.enabled || this.disposed) return;
+    if (this.foregroundDepth.has(sessionId)) return;
     const entry = this.observations.get(sessionId);
     if (entry === undefined || entry.dead === true) return;
     this.cancelCapture(sessionId);
@@ -349,12 +408,15 @@ export class ObservationService {
     const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
     let writtenPath = outPath;
     try {
-      const result = await this.deps.queue.run(sessionId, () =>
+      const result = await this.deps.queue.run(sessionId, () => {
-        this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
+        // A foreground lease may have arrived while this capture was queued.
+        if (this.foregroundDepth.has(sessionId)) return Promise.resolve(undefined);
+        return this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
           timeoutMs: 15_000,
           tag: `observation:${sessionId}`,
-        }),
+        });
-      );
+      });
+      if (result === undefined) return;
       if (result.code !== 0) {
         let code: string | undefined;
         try {
@@ -362,8 +424,12 @@ export class ObservationService {
         } catch {
           code = undefined;
         }
-        if (code === "session_not_found") {
+        if (isSessionNotFoundCode(code)) {
-          this.markDead(sessionId);
+          // Observation entries are owned sessions. If the daemon has already
+          // forgotten one (external close / restart), reconcile both stores so
+          // the PiP cannot retain a ghost strip or keep retrying screenshots.
+          this.deps.registry.remove(sessionId);
+          this.removeSession(sessionId);
           return;
         }
         throw new Error(`screenshot exited ${result.code}`);
@@ -378,12 +444,17 @@ export class ObservationService {
       this.setAvailable(true);
       this.publishFrame(sessionId, entry, data);
     } catch {
-      // Silent by design: keep the previous frame, count toward backoff.
+      // Foreground preemption is healthy scheduling, not browser/daemon
-      this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
+      // unavailability; do not let normal tool traffic trip capture backoff.
-      this.globalFailures += 1;
+      if (!this.capturePreempted.delete(sessionId)) {
-      if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
+        // Silent by design: keep the previous frame, count toward backoff.
+        this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
+        this.globalFailures += 1;
+        if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
+      }
     } finally {
       await unlink(writtenPath).catch(() => {});
+      this.capturePreempted.delete(sessionId);
       this.captureInFlight.delete(sessionId);
       if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
     }
@@ -414,14 +485,20 @@ export class ObservationService {
     this.hashes.delete(sessionId);
     this.seqs.delete(sessionId);
   }
+}
 
-  /** Mark a session dead: grey it out, stop asking for frames, keep the entry for the strip. */
+function isSessionNotFoundCode(code: string | undefined): boolean {
-  private markDead(sessionId: string): void {
+  return code === "not_found" || code === "session_not_found";
-    const entry = this.observations.get(sessionId);
+}
-    if (entry === undefined || entry.dead === true) return;
+
-    this.cancelCapture(sessionId);
+function isSessionNotFoundError(error: unknown): boolean {
-    this.put({ ...entry, dead: true, action: "idle" });
+  return error instanceof BskError && isSessionNotFoundCode(error.code);
-  }
+}
+
+function abortError(): Error {
+  const error = new Error("tool call aborted");
+  error.name = "AbortError";
+  return error;
 }
 
 /** Map a bsk command label onto its observation action verb. */

packages/dsh-plugin-browserskill/src/runner.ts

@@ -64,14 +64,17 @@ export interface BskRunner {
   killFor(tag: string): number;
 }
 
-const KILL_GRACE_MS = 2000;
+// Business RPCs translate SIGINT into the daemon's cancel(rpc_id) protocol.
+// Give that bounded reconciliation path time to settle before the hard kill.
+const KILL_GRACE_MS = 3000;
+const SESSION_BUSY_RETRY_DELAY_MS = 100;
 
 export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
   const live = new Map<ChildProcess, string | undefined>();
 
   function killChild(child: ChildProcess): void {
     if (child.exitCode !== null || child.signalCode !== null) return;
-    child.kill("SIGTERM");
+    child.kill("SIGINT");
     const force = setTimeout(() => {
       if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
     }, KILL_GRACE_MS);
@@ -163,6 +166,60 @@ export function isCommandNotFound(error: unknown): boolean {
   );
 }
 
+/** True only for the daemon's transient per-session reconciliation window. */
+export function isSessionBusyResult(result: BskRunResult): boolean {
+  if (result.code === 0) return false;
+  try {
+    const body = JSON.parse(result.stdout) as BskErrorBody;
+    const reason =
+      typeof body.data === "object" && body.data !== null && "reason" in body.data
+        ? (body.data as { reason?: unknown }).reason
+        : undefined;
+    return reason === "session_busy";
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Retry exactly once after the tiny daemon-settlement race that can follow a
+ * graceful SIGINT cancellation. Other errors and persistent busy states stay
+ * visible to callers.
+ */
+export async function runWithSessionBusyRetry(
+  run: () => Promise<BskRunResult>,
+  signal?: AbortSignal,
+): Promise<BskRunResult> {
+  const first = await run();
+  if (!isSessionBusyResult(first)) return first;
+  await abortableDelay(SESSION_BUSY_RETRY_DELAY_MS, signal);
+  return run();
+}
+
+function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.reject(abortError());
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(done, ms);
+    timer.unref();
+    const onAbort = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener("abort", onAbort);
+      reject(abortError());
+    };
+    function done() {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+function abortError(): Error {
+  const error = new Error("tool call aborted");
+  error.name = "AbortError";
+  return error;
+}
+
 /** Install guidance shown when the bsk CLI cannot be spawned. */
 export function bskInstallMessage(bskPath: string): string {
   return (
@@ -182,7 +239,7 @@ export function parseBskJson(result: BskRunResult, commandLabel: string): unknow
     throw new BskError(`bsk ${commandLabel} timed out`, { timedOut: true });
   }
   const body = result.stdout.trim();
-  // Killed by our own interrupt (SIGTERM from killFor), not by the abort path:
+  // Killed by our own interrupt (SIGINT from killFor), not by the abort path:
   // say so instead of doubling the generic label into the message.
   if (result.code === null && !result.aborted && !result.timedOut) {
     throw new BskError(`bsk ${commandLabel} was interrupted (process killed)`);

packages/dsh-plugin-browserskill/src/skill.ts

@@ -9,6 +9,7 @@
  */
 
 import type { Context } from "@deepseek-ai/cordis";
+import type { Agent } from "@deepseek-ai/dsh-agent";
 import {
   BSK_SKILL_DESCRIPTION,
   BSK_SKILL_MARKDOWN,
@@ -25,6 +26,11 @@ interface SkillsLike {
   }): () => void;
 }
 
+/** Structural view of the live-agent registry (absent in bare compositions). */
+interface AgentsLike {
+  list(): Agent[];
+}
+
 /**
  * Publish the browser skill as an embedded runtime skill. Returns the unregistration
  * disposer. A composition without the `skills` service (or with dsh-tool-skill
@@ -43,3 +49,54 @@ export function registerBskSkill(ctx: Context): () => void {
     source: "bundled",
   });
 }
+
+/**
+ * Install the DSH-specific browser skill into every exact agent scope.
+ *
+ * DSH merges skill layers nearest-first (`agent -> preset -> global`). A legacy
+ * CLI skill installed at `~/.agents/skills/browser-skill` is discovered by the
+ * preset filesystem provider and therefore shadows a plugin-level registration.
+ * Registering the embedded skill through `agent.ctx` makes the DSH protocol
+ * contract authoritative for that agent without touching the shared CLI skill.
+ *
+ * New agents are handled at `agent/session-start`, the first supported startup
+ * injection point and still before the first prompt assembly. Existing agents
+ * are registered immediately so plugin reloads take effect without recreating
+ * the conversation. Returns a disposer for all plugin-owned registrations.
+ */
+export function armAgentScopedBskSkill(ctx: Context): () => void {
+  const registrations = new Map<Agent, () => void>();
+  let active = true;
+
+  const registerForAgent = (agent: Agent): void => {
+    if (!active || registrations.has(agent)) return;
+    registrations.set(agent, registerBskSkill(agent.ctx));
+  };
+
+  let stopSessionStart = () => {};
+  let stopDisposed = () => {};
+  if (typeof ctx.on === "function") {
+    stopSessionStart = ctx.on("agent/session-start", ({ agent }) => {
+      registerForAgent(agent);
+    });
+    stopDisposed = ctx.on("agent/disposed", ({ agent }) => {
+      // Agent-scoped effects have already unwound at this lifecycle edge. Drop
+      // the retained disposer without invoking it against a disposed context.
+      registrations.delete(agent);
+    });
+  }
+
+  const agents = ctx.get("agents") as AgentsLike | undefined | null;
+  if (agents != null && typeof agents.list === "function") {
+    for (const agent of agents.list()) registerForAgent(agent);
+  }
+
+  return () => {
+    if (!active) return;
+    active = false;
+    stopDisposed();
+    stopSessionStart();
+    for (const unregister of [...registrations.values()].reverse()) unregister();
+    registrations.clear();
+  };
+}

packages/dsh-plugin-browserskill/src/tools.ts

@@ -22,7 +22,13 @@ import { trySaveScreenshot } from "./image";
 import { actionForLabel, type ObservationService } from "./observation";
 import { registerPhaseOneTools } from "./phase-one-tools";
 import type { KeyedExecutor } from "./queue";
-import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
+import {
+  type BskRunner,
+  bskInstallMessage,
+  isCommandNotFound,
+  parseBskJson,
+  runWithSessionBusyRetry,
+} from "./runner";
 import type { SessionRegistry } from "./sessions";
 import { SESSION_PARAM } from "./tool-params";
 
@@ -79,6 +85,8 @@ async function runBsk(
   observeSession?: string,
   runnerTimeoutMs?: number,
 ): Promise<unknown> {
+  const releaseForeground =
+    observeSession !== undefined ? deps.observation.acquireForeground(observeSession) : undefined;
   // beginAction must fire only when the task actually starts inside the
   // per-session queue — marking it while still queued would overwrite the
   // in-flight action's label (and flash it idle on a queued abort).
@@ -92,11 +100,15 @@ async function runBsk(
           began = true;
           deps.observation.beginAction(observeSession, actionForLabel(label));
         }
-        return deps.runner.run(args, {
+        return runWithSessionBusyRetry(
-          signal: exec.signal,
+          () =>
-          timeoutMs: runnerTimeoutMs ?? deps.config.defaultTimeoutMs,
+            deps.runner.run(args, {
-          ...(observeSession !== undefined ? { tag: observeSession } : {}),
+              signal: exec.signal,
-        });
+              timeoutMs: runnerTimeoutMs ?? deps.config.defaultTimeoutMs,
+              ...(observeSession !== undefined ? { tag: observeSession } : {}),
+            }),
+          exec.signal,
+        );
       };
       result =
         observeSession !== undefined
@@ -121,6 +133,7 @@ async function runBsk(
     if (observeSession !== undefined && began) {
       deps.observation.endAction(observeSession, actionError);
     }
+    releaseForeground?.();
   }
 }
 
@@ -267,22 +280,15 @@ export function registerTools(deps: ToolDeps): () => void {
           }
         } catch (error) {
           // A half-initialized session must not leak: stop it before surfacing.
-          // The stop funnels through the session's queue (an in-flight command
+          // Reuse the same capture-preempting, idempotent path as explicit and
-          // — e.g. an observation capture — would otherwise make the daemon
+          // overlay stops; local ownership is dropped even when daemon cleanup
-          // refuse it), with one retry for good measure.
+          // itself fails, because this start never becomes usable to the model.
-          for (let attempt = 0; attempt < 2; attempt++) {
+          try {
-            try {
+            await deps.observation.stopSession(reply.session_id);
-              const stopped = await deps.queue.run(reply.session_id, () =>
+          } catch {
-                deps.runner.run(["session", "stop", reply.session_id], { timeoutMs: 30_000 }),
+            registry.remove(reply.session_id);
-              );
+            deps.observation.removeSession(reply.session_id);
-              if (stopped.code === 0) break;
-            } catch {
-              // retry once, then give up quietly (registry/observation are
-              // cleaned below either way; the daemon reaps orphans).
-            }
           }
-          registry.remove(reply.session_id);
-          deps.observation.removeSession(reply.session_id);
           throw error;
         }
         return {
@@ -326,9 +332,15 @@ export function registerTools(deps: ToolDeps): () => void {
       },
       async execute(args, exec) {
         const sessionId = registry.resolveForStop(args.session);
-        await runBsk(deps, exec, ["session", "stop", sessionId], "session stop", sessionId);
+        try {
-        registry.remove(sessionId);
+          const stopped = await deps.observation.stopSession(sessionId, exec.signal);
-        deps.observation.removeSession(sessionId);
+          if (!stopped) throw new Error(`browser session ${sessionId} is not owned by this plugin`);
+        } catch (error) {
+          if (isCommandNotFound(error)) {
+            throw new Error(bskInstallMessage(deps.config.bskPath));
+          }
+          throw error;
+        }
         return { stopped: sessionId };
       },
       presentCall: (args) => ({
@@ -485,7 +497,9 @@ export function registerTools(deps: ToolDeps): () => void {
       ? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
         "carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
       : "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
-        "probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
+        "probes) with @eN refs. Visual-only surfaces require image understanding and their refs are " +
+        "screenshot-only; keep using any DOM/AX semantics rendered alongside them. " +
+        "Read-only: never submits input.";
     register(
       defineTool({
         name,
@@ -748,12 +762,17 @@ export function registerTools(deps: ToolDeps): () => void {
       name: "browser_screenshot",
       description:
         "Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
+        "This is the supported operation for visual surface refs emitted by browser_observe. Only use " +
+        "it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
+        "or image-reading capability, tell the user to switch to a model with image understanding; do " +
+        "not guess content or coordinates. " +
         "Returns the image itself when the deployment supports image input, otherwise a file path.",
       parameters: {
         session: SESSION_PARAM,
         ref: {
           type: "string",
-          description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
+          description:
+            "Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
         },
       },
       output: {

packages/dsh-plugin-browserskill/tests/observation.test.ts

@@ -78,7 +78,7 @@ function fakeRunner(
         if (opts.stopNotFound) {
           return {
             code: 4,
-            stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
+            stdout: JSON.stringify({ code: "not_found", message: "no such session" }),
             stderr: "",
             timedOut: false,
             aborted: false,
@@ -93,7 +93,7 @@ function fakeRunner(
         if (opts.screenshotNotFound) {
           return {
             code: 4,
-            stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
+            stdout: JSON.stringify({ code: "not_found", message: "no such session" }),
             stderr: "",
             timedOut: false,
             aborted: false,
@@ -224,6 +224,57 @@ describe("state machine", () => {
 });
 
 describe("thumbnail cadence", () => {
+  it("gives foreground work priority and resumes with one fresh capture", async () => {
+    const scheduler = fakeScheduler();
+    let finishCapture: ((result: BskRunResult) => void) | undefined;
+    const killed: string[] = [];
+    const runner: FakeRunner = {
+      calls: [],
+      killed,
+      run(args) {
+        if (args[0] !== "screenshot") {
+          return Promise.resolve({
+            code: 0,
+            stdout: "{}",
+            stderr: "",
+            timedOut: false,
+            aborted: false,
+          });
+        }
+        return new Promise((resolve) => {
+          finishCapture = resolve;
+        });
+      },
+      killAll() {},
+      killFor(tag) {
+        killed.push(tag);
+        if (tag === "observation:s1" && finishCapture !== undefined) {
+          finishCapture({
+            code: 5,
+            stdout: JSON.stringify({ code: "cancelled", message: "cancelled" }),
+            stderr: "",
+            timedOut: false,
+            aborted: false,
+          });
+          finishCapture = undefined;
+          return 1;
+        }
+        return 0;
+      },
+    };
+    const { service } = setup({ runner, scheduler });
+    service.addSession("s1");
+    scheduler.runNext();
+    await waitFor(() => finishCapture !== undefined);
+
+    const release = service.acquireForeground("s1");
+    expect(killed).toEqual(["observation:s1"]);
+    await waitFor(() => scheduler.pending().length === 0);
+    release();
+    expect(service.isAvailable()).toBe(true);
+    expect(scheduler.pending()).toEqual([0]);
+  });
+
   it("captures immediately on add and after action end, then fast-cadences while active", async () => {
     const scheduler = fakeScheduler();
     const runner = fakeRunner();
@@ -351,7 +402,7 @@ describe("stopSession", () => {
     expect(registry.isOwned("s1")).toBe(true);
 
     await expect(service.stopSession("s1")).resolves.toBe(true);
-    expect(runner.killed).toEqual(["s1"]);
+    expect(runner.killed).toEqual(["observation:s1", "s1"]);
     const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
     expect(stop?.args).toEqual(["session", "stop", "s1"]);
     expect(registry.isOwned("s1")).toBe(false);
@@ -380,13 +431,13 @@ describe("stopSession", () => {
     expect(events[events.length - 1]).toMatchObject({ type: "remove" });
   });
 
-  it("returns false and keeps the entry when the stop itself fails", async () => {
+  it("rejects and keeps the entry when the stop itself fails", async () => {
     const registry = new SessionRegistry(5);
     own(registry, "s1");
     const runner = fakeRunner({ stopFails: true });
     const { service } = setup({ registry, runner });
     service.addSession("s1");
-    await expect(service.stopSession("s1")).resolves.toBe(false);
+    await expect(service.stopSession("s1")).rejects.toThrow(/boom/);
     expect(registry.isOwned("s1")).toBe(true);
     expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
   });
@@ -590,7 +641,7 @@ describe("HTTP/SSE interface", () => {
   });
 });
 
-describe("availability and dead sessions", () => {
+describe("availability and missing sessions", () => {
   it("flips availability off after repeated global failures and back on success", async () => {
     const scheduler = fakeScheduler();
     const runner = fakeRunner({ screenshotFails: true });
@@ -610,23 +661,18 @@ describe("availability and dead sessions", () => {
     service.dispose();
   });
 
-  it("marks a session dead on session_not_found and stops asking for frames", async () => {
+  it("removes an owned ghost on not_found and stops asking for frames", async () => {
     const scheduler = fakeScheduler();
     const runner = fakeRunner({ screenshotNotFound: true });
-    const { service, events } = setup({ runner, scheduler });
+    const registry = new SessionRegistry(5);
+    own(registry, "s1");
+    const { service, events } = setup({ runner, scheduler, registry });
     service.addSession("s1");
     scheduler.runNext();
-    await waitFor(() => events.some((e) => e.type === "upsert" && e.session?.dead === true));
+    await waitFor(() => events.some((e) => e.type === "remove"));
-    expect(service.getState()[0].dead).toBe(true);
-    expect(service.getState()[0].action).toBe("idle");
-    // No further captures are scheduled for a dead session.
-    expect(scheduler.pending()).toEqual([]);
-    // Instrumentation is dropped for dead sessions.
-    service.beginAction("s1", "clicking");
-    expect(service.getState()[0].action).toBe("idle");
-    // A dead session leaves cleanly.
-    service.removeSession("s1");
     expect(service.getState()).toEqual([]);
+    expect(registry.isOwned("s1")).toBe(false);
+    expect(scheduler.pending()).toEqual([]);
   });
 
   it("clears lastError on the next action and keeps it on failure", () => {

packages/dsh-plugin-browserskill/tests/runner.test.ts

@@ -1,7 +1,14 @@
 import type { ChildProcess } from "node:child_process";
 import { EventEmitter } from "node:events";
 import { describe, expect, it } from "vitest";
-import { type BskRunResult, createBskRunner, isCommandNotFound, parseBskJson } from "../src/runner";
+import {
+  type BskRunResult,
+  createBskRunner,
+  isCommandNotFound,
+  isSessionBusyResult,
+  parseBskJson,
+  runWithSessionBusyRetry,
+} from "../src/runner";
 
 /** Minimal fake ChildProcess driven by the test. */
 class FakeChild extends EventEmitter {
@@ -63,7 +70,7 @@ describe("createBskRunner", () => {
     controller.abort();
     const result = await promise;
     expect(result.aborted).toBe(true);
-    expect(child.killedWith).toContain("SIGTERM");
+    expect(child.killedWith).toContain("SIGINT");
   });
 
   it("kills the child on timeout", async () => {
@@ -80,7 +87,7 @@ describe("createBskRunner", () => {
     const promise = runner.run(["session", "list"]);
     runner.killAll();
     const result = await promise;
-    expect(child.killedWith).toContain("SIGTERM");
+    expect(child.killedWith).toContain("SIGINT");
     expect(result.code).toBeNull();
   });
 });
@@ -140,3 +147,51 @@ describe("isCommandNotFound", () => {
     expect(isCommandNotFound(new Error("other"))).toBe(false);
   });
 });
+
+describe("session busy reconciliation", () => {
+  const busy: BskRunResult = {
+    code: 4,
+    stdout: JSON.stringify({
+      code: "timeout",
+      message: "session already has an unfinished command",
+      data: { reason: "session_busy" },
+    }),
+    stderr: "",
+    timedOut: false,
+    aborted: false,
+  };
+  const ok: BskRunResult = {
+    code: 0,
+    stdout: "{}",
+    stderr: "",
+    timedOut: false,
+    aborted: false,
+  };
+
+  it("recognizes only the structured session_busy reason", () => {
+    expect(isSessionBusyResult(busy)).toBe(true);
+    expect(isSessionBusyResult({ ...busy, stdout: '{"code":"timeout"}' })).toBe(false);
+    expect(isSessionBusyResult(ok)).toBe(false);
+  });
+
+  it("retries one transient busy result and returns the settled response", async () => {
+    const replies = [busy, ok];
+    let calls = 0;
+    const result = await runWithSessionBusyRetry(async () => {
+      calls += 1;
+      return replies.shift() ?? ok;
+    });
+    expect(result).toBe(ok);
+    expect(calls).toBe(2);
+  });
+
+  it("does not loop on a persistent busy result", async () => {
+    let calls = 0;
+    const result = await runWithSessionBusyRetry(async () => {
+      calls += 1;
+      return busy;
+    });
+    expect(result).toBe(busy);
+    expect(calls).toBe(2);
+  });
+});

packages/dsh-plugin-browserskill/tests/skill.test.ts

@@ -1,8 +1,11 @@
 // Browser skill injection: registration wiring, catalog content, progressive-load
 // weight, and silent degradation without the skill seam.
 
+import { Context } from "@deepseek-ai/cordis";
+import { createScope, scopeTarget } from "@deepseek-ai/dsh-scope";
+import { SkillRegistry } from "@deepseek-ai/dsh-skill";
 import { describe, expect, it } from "vitest";
-import { registerBskSkill } from "../src/skill";
+import { armAgentScopedBskSkill, registerBskSkill } from "../src/skill";
 
 function fakeCtx(skills?: unknown) {
   return { get: (key: string) => (key === "skills" ? skills : undefined) } as never;
@@ -100,3 +103,88 @@ describe("registerBskSkill", () => {
     expect(() => disposer()).not.toThrow();
   });
 });
+
+describe("armAgentScopedBskSkill", () => {
+  it("overrides a nearer legacy CLI skill when the DSH agent starts", async () => {
+    const root = new Context();
+    const skillFiber = root.plugin(SkillRegistry);
+    await skillFiber;
+
+    // The plugin's original registration is global.
+    registerBskSkill(skillFiber.ctx);
+
+    // Model the standard preset's filesystem discovery of
+    // ~/.agents/skills/browser-skill. Its nearer scope wins before the fix.
+    const presetKey = { preset: "standard" };
+    const presetScope = createScope(skillFiber.ctx, presetKey);
+    presetScope.ctx.skills.register({
+      name: "browser-skill",
+      description: "Legacy BrowserSkill CLI instructions",
+      content: "Use bsk shell commands.",
+      source: "user-agents",
+    });
+
+    const agentKey = { agent: "test-agent" };
+    const agentScope = createScope(skillFiber.ctx, agentKey, { parent: presetKey });
+    const agent = {
+      id: "test-agent",
+      ctx: agentScope.ctx,
+    } as never;
+    const agents = { list: () => [] };
+    const pluginCtx = skillFiber.ctx.extend({ agents });
+
+    const before = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
+    expect(before?.content).toBe("Use bsk shell commands.");
+    expect(before?.source).toBe("user-agents");
+
+    const disarm = armAgentScopedBskSkill(pluginCtx);
+    pluginCtx.emit(scopeTarget(agent, agentKey), "agent/session-start", {
+      agent,
+      source: "startup",
+    });
+
+    const after = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
+    expect(after?.content).toContain("All browser operations must use the injected tools directly");
+    expect(after?.content).not.toMatch(/\bbsk\b/i);
+    expect(after?.source).toBe("bundled");
+
+    disarm();
+    await agentScope.dispose();
+    await presetScope.dispose();
+    await skillFiber.dispose();
+  });
+
+  it("installs on existing agents for plugin reload and disposes registrations", () => {
+    const registered: Record<string, unknown>[] = [];
+    let disposed = 0;
+    const agent = {
+      ctx: fakeCtx({
+        register(skill: Record<string, unknown>) {
+          registered.push(skill);
+          return () => {
+            disposed += 1;
+          };
+        },
+      }),
+    } as never;
+    const listeners = new Map<string, (payload: { agent: never }) => void>();
+    const ctx = {
+      get: (key: string) => (key === "agents" ? { list: () => [agent] } : undefined),
+      on(event: string, listener: (payload: { agent: never }) => void) {
+        listeners.set(event, listener);
+        return () => listeners.delete(event);
+      },
+    } as never;
+
+    const disarm = armAgentScopedBskSkill(ctx);
+    expect(registered).toHaveLength(1);
+
+    // A later lifecycle notification for the same agent must not duplicate it.
+    listeners.get("agent/session-start")?.({ agent });
+    expect(registered).toHaveLength(1);
+
+    disarm();
+    expect(disposed).toBe(1);
+    expect(listeners.size).toBe(0);
+  });
+});

packages/vom/src/__tests__/render.test.ts

@@ -29,6 +29,66 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
 }
 
 describe("renderVom single-layer page", () => {
+  it("renders visual surfaces as screenshot-only refs", () => {
+    const input = scene([
+      node({ id: 1, role: "RootWebArea", frameId: "main" }),
+      node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
+    ]);
+    input.visualSurfaces = [
+      {
+        parentId: 2,
+        backendNodeId: 99,
+        frameId: "child",
+        renderingKind: "canvas",
+        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+        label: "Data grid",
+        memberCount: 2,
+      },
+    ];
+    const out = renderVom(input);
+
+    expect(out.text).toContain(
+      '@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
+    );
+    expect(out.refs).toContainEqual(
+      expect.objectContaining({
+        ref: "e1",
+        backendNodeId: 99,
+        kind: "surface",
+        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+        capabilities: ["screenshot"],
+      }),
+    );
+  });
+
+  it("renders an unnamed visual surface with a stable generic label", () => {
+    const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
+    input.visualSurfaces = [
+      {
+        parentId: 1,
+        backendNodeId: 99,
+        frameId: "main",
+        renderingKind: "canvas",
+        visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+        memberCount: 1,
+      },
+    ];
+
+    const out = renderVom(input);
+
+    expect(out.text).toContain(
+      '@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
+    );
+    expect(out.refs).toContainEqual(
+      expect.objectContaining({
+        ref: "e1",
+        name: "canvas visual surface",
+        kind: "surface",
+        capabilities: ["screenshot"],
+      }),
+    );
+  });
+
   it("does not derive handle context across frame scopes", () => {
     const out = renderVom(
       scene([
@@ -333,6 +393,84 @@ describe("renderVom single-layer page", () => {
     expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
+  it("does not treat implementation CSS classes as control names", () => {
+    const out = renderVom(
+      scene([
+        node({ id: 1, role: "RootWebArea", name: "Doc" }),
+        node({
+          id: 2,
+          parentId: 1,
+          role: "generic",
+          cursor: "pointer",
+          attrs: { onclick: "handleClick()" },
+          rect: { x: 20, y: 20, w: 40, h: 40 },
+        }),
+        node({
+          id: 3,
+          parentId: 2,
+          role: "img",
+          tag: "svg",
+          attrs: { class: "kocomz" },
+        }),
+      ]),
+    );
+
+    expect(out.text).not.toContain("kocomz");
+    expect(out.refs).toEqual([]);
+  });
+
+  it("recovers icon controls only from explicit semantic icon evidence", () => {
+    const out = renderVom(
+      scene([
+        node({ id: 1, role: "RootWebArea", name: "Doc" }),
+        node({
+          id: 2,
+          parentId: 1,
+          role: "generic",
+          cursor: "pointer",
+          attrs: { onclick: "handleFilter()" },
+          rect: { x: 20, y: 20, w: 40, h: 40 },
+        }),
+        node({
+          id: 3,
+          parentId: 2,
+          role: "img",
+          tag: "svg",
+          attrs: { "aria-label": "Filter rows", class: "kocomz" },
+        }),
+      ]),
+    );
+
+    expect(out.text).toContain('@e1 button "Filter rows"');
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+  });
+
+  it("keeps a semantic image as evidence for its clickable wrapper", () => {
+    const out = renderVom(
+      scene([
+        node({ id: 1, role: "RootWebArea", name: "Doc" }),
+        node({
+          id: 2,
+          parentId: 1,
+          role: "generic",
+          cursor: "pointer",
+          rect: { x: 20, y: 20, w: 40, h: 40 },
+        }),
+        node({
+          id: 3,
+          parentId: 2,
+          role: "img",
+          name: "Account",
+          cursor: "pointer",
+          tag: "div",
+        }),
+      ]),
+    );
+
+    expect(out.text).toContain('@e1 button "Account"');
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+  });
+
   it("keeps the deepest custom control when wrapper and child both look clickable", () => {
     const out = renderVom(
       scene([

packages/vom/src/index.ts

@@ -1,4 +1,9 @@
-export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render";
+export {
+  applyVomInteractionRecovery,
+  isVomReferenceNode,
+  isVomStructuralRole,
+  renderVom,
+} from "./render";
 export type {
   ActiveScopeBlock,
   BlockingLayer,
@@ -10,6 +15,8 @@ export type {
   VomNode,
   VomOptions,
   VomRef,
+  VomRefCapability,
   VomResult,
   VomScene,
+  VomVisualSurface,
 } from "./types";

packages/vom/src/render.ts

@@ -9,6 +9,7 @@ import type {
   VomOptions,
   VomResult,
   VomScene,
+  VomVisualSurface,
 } from "./types";
 
 const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -75,6 +76,10 @@ const STRUCTURAL_ROLES = new Set([
   "webarea",
 ]);
 
+export function isVomStructuralRole(role: string | undefined): boolean {
+  return STRUCTURAL_ROLES.has(role?.toLowerCase() ?? "");
+}
+
 const SENSITIVE_MASK = "•••";
 const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY;
 const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY;
@@ -193,13 +198,16 @@ function shouldRender(node: VomNode): boolean {
   const role = normalizedRole(node);
   if (SKIP_ROLES.has(role)) return false;
   if (INTERACTIVE_ROLES.has(role)) return true;
-  if (STRUCTURAL_ROLES.has(role)) return true;
+  if (isVomStructuralRole(role)) return true;
 
   return cleaned(node.name) !== undefined;
 }
 
 export function isVomReferenceNode(node: VomNode): boolean {
-  return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
+  return (
+    node.referenceable !== false &&
+    (INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node))
+  );
 }
 
 function hasRecoveryHandler(node: VomNode): boolean {
@@ -219,14 +227,30 @@ function isContentEditable(node: VomNode): boolean {
   return contentEditable === "true" || contentEditable === "plaintext-only";
 }
 
-function iconKeyword(value: string | undefined): string | undefined {
+const ICON_REFERENCE_MARKERS = new Set(["glyph", "icon", "icons", "symbol"]);
-  const cleanedValue = cleaned(value);
+const ICON_REFERENCE_DECORATORS = new Set(["fill", "filled", "line", "outline", "outlined"]);
-  if (!cleanedValue) return undefined;
+
-  const last = cleanedValue
+function iconReferenceName(value: string | undefined): string | undefined {
-    .split(/[#/.\s:_-]+/)
+  const reference = cleaned(value);
+  if (!reference) return undefined;
+  const fragment = reference.includes("#") ? reference.slice(reference.lastIndexOf("#") + 1) : "";
+  const tokens = (fragment || reference)
+    ?.split(/[#/.\s:_-]+/)
     .filter(Boolean)
-    .pop();
+    .map((token) => token.toLowerCase());
-  return last && /^[a-z][a-z0-9_-]{1,32}$/i.test(last) ? last.toLowerCase() : undefined;
+  if (tokens.length === 0) return undefined;
+  if (fragment) {
+    const semanticTokens = tokens.filter(
+      (token) => !ICON_REFERENCE_MARKERS.has(token) && !ICON_REFERENCE_DECORATORS.has(token),
+    );
+    return semanticTokens.length > 0 ? semanticTokens.join(" ") : undefined;
+  }
+  const marker = tokens.findLastIndex((token) => ICON_REFERENCE_MARKERS.has(token));
+  if (marker < 0) return undefined;
+  const semanticTokens = tokens
+    .slice(marker + 1)
+    .filter((token) => !ICON_REFERENCE_DECORATORS.has(token));
+  return semanticTokens.length > 0 ? semanticTokens.join(" ") : undefined;
 }
 
 function iconHintFromSubtree(
@@ -238,12 +262,12 @@ function iconHintFromSubtree(
   for (const child of children.get(node.id) ?? []) {
     const attrs = child.attrs ?? {};
     const hint =
-      iconKeyword(attrs["aria-label"]) ??
+      cleaned(attrs["aria-label"]) ??
-      iconKeyword(attrs.title) ??
+      cleaned(attrs.title) ??
-      iconKeyword(attrs.href) ??
+      (normalizedRole(child) === "img" ? cleaned(child.name) : undefined) ??
-      iconKeyword(attrs["xlink:href"]) ??
+      (["img", "svg", "use"].includes(normalizedTag(child))
-      iconKeyword(attrs.class) ??
+        ? (iconReferenceName(attrs.href) ?? iconReferenceName(attrs["xlink:href"]))
-      iconKeyword(child.text);
+        : undefined);
     if (hint) return hint;
     const nested = iconHintFromSubtree(child, children, depth + 1);
     if (nested) return nested;
@@ -269,6 +293,7 @@ function recoveredRole(
   node: VomNode,
   children?: Map<number | null, VomNode[]>,
 ): string | undefined {
+  const currentRole = normalizedRole(node);
   const explicitRole = node.attrs?.role?.toLowerCase() ?? "";
   const hasHandler = hasRecoveryHandler(node);
   const focusable = isFocusable(node);
@@ -290,7 +315,9 @@ function recoveredRole(
       : "button";
   }
   if (hasHandler && hasName) return "button";
-  if (node.cursor === "pointer" && hasName) return "button";
+  if ((!currentRole || SKIP_ROLES.has(currentRole)) && node.cursor === "pointer" && hasName) {
+    return "button";
+  }
   return undefined;
 }
 
@@ -711,6 +738,53 @@ interface RenderState {
   duplicateRefNames: Set<string>;
   surfaceMap: Map<number, CondSurface>;
   scopeMap: Map<number, ActiveScopeBlock>;
+  visualSurfaces: Map<number | null, VomVisualSurface[]>;
+}
+
+function groupedByParent<T extends { parentId: number | null }>(
+  items: T[],
+): Map<number | null, T[]> {
+  const grouped = new Map<number | null, T[]>();
+  for (const item of items) {
+    const siblings = grouped.get(item.parentId) ?? [];
+    siblings.push(item);
+    grouped.set(item.parentId, siblings);
+  }
+  return grouped;
+}
+
+function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
+  if (state.stopped) return;
+  for (const surface of state.visualSurfaces.get(parentId) ?? []) {
+    if (depth > state.maxDepth) {
+      state.truncated = true;
+      continue;
+    }
+    const ref = `e${state.nextRef}`;
+    const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
+    const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
+    const line = `${"  ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
+    const nextTokens = state.tokens + estimateTokens(line);
+    if (nextTokens > state.maxTokens) {
+      state.truncated = true;
+      state.stopped = true;
+      return;
+    }
+    state.lines.push(line);
+    state.tokens = nextTokens;
+    state.refs.push({
+      ref,
+      backendNodeId: surface.backendNodeId,
+      frameId: surface.frameId,
+      visibleRect: surface.visibleRect,
+      role: "surface",
+      name: label,
+      kind: "surface",
+      capabilities: ["screenshot"],
+      line: state.lines.length - 1,
+    });
+    state.nextRef += 1;
+  }
 }
 
 function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
@@ -786,6 +860,8 @@ function renderTree(
         ...(node.role ? { role: node.role } : {}),
         ...(name ? { name } : {}),
         ...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
+        kind: "dom",
+        capabilities: ["interact", "screenshot"],
         line: state.lines.length - 1,
       });
       state.nextRef += 1;
@@ -795,10 +871,12 @@ function renderTree(
     if (scope) emitActiveScopeBlock(scope, depth + 1, state);
 
     if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
+      emitVisualEntries(node.id, depth + 1, state);
       continue;
     }
     renderTree(children, node.id, depth + 1, state);
   }
+  emitVisualEntries(parentId, depth, state);
 }
 
 function renderNodes(
@@ -807,6 +885,7 @@ function renderNodes(
   initialLines: string[],
   surfaces: CondSurface[] = [],
   activeScopeBlocks: ActiveScopeBlock[] = [],
+  visualSurfaces: VomVisualSurface[] = [],
 ): RenderState {
   const children = buildChildren(nodes);
   const state: RenderState = {
@@ -825,6 +904,7 @@ function renderNodes(
     duplicateRefNames: duplicateReferenceNames(nodes),
     surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
     scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
+    visualSurfaces: groupedByParent(visualSurfaces),
   };
 
   renderTree(children, null, 1, state);
@@ -1030,7 +1110,16 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
     "@layers 2 focus=L1",
     `L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
   ];
-  const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
+  const visibleSurface = (parentId: number | null): boolean =>
+    parentId === null || included.has(parentId);
+  const state = renderNodes(
+    visibleNodes,
+    options,
+    header,
+    scene.surfaces,
+    scene.activeScopeBlocks,
+    scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
+  );
   state.lines.push(renderPageOcclusionLine(hiddenCount));
 
   return {
@@ -1058,6 +1147,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
     ],
     scene.surfaces,
     scene.activeScopeBlocks,
+    scene.visualSurfaces,
   );
 
   return {

packages/vom/src/types.ts

@@ -24,6 +24,8 @@ export interface VomNode {
   backendNodeId?: number;
   frameId?: string;
   contextScopeId?: string;
+  /** False for semantic-only nodes that cannot be addressed through CDP. */
+  referenceable?: boolean;
 
   role?: string;
   name?: string;
@@ -59,6 +61,8 @@ export interface VomScene {
   rootFrameId?: string;
   surfaces?: CondSurface[];
   activeScopeBlocks?: ActiveScopeBlock[];
+  /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
+  visualSurfaces?: VomVisualSurface[];
 }
 
 export interface VomOptions {
@@ -92,6 +96,22 @@ export interface VomRef {
   ref: string;
   backendNodeId: number;
   frameId?: string;
+  /** Observation-time top-viewport crop for screenshot-only surfaces. */
+  visibleRect?: Rect;
+  kind?: "dom" | "surface";
+  capabilities?: VomRefCapability[];
+}
+
+export type VomRefCapability = "interact" | "screenshot";
+
+export interface VomVisualSurface {
+  parentId: number | null;
+  backendNodeId: number;
+  frameId: string;
+  renderingKind: "canvas";
+  visibleRect: Rect;
+  label?: string;
+  memberCount: number;
 }
 
 export interface RenderedRef extends VomRef {

pnpm-lock.yaml

@@ -114,6 +114,12 @@ importers:
       '@deepseek-ai/dsh-llm':
         specifier: ^0.1.0-rc.6
         version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
+      '@deepseek-ai/dsh-scope':
+        specifier: ^0.1.0-rc.6
+        version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
+      '@deepseek-ai/dsh-skill':
+        specifier: ^0.1.0-rc.6
+        version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-llm@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))))(@deepseek-ai/dsh-scope@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
       '@deepseek-ai/dsh-tools':
         specifier: ^0.1.0-rc.6
         version: 0.1.0-rc.6(f8724372086ccc1457fc84e7becee2e0)

skill/SKILL.md

@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
 
 When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
 
+When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
+
 ## Observation priority
 
 Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: