feat(vom): method update, solve context over exposed problem

apps/extension/src/tools/__tests__/observation.test.ts

@@ -570,6 +570,82 @@ describe("buildVomScene", () => {
     ]);
   });
 
+  it("dedupes iframe fallback controls already exposed through AX", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "Iframe" },
+        backendDOMNodeId: 20,
+        childIds: ["3"],
+      },
+      {
+        nodeId: "3",
+        parentId: "2",
+        role: { type: "role", value: "textbox" },
+        name: { type: "x", value: "输入密码" },
+        backendDOMNodeId: 202,
+        properties: [{ name: "inputType", value: { value: "password" } }],
+      },
+    ];
+    const captured: CapturedViewModel = {
+      viewport: { width: 1000, height: 800 },
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "iframe",
+          attrs: {},
+          rect: { x: 100, y: 100, w: 400, h: 300 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+      iframeNodes: new Map([
+        [
+          20,
+          [
+            {
+              backendNodeId: 102,
+              parentBackendNodeId: null,
+              tag: "input",
+              attrs: { type: "password", placeholder: "输入密码" },
+              rect: { x: 0, y: 50, w: 300, h: 40 },
+              paintOrder: 1,
+              position: "static",
+              pointerEvents: "auto",
+            },
+          ],
+        ],
+      ]),
+      excludedBackendNodeIds: new Set(),
+    };
+
+    const passwordNodes = buildVomScene(axNodes, captured).nodes.filter(
+      (node) => node.role === "textbox" && node.name === "输入密码",
+    );
+
+    expect(passwordNodes).toHaveLength(1);
+    expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
+  });
+
   it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
     const scene = buildVomScene(
       [
@@ -843,6 +919,66 @@ describe("buildVomScene", () => {
     );
   });
 
+  it("uses nearby preceding text as a form control label", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 1,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "textbox" },
+        backendDOMNodeId: 20,
+      },
+    ];
+    const captured: CapturedViewModel = {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      nodes: [
+        {
+          backendNodeId: 1,
+          parentBackendNodeId: null,
+          tag: "div",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 500, h: 80 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: 1,
+          tag: "span",
+          attrs: {},
+          textContent: "验证码:",
+          rect: { x: 0, y: 0, w: 80, h: 30 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 1,
+          tag: "input",
+          attrs: { type: "text" },
+          rect: { x: 90, y: 0, w: 120, h: 30 },
+          paintOrder: 2,
+          position: "static",
+          pointerEvents: "auto",
+          cursor: "text",
+        },
+      ],
+    };
+
+    expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
+      expect.objectContaining({ role: "textbox", name: "验证码" }),
+    );
+  });
+
   it("promotes a cursor:pointer icon control to a button named from its <use> sprite", () => {
     // A close button built as <div class=close-button><svg><use href="#close"/></svg></div>
     // — the AX tree exposes it as `generic` with no name, so it would be dropped.
@@ -1065,6 +1201,284 @@ describe("buildVomScene", () => {
     expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
   });
 
+  it("builds active scope blocks from active aria-controls relationships", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "tab" },
+        name: { type: "computedString", value: "Reviews (12)" },
+        backendDOMNodeId: 20,
+      },
+    ];
+    const captured: CapturedViewModel = {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "button",
+          attrs: { role: "tab", "aria-selected": "true", "aria-controls": "reviews-panel" },
+          rect: { x: 20, y: 20, w: 120, h: 40 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 30,
+          parentBackendNodeId: 10,
+          tag: "div",
+          attrs: { id: "reviews-panel" },
+          rect: null,
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+          textContent: "Jane - ear cups are small",
+        },
+        {
+          backendNodeId: 31,
+          parentBackendNodeId: 30,
+          tag: "p",
+          attrs: {},
+          rect: null,
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+          textContent: "Bob - great sound",
+        },
+      ],
+    };
+
+    const scene = buildVomScene(axNodes, captured);
+    expect(scene.activeScopeBlocks).toEqual([
+      {
+        triggerId: 20,
+        label: "Reviews (12)",
+        lines: ["Jane - ear cups are small", "Bob - great sound"],
+      },
+    ]);
+    expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
+  });
+
+  it("maps hover surface probes to matching node labels", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "button" },
+        name: { type: "computedString", value: "Products" },
+        backendDOMNodeId: 20,
+      },
+    ];
+    const scene = buildVomScene(axNodes, {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      surfaceProbes: [
+        { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
+      ],
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "button",
+          attrs: {},
+          rect: { x: 20, y: 20, w: 120, h: 40 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    });
+
+    expect(scene.surfaces).toEqual([
+      { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
+    ]);
+    expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
+  });
+
+  it("enriches names and active scope signals from AX properties", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 10,
+        childIds: ["2", "3"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "button" },
+        name: { type: "computedString", value: "Products" },
+        backendDOMNodeId: 20,
+        properties: [
+          { name: "hasPopup", value: { value: "menu" } },
+          { name: "expanded", value: { value: "true" } },
+          { name: "controls", value: { value: "products-panel" } },
+        ],
+      },
+      {
+        nodeId: "3",
+        parentId: "1",
+        role: { type: "role", value: "textbox" },
+        name: { type: "computedString", value: "Card number" },
+        backendDOMNodeId: 30,
+        properties: [{ name: "inputType", value: { value: "credit-card" } }],
+      },
+    ];
+    const scene = buildVomScene(axNodes, {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      nodes: [
+        {
+          backendNodeId: 10,
+          parentBackendNodeId: null,
+          tag: "body",
+          attrs: {},
+          rect: { x: 0, y: 0, w: 1000, h: 800 },
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: 10,
+          tag: "button",
+          attrs: {},
+          rect: { x: 20, y: 20, w: 120, h: 40 },
+          paintOrder: 1,
+          position: "static",
+          pointerEvents: "auto",
+        },
+        {
+          backendNodeId: 25,
+          parentBackendNodeId: 10,
+          tag: "div",
+          attrs: { id: "products-panel" },
+          rect: null,
+          paintOrder: 0,
+          position: "static",
+          pointerEvents: "auto",
+          textContent: "Shoes Bags",
+        },
+        {
+          backendNodeId: 30,
+          parentBackendNodeId: 10,
+          tag: "input",
+          attrs: {},
+          rect: { x: 20, y: 80, w: 200, h: 40 },
+          paintOrder: 2,
+          position: "static",
+          pointerEvents: "auto",
+        },
+      ],
+    });
+
+    expect(scene.nodes.find((node) => node.id === 20)).toEqual(
+      expect.objectContaining({
+        name: "Products [expanded]",
+        attrs: expect.objectContaining({
+          "aria-expanded": "true",
+          "aria-controls": "products-panel",
+        }),
+      }),
+    );
+    expect(scene.activeScopeBlocks).toEqual([
+      { triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] },
+    ]);
+    expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true);
+  });
+
+  it("aggregates AX virtual text into unnamed structural nodes", () => {
+    const scene = buildVomScene(
+      [
+        {
+          nodeId: "1",
+          role: { type: "role", value: "RootWebArea" },
+          backendDOMNodeId: 10,
+          childIds: ["2"],
+        },
+        {
+          nodeId: "2",
+          parentId: "1",
+          role: { type: "role", value: "paragraph" },
+          backendDOMNodeId: 20,
+          childIds: ["3"],
+        },
+        {
+          nodeId: "3",
+          parentId: "2",
+          role: { type: "role", value: "InlineTextBox" },
+          name: { type: "computedString", value: "Inline only text" },
+        },
+      ],
+      {
+        viewport: { width: 1000, height: 800 },
+        iframeNodes: new Map(),
+        excludedBackendNodeIds: new Set(),
+        nodes: [
+          {
+            backendNodeId: 10,
+            parentBackendNodeId: null,
+            tag: "body",
+            attrs: {},
+            rect: { x: 0, y: 0, w: 1000, h: 800 },
+            paintOrder: 0,
+            position: "static",
+            pointerEvents: "auto",
+          },
+          {
+            backendNodeId: 20,
+            parentBackendNodeId: 10,
+            tag: "p",
+            attrs: {},
+            rect: { x: 20, y: 20, w: 400, h: 40 },
+            paintOrder: 1,
+            position: "static",
+            pointerEvents: "auto",
+          },
+        ],
+      },
+    );
+
+    expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text");
+  });
+
   it("marks captured dialog elements as modal without AX role or aria-modal", () => {
     const scene = buildVomScene([], {
       viewport: { width: 640, height: 480 },
@@ -1533,7 +1947,8 @@ describe("handleSnapshot", () => {
     expect(res.text).toContain("L1 modal cover=100%");
     // Iframe content must appear in L1 — both inputs rendered
     expect(res.text).toMatch(/textbox "请输入手机号"/);
-    expect(res.text).toMatch(/textbox "密码" ="•••"/);
+    expect(res.text).toMatch(/textbox "密码"/);
+    expect(res.text).not.toMatch(/textbox "密码" ="•••"/);
     // Ref-store must include the iframe's input backendNodeIds
     const ctx = sm["sessions"].get("aa11")!;
     const phoneRef = res.text.match(/@(e\d+) textbox "请输入手机号"/)?.[1];

apps/extension/src/tools/observation.ts

@@ -3,7 +3,13 @@
 // tab (defaulting to the Agent Window's active tab when omitted) and
 // returns a payload that mirrors the bsk-protocol Rust structs.
 
-import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom";
+import {
+  type ActiveScopeBlock,
+  type CondSurface,
+  renderVom,
+  type VomNode,
+  type VomScene,
+} from "@browser-skill/vom";
 import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
 import type { SessionManager } from "@/session-manager/manager";
 import type {
@@ -234,6 +240,7 @@ export interface CdpAxNode {
   name?: { type: string; value?: string };
   description?: { value?: string };
   value?: { value?: string | number | boolean };
+  properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
   childIds?: string[];
 }
 
@@ -268,6 +275,120 @@ function isSensitive(capturedNode: CapturedNode | undefined): boolean {
   return (attrs.type ?? "").toLowerCase() === "password";
 }
 
+interface AxNodeSignals {
+  hasPopup: boolean;
+  expanded: boolean;
+  selected: boolean;
+  controls: string;
+  sensitive: boolean;
+  aggregatedText?: string;
+}
+
+const AX_TEXT_AGGREGATE_ROLES = new Set([
+  "paragraph",
+  "listitem",
+  "term",
+  "definition",
+  "cell",
+  "gridcell",
+  "caption",
+  "figcaption",
+  "blockquote",
+  "note",
+  "status",
+  "log",
+  "generic",
+  "section",
+]);
+const AX_TEXT_LEAF_ROLES = new Set(["inlinetextbox", "statictext", "text"]);
+const AX_TEXT_STOP_ROLES = new Set([
+  "button",
+  "link",
+  "combobox",
+  "listbox",
+  "menuitem",
+  "menuitemcheckbox",
+  "menuitemradio",
+  "option",
+  "radio",
+  "checkbox",
+  "textbox",
+  "searchbox",
+  "spinbutton",
+  "slider",
+  "switch",
+  "tab",
+  "treeitem",
+  "columnheader",
+  "rowheader",
+]);
+const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
+
+function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
+  const prop = axNode.properties?.find((item) => item.name === name);
+  return axString(prop?.value);
+}
+
+function buildAxSignals(axNodes: CdpAxNode[]): Map<string, AxNodeSignals> {
+  const axByNodeId = new Map(axNodes.map((node) => [node.nodeId, node]));
+  const virtualText = new Map<string, string>();
+  for (const node of axNodes) {
+    if (typeof node.backendDOMNodeId === "number") continue;
+    const role = axString(node.role)?.toLowerCase() ?? "";
+    const name = axString(node.name);
+    if (name && AX_TEXT_LEAF_ROLES.has(role)) virtualText.set(node.nodeId, name);
+  }
+
+  const collectLeafText = (nodeId: string, depth: number): string[] => {
+    if (depth > 8) return [];
+    const node = axByNodeId.get(nodeId);
+    if (!node) return [];
+    const parts: string[] = [];
+    for (const childId of node.childIds ?? []) {
+      const vtext = virtualText.get(childId);
+      if (vtext) {
+        parts.push(vtext);
+        continue;
+      }
+      const child = axByNodeId.get(childId);
+      if (!child) continue;
+      const childRole = axString(child.role)?.toLowerCase() ?? "";
+      if (AX_TEXT_LEAF_ROLES.has(childRole)) continue;
+      if (AX_TEXT_STOP_ROLES.has(childRole)) continue;
+      const childName = axString(child.name);
+      if (childName && !AX_TEXT_AGGREGATE_ROLES.has(childRole)) {
+        parts.push(childName);
+      } else {
+        parts.push(...collectLeafText(childId, depth + 1));
+      }
+    }
+    return parts;
+  };
+
+  const signals = new Map<string, AxNodeSignals>();
+  for (const node of axNodes) {
+    const role = axString(node.role)?.toLowerCase() ?? "";
+    const expanded = axPropertyString(node, "expanded") === "true";
+    const selected = axPropertyString(node, "selected") === "true";
+    const controls = axPropertyString(node, "controls") ?? "";
+    const hasPopupValue = axPropertyString(node, "hasPopup") ?? "";
+    const inputType = axPropertyString(node, "inputType") ?? "";
+    let aggregatedText: string | undefined;
+    if (AX_TEXT_AGGREGATE_ROLES.has(role) && !axString(node.name)) {
+      aggregatedText = cleanAttr(collectLeafText(node.nodeId, 0).join(" "));
+    }
+    signals.set(node.nodeId, {
+      hasPopup: hasPopupValue !== "" && hasPopupValue !== "false",
+      expanded,
+      selected,
+      controls,
+      sensitive: SENSITIVE_AX_INPUT_TYPES.has(inputType),
+      aggregatedText,
+    });
+  }
+  return signals;
+}
+
 function findAxAncestor<T>(
   axNode: CdpAxNode,
   axById: Map<string, CdpAxNode>,
@@ -331,6 +452,40 @@ function isRenderableIframeControl(node: CapturedNode): boolean {
   return tag !== "a" || name !== undefined;
 }
 
+function normalizedControlType(node: VomNode | CapturedNode): string {
+  const attrs = node.attrs ?? {};
+  const tag = normalizeTag(node.tag);
+  if (tag === "input") return (attrs.type ?? "text").toLowerCase();
+  if ("sensitive" in node && node.sensitive) return "password";
+  return tag;
+}
+
+function sameLogicalIframeControl(existing: VomNode, candidate: CapturedNode, iframeId: number) {
+  const role = iframeRoleFor(candidate);
+  const name = cleanAttr(iframeNameFor(candidate));
+  if (!role || !name) return false;
+  if ((existing.role ?? "").toLowerCase() !== role) return false;
+  if (cleanAttr(existing.name) !== name) return false;
+
+  const candidateType = normalizedControlType(candidate);
+  const existingType = normalizedControlType(existing);
+  if (candidateType !== existingType) return false;
+
+  return (
+    existing.parentId === iframeId ||
+    existing.domParentId === iframeId ||
+    existing.domAncestorIds?.includes(iframeId) === true
+  );
+}
+
+function hasEquivalentIframeControl(
+  nodes: VomNode[],
+  candidate: CapturedNode,
+  iframeId: number,
+): boolean {
+  return nodes.some((node) => sameLogicalIframeControl(node, candidate, iframeId));
+}
+
 function capturedIframeNameFor(node: CapturedNode): string | undefined {
   const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
   if (ariaLabel) return ariaLabel;
@@ -350,6 +505,11 @@ function cleanAttr(value: string | undefined): string | undefined {
 const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]);
 
 const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
+const ACTIVE_SCOPE_MAX_BLOCKS = 8;
+const ACTIVE_SCOPE_MAX_LINES = 40;
+const ACTIVE_SCOPE_MAX_LINE_LENGTH = 160;
+const ACTIVE_SCOPE_MAX_TOTAL_CHARS = 8_000;
+const ACTIVE_SCOPE_SKIP_TAGS = new Set(["script", "style", "noscript", "template"]);
 
 function buildCapturedChildren(capturedNodes: CapturedNode[]): Map<number, CapturedNode[]> {
   const children = new Map<number, CapturedNode[]>();
@@ -427,6 +587,24 @@ function nearbyTextFor(
   return labels.length > 0 ? labels.join(" ") : undefined;
 }
 
+function previousSiblingTextFor(
+  node: CapturedNode,
+  childrenByParentId: Map<number, CapturedNode[]>,
+): string | undefined {
+  if (node.parentBackendNodeId === null) return undefined;
+  const siblings = childrenByParentId.get(node.parentBackendNodeId) ?? [];
+  const index = siblings.findIndex((sibling) => sibling.backendNodeId === node.backendNodeId);
+  if (index <= 0) return undefined;
+
+  for (let i = index - 1; i >= Math.max(0, index - 4); i -= 1) {
+    const sibling = siblings[i];
+    if (["input", "textarea", "select", "button", "a"].includes(normalizeTag(sibling.tag))) break;
+    const text = cleanAttr(sibling.textContent)?.replace(/[：:]\s*$/, "");
+    if (text && text.length <= 40) return text;
+  }
+  return undefined;
+}
+
 interface VomNodeDomSignals {
   capturedByBackendId: Map<number, CapturedNode>;
   childrenByParentId: Map<number, CapturedNode[]>;
@@ -470,13 +648,18 @@ function formControlName(
   captured: CapturedNode | undefined,
   axName: string | undefined,
   axValue: string | undefined,
+  signals: VomNodeDomSignals,
 ): string | undefined {
   const placeholder = cleanAttr(captured?.attrs.placeholder);
   const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
   const title = cleanAttr(captured?.attrs.title);
+  const nearbyLabel = captured
+    ? previousSiblingTextFor(captured, signals.childrenByParentId)
+    : undefined;
   const hasValue = axValue !== undefined && axValue !== "";
+  if (nearbyLabel) return nearbyLabel;
   if (!hasValue && placeholder) return placeholder;
-  return axName ?? ariaLabel ?? placeholder ?? title;
+  return axName ?? ariaLabel ?? placeholder ?? title ?? nearbyLabel;
 }
 
 function vomNodeFromCaptured(
@@ -516,12 +699,17 @@ function axVomNode(
   capturedNode: CapturedNode | undefined,
   axById: Map<string, CdpAxNode>,
   signals: VomNodeDomSignals,
+  axSignals: Map<string, AxNodeSignals>,
 ): VomNode {
   const role = axString(axNode.role);
   const value = axValue(axNode.value);
-  const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
-    ? formControlName(capturedNode, axString(axNode.name), value)
-    : axString(axNode.name);
+  const signalsForNode = axSignals.get(axNode.nodeId);
+  let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
+    ? formControlName(capturedNode, axString(axNode.name), value, signals)
+    : (axString(axNode.name) ?? signalsForNode?.aggregatedText);
+  if (name && signalsForNode?.hasPopup) {
+    name = signalsForNode.expanded ? `${name} [expanded]` : `${name} [has-submenu]`;
+  }
   const node: VomNode = {
     id: axNode.backendDOMNodeId as number,
     parentId: nearestBackendParent(axNode, axById),
@@ -531,12 +719,22 @@ function axVomNode(
     position: capturedNode?.position || "static",
     pointerEvents: capturedNode?.pointerEvents || "auto",
     modal: isModalSignal(axNode, capturedNode),
-    sensitive: isSensitive(capturedNode),
+    sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
   };
   if (role) node.role = role;
   if (name) node.name = name;
   if (value !== undefined) node.value = value;
-  return applyCapturedSignals(node, capturedNode, signals);
+  const enriched = applyCapturedSignals(node, capturedNode, signals);
+  if (signalsForNode?.selected && !enriched.attrs?.["aria-selected"]) {
+    enriched.attrs = { ...(enriched.attrs ?? {}), "aria-selected": "true" };
+  }
+  if (signalsForNode?.expanded && !enriched.attrs?.["aria-expanded"]) {
+    enriched.attrs = { ...(enriched.attrs ?? {}), "aria-expanded": "true" };
+  }
+  if (signalsForNode?.controls && !enriched.attrs?.["aria-controls"]) {
+    enriched.attrs = { ...(enriched.attrs ?? {}), "aria-controls": signalsForNode.controls };
+  }
+  return enriched;
 }
 
 function iframeSignals(iframeNodes: CapturedNode[]): VomNodeDomSignals {
@@ -561,6 +759,144 @@ function capturedOnlySignals(capturedNodes: CapturedNode[]): VomNodeDomSignals {
   };
 }
 
+function normalizeProbeKey(value: string | undefined): string {
+  return cleanAttr(value)?.toLowerCase() ?? "";
+}
+
+function controlledIds(attrs: Record<string, string>): string[] {
+  return (attrs["aria-controls"] ?? "").split(/\s+/).filter(Boolean);
+}
+
+function isActiveScopeTrigger(node: VomNode): boolean {
+  const attrs = node.attrs ?? {};
+  if (controlledIds(attrs).length === 0) return false;
+  const role = normalizeTag(attrs.role) || normalizedVomRole(node);
+  return (
+    (role === "tab" && (attrs["aria-selected"] ?? "").toLowerCase() === "true") ||
+    (attrs["aria-expanded"] ?? "").toLowerCase() === "true"
+  );
+}
+
+function normalizedVomRole(node: VomNode): string {
+  return node.role?.toLowerCase() ?? "";
+}
+
+function collectScopeLines(
+  root: CapturedNode,
+  triggerLabel: string,
+  childrenByParentId: Map<number, CapturedNode[]>,
+): string[] {
+  const lines: string[] = [];
+  const seen = new Set<string>();
+  const triggerKey = normalizeProbeKey(triggerLabel);
+  const stack = [root];
+  while (stack.length > 0 && lines.length < ACTIVE_SCOPE_MAX_LINES) {
+    const node = stack.shift() as CapturedNode;
+    const tag = normalizeTag(node.tag);
+    if (ACTIVE_SCOPE_SKIP_TAGS.has(tag)) continue;
+    if ((node.attrs.type ?? "").toLowerCase() === "password") continue;
+
+    const text = cleanAttr(node.textContent);
+    const key = normalizeProbeKey(text);
+    if (text && key !== triggerKey && !seen.has(key)) {
+      seen.add(key);
+      lines.push(
+        text.length > ACTIVE_SCOPE_MAX_LINE_LENGTH
+          ? text.slice(0, ACTIVE_SCOPE_MAX_LINE_LENGTH)
+          : text,
+      );
+    }
+    stack.push(...(childrenByParentId.get(node.backendNodeId) ?? []));
+  }
+  return lines;
+}
+
+function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): ActiveScopeBlock[] {
+  const panelByDomId = new Map<string, CapturedNode>();
+  for (const capturedNode of signals.capturedByBackendId.values()) {
+    const domId = capturedNode.attrs.id;
+    if (domId) panelByDomId.set(domId, capturedNode);
+  }
+
+  const blocks: ActiveScopeBlock[] = [];
+  let totalChars = 0;
+  for (const node of nodes) {
+    if (blocks.length >= ACTIVE_SCOPE_MAX_BLOCKS) break;
+    if (!isActiveScopeTrigger(node)) continue;
+    const label =
+      cleanAttr(node.name) ?? cleanAttr(node.text) ?? cleanAttr(node.attrs?.["aria-label"]);
+    if (!label) continue;
+
+    const lines: string[] = [];
+    for (const controlId of controlledIds(node.attrs ?? {})) {
+      const panel = panelByDomId.get(controlId);
+      if (!panel) continue;
+      lines.push(...collectScopeLines(panel, label, signals.childrenByParentId));
+      if (lines.length >= ACTIVE_SCOPE_MAX_LINES) break;
+    }
+
+    const uniqueLines: string[] = [];
+    const seen = new Set<string>();
+    for (const line of lines) {
+      const key = normalizeProbeKey(line);
+      if (!key || seen.has(key)) continue;
+      seen.add(key);
+      uniqueLines.push(line);
+      totalChars += line.length;
+      if (
+        uniqueLines.length >= ACTIVE_SCOPE_MAX_LINES ||
+        totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS
+      ) {
+        break;
+      }
+    }
+    if (uniqueLines.length === 0) continue;
+    blocks.push({ triggerId: node.id, label, lines: uniqueLines });
+    if (totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS) break;
+  }
+  return blocks;
+}
+
+function labelForSurfaceMatch(node: VomNode): string {
+  return (
+    cleanAttr(node.name) ??
+    cleanAttr(node.text) ??
+    cleanAttr(node.attrs?.["aria-label"]) ??
+    cleanAttr(node.attrs?.title) ??
+    ""
+  );
+}
+
+function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
+  const probes = captured.surfaceProbes ?? [];
+  if (probes.length === 0) return [];
+
+  const surfaces: CondSurface[] = [];
+  const nodesWithLabels = nodes
+    .map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
+    .filter((entry) => entry.key.length > 0);
+  const used = new Set<number>();
+  for (const probe of probes) {
+    const probeKey = normalizeProbeKey(probe.triggerLabel);
+    if (!probeKey || probe.subItems.length === 0) continue;
+    const match =
+      nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
+      nodesWithLabels.find(
+        (entry) =>
+          !used.has(entry.node.id) &&
+          (probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
+      );
+    if (!match) continue;
+    used.add(match.node.id);
+    surfaces.push({
+      triggerId: match.node.id,
+      triggerAction: probe.triggerAction,
+      subItems: probe.subItems,
+    });
+  }
+  return surfaces;
+}
+
 function axNodeInOverlaySubtree(
   axNode: CdpAxNode,
   axById: Map<string, CdpAxNode>,
@@ -585,6 +921,7 @@ function axNodeInOverlaySubtree(
 export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene {
   const signals = capturedOnlySignals(captured.nodes);
   const { capturedByBackendId } = signals;
+  const axSignals = buildAxSignals(axNodes);
 
   const axById = new Map<string, CdpAxNode>();
   for (const node of axNodes) {
@@ -601,7 +938,7 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
 
     const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId);
     const role = axString(axNode.role);
-    const vomNode = axVomNode(axNode, capturedNode, axById, signals);
+    const vomNode = axVomNode(axNode, capturedNode, axById, signals, axSignals);
 
     // For link nodes, attach external hostname so the renderer can annotate it.
     // Relative hrefs and same-origin hrefs are omitted to avoid noise.
@@ -633,7 +970,8 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
       if (
         seenBackendIds.has(iframeNode.backendNodeId) ||
         excludedBackendNodeIds.has(iframeNode.backendNodeId) ||
-        !isRenderableIframeControl(iframeNode)
+        !isRenderableIframeControl(iframeNode) ||
+        hasEquivalentIframeControl(nodes, iframeNode, iframeBackendId)
       ) {
         continue;
       }
@@ -651,7 +989,14 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
     }
   }
 
-  return { viewport: captured.viewport, nodes };
+  const activeScopeBlocks = buildActiveScopeBlocks(nodes, signals);
+  const surfaces = buildConditionalSurfaces(nodes, captured);
+  return {
+    viewport: captured.viewport,
+    nodes,
+    ...(surfaces.length > 0 ? { surfaces } : {}),
+    ...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
+  };
 }
 
 export interface SnapshotDeps {
@@ -825,7 +1170,11 @@ export async function handleSnapshot(
     const axNodes = result.nodes ?? [];
     const captured = await captureForVom(deps.cdp, target.tabId);
     const scene = buildVomScene(axNodes, captured);
-    const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens });
+    const rendered = renderVom(scene, {
+      maxDepth: params.max_depth,
+      maxTokens: params.max_tokens,
+      activeRegionPolicy: true,
+    });
     ctx.refStore.replace(
       rendered.refs.map(
         (r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,

apps/extension/src/tools/vom/capture.ts

@@ -35,10 +35,17 @@ export interface CapturedNode {
 
 export type CapturedIframeNodes = Map<number, CapturedNode[]>;
 
+export interface CapturedSurfaceProbe {
+  triggerLabel: string;
+  triggerAction: "hover" | "focus" | string;
+  subItems: string[];
+}
+
 export interface CapturedViewModel {
   nodes: CapturedNode[];
   viewport: Viewport;
   iframeNodes: CapturedIframeNodes;
+  surfaceProbes?: CapturedSurfaceProbe[];
   /** Backend node ids belonging to the agent overlay host + its shadow subtree. */
   excludedBackendNodeIds: Set<number>;
 }
@@ -88,6 +95,20 @@ interface LayoutMetricsReply {
   layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
 }
 
+interface RuntimeEvaluateReply {
+  result?: {
+    value?: unknown;
+  };
+}
+
+interface HoverCandidate {
+  triggerSel: string;
+  affectedSub: string;
+  label: string;
+  x: number;
+  y: number;
+}
+
 interface CdpDomNode {
   backendNodeId?: number;
   children?: CdpDomNode[];
@@ -126,6 +147,160 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
   }
 }
 
+const MAX_HOVER_PROBE_MS = 2_000;
+const MAX_HOVER_TRIGGERS = 8;
+const HOVER_SETTLE_MS = 200;
+
+function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
+  return reply.result?.value as T | undefined;
+}
+
+function hoverCssScanExpression(): string {
+  return `(() => {
+    const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
+    const pairs = [];
+    const seenRules = new Set();
+    for (const sheet of Array.from(document.styleSheets)) {
+      let rules;
+      try { rules = sheet.cssRules; } catch { continue; }
+      if (!rules) continue;
+      for (const rule of Array.from(rules)) {
+        if (rule.type !== 1 || !rule.selectorText || !rule.style) continue;
+        const selectorText = String(rule.selectorText);
+        if (!selectorText.includes(":hover")) continue;
+        if (!visibilityProps.some((prop) => rule.style[prop])) continue;
+        for (const rawPart of selectorText.split(",")) {
+          const part = rawPart.trim();
+          const hoverIndex = part.indexOf(":hover");
+          if (hoverIndex < 0) continue;
+          const triggerSel = part.slice(0, hoverIndex).trim();
+          const affectedSub = part.slice(hoverIndex + 6).trim();
+          if (!triggerSel) continue;
+          const key = triggerSel + "||" + affectedSub;
+          if (seenRules.has(key)) continue;
+          seenRules.add(key);
+          pairs.push({ triggerSel, affectedSub });
+        }
+      }
+    }
+
+    const candidates = [];
+    const seenLabels = new Set();
+    for (const pair of pairs) {
+      let elements;
+      try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
+      for (const el of elements) {
+        if (!(el instanceof Element)) continue;
+        const rect = el.getBoundingClientRect();
+        const style = getComputedStyle(el);
+        if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
+        const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
+        if (!label || seenLabels.has(label)) continue;
+        seenLabels.add(label);
+        candidates.push({
+          triggerSel: pair.triggerSel,
+          affectedSub: pair.affectedSub,
+          label,
+          x: rect.left + rect.width / 2,
+          y: rect.top + rect.height / 2,
+        });
+        if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
+      }
+    }
+    return candidates;
+  })()`;
+}
+
+function hoverCollectExpression(candidate: HoverCandidate): string {
+  return `(() => {
+    const triggerSel = ${JSON.stringify(candidate.triggerSel)};
+    const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
+    const x = ${JSON.stringify(candidate.x)};
+    const y = ${JSON.stringify(candidate.y)};
+    const hit = document.elementFromPoint(x, y);
+    const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
+    if (!trigger) return [];
+
+    const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
+    let targets = [];
+    if (!affectedSub) {
+      targets = [trigger];
+    } else {
+      try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
+    }
+    const items = [];
+    const seen = new Set();
+    const push = (value) => {
+      const text = String(value || "").replace(/\\s+/g, " ").trim();
+      if (!text || seen.has(text)) return;
+      seen.add(text);
+      items.push(text);
+    };
+    for (const target of targets) {
+      if (!(target instanceof Element)) continue;
+      const style = getComputedStyle(target);
+      if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
+      const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
+      if (clickable.length > 0) {
+        for (const child of Array.from(clickable)) push(child.textContent);
+      } else {
+        push(target.textContent);
+      }
+    }
+    return items.slice(0, 12);
+  })()`;
+}
+
+async function wait(ms: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
+  const started = Date.now();
+  try {
+    const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
+      expression: hoverCssScanExpression(),
+      returnByValue: true,
+    });
+    const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
+    const results: CapturedSurfaceProbe[] = [];
+    const seen = new Set<string>();
+    for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
+      if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
+      if (!candidate.label || seen.has(candidate.label)) continue;
+      try {
+        await cdp.send(tabId, "Input.dispatchMouseEvent", {
+          type: "mouseMoved",
+          x: candidate.x,
+          y: candidate.y,
+        });
+        await wait(HOVER_SETTLE_MS);
+        const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
+          expression: hoverCollectExpression(candidate),
+          returnByValue: true,
+        });
+        await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
+        const subItems = (runtimeValue<string[]>(collected) ?? [])
+          .map((item) => item.replace(/\s+/g, " ").trim())
+          .filter(Boolean);
+        if (subItems.length === 0) continue;
+        seen.add(candidate.label);
+        results.push({
+          triggerLabel: candidate.label,
+          triggerAction: "hover",
+          subItems,
+        });
+      } catch {
+        continue;
+      }
+    }
+    return results;
+  } catch (err) {
+    console.debug("[bsk capture] hover surface probe failed", err);
+    return [];
+  }
+}
+
 /**
  * When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
  * collect every backendNodeId in its pierced subtree (open shadow included).
@@ -319,7 +494,13 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
   const documents = snap.documents ?? [];
   const doc0 = documents[0];
   if (!doc0?.nodes?.backendNodeId) {
-    return { nodes: [], viewport, iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
+    return {
+      nodes: [],
+      viewport,
+      iframeNodes: new Map(),
+      surfaceProbes: [],
+      excludedBackendNodeIds: new Set(),
+    };
   }
 
   const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
@@ -352,5 +533,7 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
     }
   }
 
-  return { nodes, viewport, iframeNodes, excludedBackendNodeIds };
+  const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
+
+  return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
 }