feat(vom): vom method bug fix

apps/extension/src/tools/__tests__/observation.test.ts

@@ -848,11 +848,7 @@ describe("buildVomScene", () => {
     );
   });
 
-  it("prefers an empty field's placeholder over a label-polluted AX name", () => {
-    // Mirrors xiaohongshu: the <input> is wrapped in a <label> that also holds
-    // a "+86" prefix, so the AX accessible name becomes "+86" and the real
-    // placeholder ("输入手机号") is lost. VOM models perceived state — the empty
-    // field shows its placeholder — so the placeholder wins.
+  it("keeps placeholder separate from the accessible name", () => {
     const axNodes: CdpAxNode[] = [
       {
         nodeId: "1",
@@ -881,7 +877,12 @@ describe("buildVomScene", () => {
     };
 
     expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
-      expect.objectContaining({ id: 10, role: "textbox", name: "输入手机号" }),
+      expect.objectContaining({
+        id: 10,
+        role: "textbox",
+        name: "+86",
+        placeholder: "输入手机号",
+      }),
     );
   });
 
@@ -979,6 +980,89 @@ describe("buildVomScene", () => {
     );
   });
 
+  it("uses preceding AX static text as a form control label", () => {
+    const axNodes: CdpAxNode[] = [
+      {
+        nodeId: "1",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 1,
+        childIds: ["2", "3"],
+      },
+      {
+        nodeId: "2",
+        parentId: "1",
+        role: { type: "role", value: "StaticText" },
+        name: { type: "computedString", value: "验证码:" },
+      },
+      {
+        nodeId: "3",
+        parentId: "1",
+        role: { type: "role", value: "textbox" },
+        backendDOMNodeId: 20,
+      },
+    ];
+    const captured: CapturedViewModel = {
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      excludedBackendNodeIds: new Set(),
+      nodes: [
+        {
+          backendNodeId: 20,
+          parentBackendNodeId: null,
+          tag: "input",
+          attrs: { type: "text" },
+          rect: { x: 90, y: 0, w: 120, h: 30 },
+          paintOrder: 2,
+          position: "static",
+          pointerEvents: "auto",
+          cursor: "text",
+        },
+      ],
+    };
+
+    expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
+      expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }),
+    );
+  });
+
+  it("marks runtime default input values without treating them as ordinary filled input", () => {
+    const scene = buildVomScene(
+      [
+        {
+          nodeId: "1",
+          role: { type: "role", value: "textbox" },
+          backendDOMNodeId: 10,
+        },
+      ],
+      {
+        viewport: { width: 1000, height: 800 },
+        iframeNodes: new Map(),
+        excludedBackendNodeIds: new Set(),
+        nodes: [
+          {
+            backendNodeId: 10,
+            parentBackendNodeId: null,
+            tag: "input",
+            attrs: { type: "text" },
+            formValue: "没有手机号的用编码代替",
+            formDefaultValue: "没有手机号的用编码代替",
+            rect: { x: 0, y: 0, w: 240, h: 48 },
+            paintOrder: 1,
+            position: "static",
+            pointerEvents: "auto",
+          },
+        ],
+      },
+    );
+
+    expect(scene.nodes[0]).toEqual(
+      expect.objectContaining({
+        inputState: "default",
+        value: "没有手机号的用编码代替",
+      }),
+    );
+  });
+
   it("promotes a cursor:pointer icon control to a button named from its <use> sprite", () => {
     // A close button built as <div class=close-button><svg><use href="#close"/></svg></div>
     // — the AX tree exposes it as `generic` with no name, so it would be dropped.

apps/extension/src/tools/observation.ts

@@ -329,6 +329,27 @@ function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
   return axString(prop?.value);
 }
 
+function axSiblingLabel(axNode: CdpAxNode, axById: Map<string, CdpAxNode>): string | undefined {
+  const parent = axNode.parentId ? axById.get(axNode.parentId) : undefined;
+  const siblings = parent?.childIds ?? [];
+  const index = siblings.indexOf(axNode.nodeId);
+  if (index <= 0) return undefined;
+
+  for (let i = index - 1; i >= Math.max(0, index - 4); i -= 1) {
+    const sibling = axById.get(siblings[i]);
+    if (!sibling) continue;
+    const role = axString(sibling.role)?.toLowerCase() ?? "";
+    if (AX_TEXT_STOP_ROLES.has(role)) break;
+    if (!["statictext", "labeltext", "text", "generic"].includes(role)) continue;
+    const label = cleanAttr(axString(sibling.name) ?? axString(sibling.value))?.replace(
+      /[：:]\s*$/,
+      "",
+    );
+    if (label && label.length <= 40) return label;
+  }
+  return undefined;
+}
+
 function buildAxSignals(axNodes: CdpAxNode[]): Map<string, AxNodeSignals> {
   const axByNodeId = new Map(axNodes.map((node) => [node.nodeId, node]));
   const virtualText = new Map<string, string>();
@@ -619,6 +640,7 @@ function applyCapturedSignals(
   const attrs = capturedNode.attrs;
   const text = cleanAttr(capturedNode.textContent);
   const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
+  const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
   return {
     ...node,
     domParentId: capturedNode.parentBackendNodeId,
@@ -627,6 +649,7 @@ function applyCapturedSignals(
     attrs,
     ...(text ? { text } : {}),
     ...(nearbyText ? { nearbyText } : {}),
+    ...(placeholder ? { placeholder } : {}),
     disabled:
       Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
       (attrs["aria-disabled"] ?? "").toLowerCase() === "true",
@@ -636,6 +659,20 @@ function applyCapturedSignals(
   };
 }
 
+function inputStateFor(
+  capturedNode: CapturedNode | undefined,
+  value: string | undefined,
+): VomNode["inputState"] {
+  if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined;
+  const formValue = capturedNode.formValue;
+  if (formValue !== undefined) {
+    if (formValue === "") return "empty";
+    if (formValue === (capturedNode.formDefaultValue ?? "")) return "default";
+    return "filled";
+  }
+  return value === undefined || value === "" ? "empty" : "filled";
+}
+
 /**
  * Name for a native form control. VOM models the *perceived* viewport
  * (spec §1): an empty field displays its placeholder, so when the field has
@@ -647,19 +684,17 @@ function applyCapturedSignals(
 function formControlName(
   captured: CapturedNode | undefined,
   axName: string | undefined,
-  axValue: string | undefined,
+  axSiblingName: string | undefined,
   signals: VomNodeDomSignals,
 ): string | undefined {
-  const placeholder = cleanAttr(captured?.attrs.placeholder);
   const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
   const title = cleanAttr(captured?.attrs.title);
+  const placeholder =
+    cleanAttr(captured?.formPlaceholder) ?? cleanAttr(captured?.attrs.placeholder);
   const nearbyLabel = captured
     ? previousSiblingTextFor(captured, signals.childrenByParentId)
     : undefined;
-  const hasValue = axValue !== undefined && axValue !== "";
-  if (nearbyLabel) return nearbyLabel;
-  if (!hasValue && placeholder) return placeholder;
-  return axName ?? ariaLabel ?? placeholder ?? title ?? nearbyLabel;
+  return ariaLabel ?? title ?? axName ?? axSiblingName ?? nearbyLabel ?? placeholder;
 }
 
 function vomNodeFromCaptured(
@@ -668,7 +703,8 @@ function vomNodeFromCaptured(
   signals: VomNodeDomSignals,
 ): VomNode {
   const sensitive = isSensitive(capturedNode);
-  const value = sensitive ? (capturedNode.attrs.value ?? "") : undefined;
+  const value =
+    capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
   const tag = normalizeTag(capturedNode.tag);
   const node = applyCapturedSignals(
     {
@@ -681,6 +717,7 @@ function vomNodeFromCaptured(
       pointerEvents: capturedNode.pointerEvents || "auto",
       modal: isModalSignal(undefined, capturedNode),
       sensitive,
+      inputState: inputStateFor(capturedNode, value),
       ...(value !== undefined ? { value } : {}),
     },
     capturedNode,
@@ -702,10 +739,11 @@ function axVomNode(
   axSignals: Map<string, AxNodeSignals>,
 ): VomNode {
   const role = axString(axNode.role);
-  const value = axValue(axNode.value);
+  const capturedValue = capturedNode?.formValue;
+  const value = capturedValue ?? axValue(axNode.value);
   const signalsForNode = axSignals.get(axNode.nodeId);
   let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
-    ? formControlName(capturedNode, axString(axNode.name), value, signals)
+    ? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
     : (axString(axNode.name) ?? signalsForNode?.aggregatedText);
   if (name && signalsForNode?.hasPopup) {
     name = signalsForNode.expanded ? `${name} [expanded]` : `${name} [has-submenu]`;
@@ -720,6 +758,7 @@ function axVomNode(
     pointerEvents: capturedNode?.pointerEvents || "auto",
     modal: isModalSignal(axNode, capturedNode),
     sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
+    inputState: inputStateFor(capturedNode, value),
   };
   if (role) node.role = role;
   if (name) node.name = name;

apps/extension/src/tools/vom/capture.ts

@@ -31,6 +31,9 @@ export interface CapturedNode {
    */
   cursor?: string;
   textContent?: string;
+  formValue?: string;
+  formDefaultValue?: string;
+  formPlaceholder?: string;
 }
 
 export type CapturedIframeNodes = Map<number, CapturedNode[]>;
@@ -95,6 +98,52 @@ interface LayoutMetricsReply {
   layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
 }
 
+function isFormControlTag(tag: string): boolean {
+  return tag === "input" || tag === "textarea" || tag === "select";
+}
+
+async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) {
+  for (const node of nodes) {
+    if (!isFormControlTag(node.tag)) continue;
+    let objectId: string | undefined;
+    try {
+      const resolved = await cdp.send<{ object?: { objectId?: string } }>(
+        tabId,
+        "DOM.resolveNode",
+        {
+          backendNodeId: node.backendNodeId,
+        },
+      );
+      objectId = resolved.object?.objectId;
+      if (!objectId) continue;
+      const result = await cdp.send<{
+        result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } };
+      }>(tabId, "Runtime.callFunctionOn", {
+        objectId,
+        returnByValue: true,
+        functionDeclaration: `function(){
+          return {
+            value: typeof this.value === "string" ? this.value : "",
+            defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "",
+            placeholder: typeof this.placeholder === "string" ? this.placeholder : ""
+          };
+        }`,
+      });
+      const value = result.result?.value;
+      if (!value) continue;
+      node.formValue = value.value ?? "";
+      node.formDefaultValue = value.defaultValue ?? "";
+      node.formPlaceholder = value.placeholder ?? "";
+    } catch {
+      // Best-effort enrichment. DOMSnapshot/AX data still carries the node.
+    } finally {
+      if (objectId) {
+        await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
+      }
+    }
+  }
+}
+
 interface RuntimeEvaluateReply {
   result?: {
     value?: unknown;
@@ -533,6 +582,11 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
     }
   }
 
+  await enrichFormControlState(cdp, tabId, nodes);
+  for (const subNodes of iframeNodes.values()) {
+    await enrichFormControlState(cdp, tabId, subNodes);
+  }
+
   const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
 
   return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };