Merge pull request #68 from Tencent/feat/more-browser-actions

feat(extension): hover end to end capability support

apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts

@@ -0,0 +1,125 @@
+import { describe, expect, it } from "vitest";
+import { evaluateHoverTrigger } from "../hover-trigger-policy";
+
+describe("evaluateHoverTrigger", () => {
+  it("accepts explicit popup triggers", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "button",
+      role: "button",
+      label: "Open user navigation menu",
+      attrs: { "aria-haspopup": "menu", "aria-expanded": "false" },
+      rect: { x: 900, y: 8, w: 32, h: 32 },
+      cursor: "pointer",
+      pointerEvents: "auto",
+    });
+
+    expect(decision.eligible).toBe(true);
+    expect(decision.reasons).toEqual(
+      expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]),
+    );
+  });
+
+  it("accepts compact topbar image buttons without popup attributes", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "button",
+      role: "button",
+      label: "image",
+      attrs: {},
+      rect: { x: 900, y: 8, w: 32, h: 32 },
+      pointerEvents: "auto",
+      hasGraphicDescendant: true,
+    });
+
+    expect(decision.eligible).toBe(true);
+    expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
+  });
+
+  it("accepts compact topbar image links without popup attributes", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "a",
+      role: "link",
+      label: "image",
+      attrs: {},
+      rect: { x: 900, y: 8, w: 32, h: 32 },
+      pointerEvents: "auto",
+      hasGraphicDescendant: true,
+    });
+
+    expect(decision.eligible).toBe(true);
+    expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
+  });
+
+  it("rejects topbar menu links without surface trigger evidence", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "a",
+      role: "link",
+      label: "My profile",
+      attrs: {},
+      rect: { x: 820, y: 72, w: 96, h: 32 },
+      cursor: "pointer",
+      pointerEvents: "auto",
+    });
+
+    expect(decision.eligible).toBe(false);
+  });
+
+  it("rejects text navigation items even when their attributes contain popup words", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "a",
+      role: "link",
+      label: "My profile",
+      attrs: { class: "dropdown-item profile-link" },
+      rect: { x: 820, y: 72, w: 96, h: 32 },
+      cursor: "pointer",
+      pointerEvents: "auto",
+    });
+
+    expect(decision.eligible).toBe(false);
+  });
+
+  it("rejects labelled dropdown list items as hover triggers", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "li",
+      label: "文档C+D",
+      attrs: { class: "dropdown-item", tabindex: "0" },
+      rect: { x: 820, y: 48, w: 160, h: 32 },
+      cursor: "pointer",
+      pointerEvents: "auto",
+    });
+
+    expect(decision.eligible).toBe(false);
+  });
+
+  it("rejects ordinary controls without hover popup signals", () => {
+    const decision = evaluateHoverTrigger({
+      tag: "button",
+      role: "button",
+      label: "Search",
+      attrs: {},
+      rect: { x: 100, y: 240, w: 120, h: 36 },
+      pointerEvents: "auto",
+    });
+
+    expect(decision.eligible).toBe(false);
+  });
+
+  it("rejects unsafe or non-visible triggers", () => {
+    expect(
+      evaluateHoverTrigger({
+        tag: "input",
+        attrs: {},
+        rect: { x: 10, y: 10, w: 80, h: 24 },
+        pointerEvents: "auto",
+      }).eligible,
+    ).toBe(false);
+    expect(
+      evaluateHoverTrigger({
+        tag: "button",
+        role: "button",
+        attrs: { hidden: "" },
+        rect: { x: 10, y: 10, w: 80, h: 24 },
+        pointerEvents: "auto",
+      }).eligible,
+    ).toBe(false);
+  });
+});

apps/extension/src/lib/__tests__/trace-reducer.test.ts

@@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => {
     });
   });
 
+  it("keeps hover steps before menu clicks", () => {
+    const { steps } = reduceTraceSteps(
+      [
+        {
+          op: "hover",
+          target: { tag: "span", role: "button", name: "Account" },
+          page_url: "https://example.com/app",
+        },
+        {
+          op: "click",
+          target: { tag: "a", role: "link", name: "Profile" },
+          page_url: "https://example.com/app",
+        },
+      ],
+      "https://example.com/app",
+    );
+    expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
+    expect(steps[0]).toMatchObject({
+      op: "hover",
+      page: "p1",
+      target: { name: "Account" },
+    });
+  });
+
   it("resolveTraceStartUrl prefers explicit start URL", () => {
     expect(
       resolveTraceStartUrl(

apps/extension/src/lib/describe-target.ts

@@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null {
   return null;
 }
 
+export function resolveHoverElement(target: Element): Element | null {
+  const clickable = resolveClickableElement(target);
+  if (clickable) return clickable;
+
+  let node: Element | null = target;
+  let depth = 0;
+  while (node && node !== document.body && node !== document.documentElement && depth < 8) {
+    if (looksClickable(node)) return node;
+    node = node.parentElement;
+    depth += 1;
+  }
+  return null;
+}
+
 /**
  * Teachable clicks: the LLM must get a label it can later find via snapshot
  * (visible name), or at least a form `name_attr` for checkbox/radio.

apps/extension/src/lib/hover-trigger-policy.ts

@@ -0,0 +1,178 @@
+export interface HoverTriggerRect {
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+}
+
+export interface HoverTriggerSignals {
+  tag: string;
+  role?: string;
+  label?: string;
+  attrs?: Record<string, string | undefined>;
+  rect?: HoverTriggerRect | null;
+  cursor?: string;
+  pointerEvents?: string;
+  hasGraphicDescendant?: boolean;
+  cssHoverMatch?: boolean;
+}
+
+export interface HoverTriggerDecision {
+  eligible: boolean;
+  score: number;
+  reasons: string[];
+}
+
+const HOVER_TRIGGER_MIN_SCORE = 45;
+const HOVER_POPUP_SIGNAL_RE =
+  /(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
+const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
+
+function attr(signals: HoverTriggerSignals, name: string): string | undefined {
+  return signals.attrs?.[name.toLowerCase()];
+}
+
+export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean {
+  const tag = signals.tag.toLowerCase();
+  if (["input", "textarea", "select", "option"].includes(tag)) return true;
+  if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true;
+  if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true;
+  return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true";
+}
+
+export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
+  const attrs = signals.attrs ?? {};
+  const haystack = [
+    attrs.id,
+    attrs.class,
+    attrs["data-testid"],
+    attrs["data-test"],
+    attrs["data-cy"],
+    attrs["aria-label"],
+    attrs["aria-haspopup"],
+    attrs["aria-controls"],
+    attrs.title,
+    signals.role === "button" ? signals.label : undefined,
+  ]
+    .filter(Boolean)
+    .join(" ")
+    .toLowerCase();
+  return HOVER_POPUP_SIGNAL_RE.test(haystack);
+}
+
+export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
+  return (
+    signals.cssHoverMatch === true ||
+    attr(signals, "aria-haspopup") !== undefined ||
+    (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
+  );
+}
+
+export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
+  const tag = signals.tag.toLowerCase();
+  const role = (signals.role ?? "").toLowerCase();
+  return (
+    ["button", "a", "summary"].includes(tag) ||
+    ["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
+    attr(signals, "tabindex") !== undefined ||
+    signals.cursor === "pointer" ||
+    attr(signals, "onclick") !== undefined ||
+    attr(signals, "onmouseenter") !== undefined ||
+    attr(signals, "onmouseover") !== undefined ||
+    attr(signals, "aria-haspopup") !== undefined ||
+    attr(signals, "aria-expanded") === "false"
+  );
+}
+
+export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision {
+  const rect = signals.rect;
+  if (
+    !rect ||
+    rect.w <= 0 ||
+    rect.h <= 0 ||
+    signals.pointerEvents === "none" ||
+    attr(signals, "hidden") !== undefined ||
+    attr(signals, "inert") !== undefined ||
+    (attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" ||
+    isUnsafeHoverTrigger(signals)
+  ) {
+    return { eligible: false, score: 0, reasons: [] };
+  }
+
+  const area = rect.w * rect.h;
+  if (area <= 0 || area > 160_000) {
+    return { eligible: false, score: 0, reasons: [] };
+  }
+
+  const reasons: string[] = [];
+  let score = 0;
+  const tag = signals.tag.toLowerCase();
+  const role = (signals.role ?? "").toLowerCase();
+  const label = signals.label?.trim();
+  const graphic = signals.hasGraphicDescendant === true;
+  const directInteractive = hasDirectHoverInteractiveSignal(signals);
+  const popupSignal = hasHoverPopupSignal(signals);
+  const isTextNavigationItem =
+    (tag === "a" ||
+      tag === "li" ||
+      role === "link" ||
+      role === "option" ||
+      role.startsWith("menuitem") ||
+      HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
+    !graphic &&
+    !!label &&
+    label.toLowerCase() !== "image";
+  const compactTopbarIcon =
+    (role === "button" || role === "link" || tag === "button" || tag === "a") &&
+    graphic &&
+    rect.y < 120 &&
+    rect.w <= 96 &&
+    rect.h <= 96 &&
+    (!label || label.toLowerCase() === "image");
+  const surfaceTriggerEvidence =
+    (popupSignal && !isTextNavigationItem) ||
+    compactTopbarIcon ||
+    hasStrongHoverExpansionSignal(signals);
+  if (!directInteractive || !surfaceTriggerEvidence) {
+    return { eligible: false, score: 0, reasons: [] };
+  }
+
+  if (attr(signals, "aria-haspopup") !== undefined) {
+    score += 80;
+    reasons.push("aria-haspopup");
+  }
+  if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") {
+    score += 55;
+    reasons.push("collapsed");
+  }
+  if (popupSignal) {
+    score += 45;
+    reasons.push("popup-signal");
+  }
+  if (tag === "button" || role === "button") {
+    score += 30;
+    reasons.push("button");
+  }
+  if (signals.cursor === "pointer") {
+    score += 25;
+    reasons.push("pointer");
+  }
+  if (graphic && (!label || label.toLowerCase() === "image")) {
+    score += 40;
+    reasons.push("icon-only");
+  }
+  if (rect.y < 120) {
+    score += 25;
+    reasons.push("topbar");
+  }
+  if (rect.w <= 80 && rect.h <= 80) {
+    score += 20;
+    reasons.push("compact");
+  }
+  if (signals.cssHoverMatch) {
+    score += 40;
+    reasons.push("css-hover");
+  }
+
+  return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons };
+}

apps/extension/src/lib/record-bridge.ts

@@ -41,7 +41,7 @@ export interface RecordStartMessage {
 }
 
 export interface RecordStepPayload {
-  op: "click" | "fill" | "press" | "select" | "navigate";
+  op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
   target?: TargetDescriptor;
   value?: string;
   key?: string;

apps/extension/src/lib/recording-step-buffer.ts

@@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
             ...(pageUrl ? { page_url: pageUrl } : {}),
           }
         : null;
+    case "hover":
+      return payload.target
+        ? {
+            op: "hover",
+            target: payload.target,
+            ...(pageUrl ? { page_url: pageUrl } : {}),
+          }
+        : null;
     case "fill":
       return payload.target
         ? {

apps/extension/src/lib/trace-reducer.ts

@@ -139,6 +139,13 @@ function toV2Step(
         },
         effectForNavigation(step.navigated_to, urlToId),
       );
+    case "hover":
+      return {
+        op: "hover",
+        id,
+        page,
+        target: step.target,
+      };
     case "fill":
       return {
         op: "fill",