From 2ccc0aafb932377d5721b1d98c07be8f1ed5273b Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 7 Aug 2026 15:45:38 +0800 Subject: [PATCH] feat(extension): hover chain node --- .../content/__tests__/record-capture.test.ts | 508 +++++++++++++++++- apps/extension/src/content/record-capture.ts | 275 ++++++++-- .../src/content/record-hover-surface.ts | 75 ++- 3 files changed, 783 insertions(+), 75 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 16059d0..051b743 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -25,13 +25,13 @@ function mockRect( }); } -function mockHoverStyle(pointerElements: Element[]): void { +function mockHoverStyle(pointerElements: Element[], positionedElements: Element[] = []): void { vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { const style = { cursor: pointerElements.includes(el) ? "pointer" : "", display: "block", pointerEvents: "auto", - position: "static", + position: positionedElements.includes(el) ? "absolute" : "static", visibility: "visible", } as CSSStyleDeclaration; return style; @@ -204,6 +204,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-menu", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -216,6 +217,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); @@ -244,6 +246,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-topbar", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -256,6 +259,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); @@ -280,18 +284,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-contained-menu", (step) => steps.push(step)); const trigger = document.querySelector('[role="button"]')!; + const menu = document.querySelector("ul")!; const item = document.querySelector("a")!; - vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ - x: 900, - y: 8, - top: 8, - left: 900, - right: 932, - bottom: 40, - width: 32, - height: 32, - toJSON: () => ({}), - }); + mockRect(trigger, { left: 900, top: 8, width: 32, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); @@ -320,6 +316,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-topbar-link", (step) => steps.push(step)); const trigger = document.querySelector('a[aria-label="image"]')!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("ul a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -343,6 +340,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { const style = { cursor: el === item ? "pointer" : "", @@ -428,8 +426,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector("li")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item]); @@ -458,8 +458,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector(".tg-menu-item")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item]); @@ -493,8 +495,10 @@ describe("record-capture semantic", () => { steps.push(step), ); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector(".tg-menu-item")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item]); @@ -527,8 +531,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector(".tg-menu-item")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([trigger, item]); @@ -557,9 +563,11 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector("li")!; const itemInner = document.querySelector("li div")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item, itemInner]); @@ -591,14 +599,18 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step)); const trigger = document.querySelector("button[aria-label]")!; + const menu = document.querySelector(".create-menu")!; + const submenu = document.querySelector(".template-submenu")!; const nestedTrigger = document.querySelector("li")!; const finalAction = document.querySelector(".template-submenu button")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); - mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); mockHoverStyle([nestedTrigger, finalAction]); mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); mouseOver(nestedTrigger); click(finalAction); capture.dispose(); @@ -618,6 +630,468 @@ describe("record-capture semantic", () => { }); }); + it("infers the full hover opener chain for nested menu actions", () => { + document.body.innerHTML = ` + +
+ +
+ + `; + const capture = startRecordCapture("rec-hover-nested-infer", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const submenu = document.querySelector(".format-submenu")!; + const nestedTrigger = document.querySelector("li")!; + const finalAction = document.querySelector("span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "文档C+D" }, + }); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "Markdown" }, + }); + }); + + it("uses the nested hover trigger label without descendant submenu text", () => { + document.body.innerHTML = ` + +
+
  • + 多维表格 +
    + 表格视图 +
    看板视图
    + 甘特视图 + 日历视图 + 相册视图 + 架构视图 + 神奇表单 +
    +
    • +
    + `; + const capture = startRecordCapture("rec-hover-nested-compact-label", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const nestedTrigger = document.querySelector("li")!; + const submenu = document.querySelector("li > div")!; + const finalAction = document.querySelector("li > div div")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, nestedTrigger, finalAction], [submenu]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 180, height: 220 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps[1]?.target)).not.toContain("看板视图"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "div", name: "看板视图" }, + }); + }); + + it("does not let a stale pass-through shortcut hover claim a later submenu", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T06:39:40.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    + 看板视图 +
    +
    • +
    + `; + const capture = startRecordCapture("rec-hover-stale-shortcut", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const shortcut = document.querySelector(".text-font-tips")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 160 }); + mockRect(shortcut, { left: 950, top: 56, width: 29, height: 22 }); + mockRect(nestedTrigger, { left: 820, top: 92, width: 160, height: 32 }); + mockHoverStyle([trigger, shortcut, nestedTrigger, finalAction], [submenu]); + + mouseOver(trigger); + mouseOver(shortcut); + mockRect(submenu, { left: 984, top: 92, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 92, width: 136, height: 32 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("C+D"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "看板视图" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("does not let an item inside a newly opened menu claim the parent surface", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:12:08.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
  • + 多维表格 +
    • +
    +
    + 看板视图 +
    + `; + const capture = startRecordCapture("rec-hover-parent-surface-owner", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mouseOver(passThrough); + vi.runOnlyPendingTimers(); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + } finally { + vi.useRealTimers(); + } + }); + + it("infers an unowned existing parent surface when a nested submenu is opened", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:28:10.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    + 看板视图 +
    + `; + const capture = startRecordCapture("rec-hover-existing-parent-surface", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(passThrough); + mouseOver(trigger); + vi.runOnlyPendingTimers(); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + } finally { + vi.useRealTimers(); + } + }); + + it("assigns a side submenu to the vertically aligned hover trigger", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T06:55:38.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    +
  • + 看板视图 +
    • +
    + `; + const capture = startRecordCapture("rec-hover-side-submenu-alignment", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(trigger); + mouseOver(passThrough); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + vi.runOnlyPendingTimers(); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "看板视图" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("outputs only one parent path for noisy nested menu hover movement", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:02:36.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    +
    +
    + 表格视图 +
    +
    +
    + `; + const capture = startRecordCapture("rec-hover-noisy-nested-path", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const nestedInner = document.querySelector(".t-dropdown__item-content")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalItem = document.querySelector(".create-submenu-item-vika-kanban")!; + const finalHoverItem = document.querySelector(".create-submenu-item-label")!; + const finalAction = document.querySelector(".create-submenu-item-vika-kanban")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockRect(nestedInner, { left: 25, top: 234, width: 168, height: 22 }); + mockHoverStyle([ + trigger, + passThrough, + nestedTrigger, + nestedInner, + finalItem, + finalHoverItem, + finalAction, + ]); + + mouseOver(trigger); + mouseOver(nestedTrigger); + mouseOver(nestedInner); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalItem, { left: 209, top: 257, width: 97, height: 34 }); + mockRect(finalHoverItem, { left: 209, top: 257, width: 97, height: 34 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(passThrough); + mouseOver(nestedInner); + mouseOver(finalHoverItem); + vi.runOnlyPendingTimers(); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + expect(JSON.stringify(steps)).not.toContain('"表格视图","tag":"div"'); + expect(steps.filter((step) => step.op === "hover")).toHaveLength(2); + } finally { + vi.useRealTimers(); + } + }); + + it("does not infer sibling items in the same surface as hover openers", () => { + document.body.innerHTML = ` + +
    + + Markdown +
    + `; + const capture = startRecordCapture("rec-hover-same-surface-action", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const category = document.querySelector("li")!; + const finalAction = document.querySelector("span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 320, height: 80 }); + mockRect(category, { left: 820, top: 48, width: 160, height: 32 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mockHoverStyle([trigger, category, finalAction]); + + mouseOver(trigger); + mouseOver(category); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "span", name: "Markdown" }, + }); + }); + it("records avatar div hover with a semantic image target", () => { document.body.innerHTML = `
    @@ -630,6 +1104,7 @@ describe("record-capture semantic", () => { const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step)); const trigger = document.querySelector(".tg-avatar")!; const image = document.querySelector("img")!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -653,6 +1128,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); vi.spyOn(window, "getComputedStyle").mockReturnValue({ cursor: "pointer", pointerEvents: "auto", @@ -717,6 +1193,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const input = document.querySelector("input")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -729,6 +1206,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 96 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); @@ -762,6 +1240,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const select = document.querySelector("select")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -774,6 +1253,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 96 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); select.value = "away"; diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 530085b..d693415 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -28,7 +28,13 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; -import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface"; +import { + closestHoverSurfaceCandidate, + collectHoverSurfaceStates, + decideHoverSurfaceRelation, + type HoverSurfaceState, + isHoverSurfaceCandidateElement, +} from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -68,16 +74,23 @@ interface HoverCandidate { target: TargetDescriptor; recordedAt: number; score: number; - hasExpansionSignal: boolean; eligible: boolean; } +interface HoverSurfaceNode { + element: Element; + signature: string; + owner: HoverCandidate; + parent?: HoverSurfaceNode; +} + type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000; const HOVER_REPLACE_SCORE_MARGIN = 50; const HOVER_CANDIDATE_LIMIT = 24; +const HOVER_TRIGGER_LABEL_MAX = 48; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -191,6 +204,51 @@ function hasHoverGraphicDescendant(el: Element): boolean { return el.querySelector("img,svg,use,path,i") !== null; } +function normalizeLabelText(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string { + const normalized = normalizeLabelText(value); + if (normalized.length <= max) return normalized; + return `${normalized.slice(0, max - 1)}…`; +} + +function collectHoverTriggerLabelText(root: Element): string { + let text = ""; + const visit = (node: Node) => { + if (node.nodeType === Node.TEXT_NODE) { + text += ` ${node.textContent ?? ""}`; + return; + } + if (!(node instanceof Element)) return; + if (node !== root && isHoverSurfaceCandidateElement(node)) { + return; + } + for (const child of node.childNodes) visit(child); + }; + for (const child of root.childNodes) visit(child); + return normalizeLabelText(text); +} + +function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor { + if (!desc.name) return desc; + const fullText = normalizeLabelText(el.textContent ?? ""); + const compactName = collectHoverTriggerLabelText(el); + if (!compactName || compactName === desc.name) return desc; + const compactComparable = compactName.replace(/\s+/g, ""); + const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, ""); + const fullTextComparable = fullText.replace(/\s+/g, ""); + if ( + descComparable.startsWith(compactComparable) && + fullTextComparable.startsWith(descComparable) && + compactName.length < desc.name.length + ) { + return { ...desc, name: truncateLabel(compactName) }; + } + return desc; +} + function isWeakHoverTarget(target: TargetDescriptor): boolean { return !target.role && !target.name && target.tag === "div"; } @@ -245,13 +303,17 @@ function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | n const hasDirectSignal = hasDirectHoverInteractiveSignal(signals); if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null; if (!decision.eligible && !desc.name && !desc.role) return null; - const normalizedTarget = normalizeHoverTarget(element, desc, decision); + const normalizedTarget = normalizeHoverTarget( + element, + compactHoverTargetName(element, desc), + decision, + ); + const now = Date.now(); return { element, target: normalizedTarget, - recordedAt: Date.now(), + recordedAt: now, score: decision.score, - hasExpansionSignal, eligible: decision.eligible, }; } @@ -321,6 +383,8 @@ export function startRecordCapture( let pendingHover: HoverCandidate | null = null; const recentHoverCandidates: HoverCandidate[] = []; const emittedHoverElements = new WeakSet(); + let hoverSurfaceStates = new Map(); + const hoverSurfaceNodes = new Map(); let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -397,25 +461,126 @@ export function startRecordCapture( } }; - const actionIsOnHoverTriggerItself = ( - hover: HoverCandidate, - actionElement: Element | null, - relationTarget: Element, - ): boolean => { - const target = actionElement ?? relationTarget; - if (target === hover.element) return true; - if (!hover.element.contains(target)) return false; - const surface = closestRecognizedHoverSurface(target); - return !surface || surface === hover.element || !hover.element.contains(surface); + const surfaceArea = (el: Element): number => { + const rect = el.getBoundingClientRect(); + return rect.width * rect.height; }; - const hoverRelatedToAction = ( - hover: HoverCandidate, - relationTarget: Element, + const smallestContaining = ( + target: Element, + items: Iterable, + elementFor: (item: T) => Element, + ): T | undefined => { + let best: T | undefined; + for (const item of items) { + const element = elementFor(item); + if (!element.contains(target)) continue; + if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) { + best = item; + } + } + return best; + }; + + const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined => + smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element); + + const surfaceStateContaining = ( + target: Element, + states: HoverSurfaceState[], + ): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element); + + const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => { + const closestSurface = closestHoverSurfaceCandidate(target); + if (closestSurface) { + const owned = hoverSurfaceNodes.get(closestSurface); + if (owned) return owned; + } + return ownedSurfaceContaining(target); + }; + + const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => { + const current = new Set(currentStates.map((state) => state.element)); + for (const element of hoverSurfaceNodes.keys()) { + if (!current.has(element)) hoverSurfaceNodes.delete(element); + } + }; + + const inferOwnerForParentSurface = ( + surface: Element, + childOwner: HoverCandidate, now: number, - ): boolean => { - if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false; - return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related; + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate || candidate.element === childOwner.element) continue; + if (surface.contains(candidate.element)) continue; + if (candidate.recordedAt > childOwner.recordedAt) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface); + if (relation.related) return candidate; + } + return undefined; + }; + + const parentSurfaceNodeForOwner = ( + owner: HoverCandidate, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const ownedParent = ownedSurfaceContaining(owner.element); + if (ownedParent) return ownedParent; + const parentState = surfaceStateContaining(owner.element, currentStates); + if (!parentState) return undefined; + const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now); + if (!parentOwner) return undefined; + const parentNode: HoverSurfaceNode = { + element: parentState.element, + signature: parentState.signature, + owner: parentOwner, + }; + hoverSurfaceNodes.set(parentState.element, parentNode); + return parentNode; + }; + + const bindOwnedHoverSurfaces = ( + owner: HoverCandidate, + previousStates: Map, + currentStates: HoverSurfaceState[], + now: number, + ) => { + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + for (const state of currentStates) { + if (previousStates.get(state.element) === state.signature) continue; + if (state.element.contains(owner.element)) continue; + if (parent?.element === state.element) continue; + const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element); + if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + hoverSurfaceNodes.set(state.element, { + element: state.element, + signature: state.signature, + owner, + ...(parent ? { parent } : {}), + }); + } + }; + + const refreshCausedHoverSurfaces = ( + owner: HoverCandidate, + previousStates = hoverSurfaceStates, + ) => { + const now = Date.now(); + const currentStates = collectHoverSurfaceStates(); + pruneGoneHoverSurfaces(currentStates); + bindOwnedHoverSurfaces(owner, previousStates, currentStates, now); + hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature])); + }; + + const scheduleCausedHoverSurfaceRefresh = ( + hover: HoverCandidate, + previousStates: Map, + ) => { + setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0); }; const emitHoverStep = (hover: HoverCandidate) => { @@ -427,25 +592,31 @@ export function startRecordCapture( emittedHoverElements.add(hover.element); }; - const findHoverForAction = ( - actionElement: Element | null, - relationTarget: Element, + const surfaceOwnerChain = ( + surface: HoverSurfaceNode, + actionElement: Element, now: number, - ): HoverCandidate | null => { - const candidates = pendingHover - ? [ - ...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element), - pendingHover, - ] - : recentHoverCandidates; - for (let index = candidates.length - 1; index >= 0; index -= 1) { - const hover = candidates[index]; - if (!hover) continue; - if (emittedHoverElements.has(hover.element)) continue; - if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue; - if (hoverRelatedToAction(hover, relationTarget, now)) return hover; + ): HoverCandidate[] => { + const chain: HoverCandidate[] = []; + const selected = new WeakSet(); + const surfaces: HoverSurfaceNode[] = []; + let node: HoverSurfaceNode | undefined = surface; + while (node && surfaces.length < 4) { + surfaces.unshift(node); + node = node.parent; } - return null; + for (const owned of surfaces) { + const owner = owned.owner; + if (owner.element === actionElement || actionElement.contains(owner.element)) { + continue; + } + if (selected.has(owner.element)) continue; + if (emittedHoverElements.has(owner.element)) continue; + if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + selected.add(owner.element); + chain.push(owner); + } + return chain; }; const emitClick = (event: MouseEvent) => { @@ -462,14 +633,14 @@ export function startRecordCapture( const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => { if (!(actionTarget instanceof Element)) return; - const actionElement = resolveClickableElement(actionTarget); - const relationTarget = actionElement ?? actionTarget; - const hover = findHoverForAction(actionElement, relationTarget, Date.now()); - if (!hover) { + const actionElement = resolveClickableElement(actionTarget) ?? actionTarget; + const surface = ownedSurfaceForAction(actionElement); + const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : []; + if (hoverChain.length === 0) { pendingHover = null; return; } - emitHoverStep(hover); + for (const hover of hoverChain) emitHoverStep(hover); pendingHover = null; }; @@ -482,21 +653,16 @@ export function startRecordCapture( pendingHover = null; } const candidate = hoverCandidateFromEvent(target); - if (candidate) rememberHoverCandidate(candidate); + if (candidate) { + const previousSurfaceStates = new Map(hoverSurfaceStates); + rememberHoverCandidate(candidate); + refreshCausedHoverSurfaces(candidate, previousSurfaceStates); + scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates); + } const hover = candidate?.eligible ? candidate : null; if (pendingHover && target instanceof Element) { const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target); if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) { - if (!hover || !hover.hasExpansionSignal) return; - if ( - hover.element === pendingHover.element || - pendingHover.element.contains(hover.element) - ) { - return; - } - emitHoverStep(pendingHover); - pendingHover = hover; - rememberHoverCandidate(hover); return; } } @@ -505,7 +671,6 @@ export function startRecordCapture( return; } pendingHover = hover; - rememberHoverCandidate(hover); }; const onClick = (event: MouseEvent) => { diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index c6fbd43..e737cd3 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -8,6 +8,11 @@ export interface HoverSurfaceContext { triggerElement: Element; } +export interface HoverSurfaceState { + element: Element; + signature: string; +} + const HOVER_SURFACE_SELECTOR = [ '[role="menu"]', '[role="menubar"]', @@ -22,8 +27,11 @@ const HOVER_SURFACE_SELECTOR = [ ].join(","); const HOVER_SURFACE_WORD_RE = - /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i; + /(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; +const HOVER_SURFACE_ITEM_RE = + /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; const HOVER_SURFACE_MAX_AXIS_GAP = 96; +const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16; const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; function elementTokenText(el: Element): string { @@ -40,9 +48,12 @@ function elementTokenText(el: Element): string { .join(" "); } -function isHoverSurfaceElement(el: Element): boolean { +export function isRecognizedHoverSurfaceElement(el: Element): boolean { if (el.matches(HOVER_SURFACE_SELECTOR)) return true; - if (HOVER_SURFACE_WORD_RE.test(elementTokenText(el))) return true; + const tokenText = elementTokenText(el); + if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) { + return true; + } const tag = el.tagName.toLowerCase(); if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) { return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el)); @@ -50,18 +61,27 @@ function isHoverSurfaceElement(el: Element): boolean { return false; } +function isVisibleSurfaceElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + const style = getComputedStyle(el); + if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { + return false; + } + return rectFor(el) !== null; +} + export function closestRecognizedHoverSurface(element: Element): Element | null { let node: Element | null = element; let depth = 0; while (node && node !== document.body && node !== document.documentElement && depth < 10) { - if (isHoverSurfaceElement(node)) return node; + if (isRecognizedHoverSurfaceElement(node)) return node; node = node.parentElement; depth += 1; } return null; } -function isPositionedFloatingElement(el: Element): boolean { +export function isPositionedFloatingElement(el: Element): boolean { if (!(el instanceof HTMLElement)) return false; const style = getComputedStyle(el); if (!["absolute", "fixed"].includes(style.position)) return false; @@ -77,6 +97,21 @@ function isPositionedFloatingElement(el: Element): boolean { ); } +export function isHoverSurfaceCandidateElement(el: Element): boolean { + return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el); +} + +export function closestHoverSurfaceCandidate(element: Element): Element | null { + let node: Element | null = element; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isHoverSurfaceCandidateElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + function closestFloatingSurface(clickElement: Element): Element | null { let node: Element | null = clickElement; let depth = 0; @@ -88,6 +123,26 @@ function closestFloatingSurface(clickElement: Element): Element | null { return null; } +function hoverSurfaceSignature(el: Element): string { + const rect = rectFor(el); + return [ + rect + ? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}` + : "", + (el.textContent ?? "").trim().slice(0, 512), + ].join("|"); +} + +export function collectHoverSurfaceStates(): HoverSurfaceState[] { + const states: HoverSurfaceState[] = []; + for (const el of document.querySelectorAll("*")) { + if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) { + states.push({ element: el, signature: hoverSurfaceSignature(el) }); + } + } + return states; +} + function rectFor(el: Element): DOMRect | null { const rect = el.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return null; @@ -98,6 +153,10 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { const surfaceRect = rectFor(surface); const triggerRect = rectFor(trigger); if (!surfaceRect || !triggerRect) return true; + const horizontalOverlap = + Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left); + const verticalOverlap = + Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top); const horizontalGap = Math.max( 0, triggerRect.left - surfaceRect.right, @@ -108,7 +167,11 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { triggerRect.top - surfaceRect.bottom, surfaceRect.top - triggerRect.bottom, ); - return horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP; + if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; + if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; + return ( + horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP + ); } export function decideHoverSurfaceRelation(