diff --git a/apps/extension/src/content/__tests__/help-request.test.ts b/apps/extension/src/content/__tests__/help-request.test.ts new file mode 100644 index 0000000..e1da24e --- /dev/null +++ b/apps/extension/src/content/__tests__/help-request.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from "vitest"; +import { createHelpRequestData } from "../help-request"; + +describe("createHelpRequestData", () => { + it("gives live delivery and recovery the same rectangle refresh mechanism", async () => { + const finish = vi.fn(); + const query = vi.fn(async () => ({ + active: true, + request: { + requestId: "help-1", + prompt: "complete the challenge", + selectors: [], + rects: [{ top: 80, left: 90, width: 140, height: 60 }], + timeoutMs: 1_000, + }, + })); + const request = createHelpRequestData( + { + requestId: "help-1", + prompt: "complete the challenge", + selectors: [], + rects: [{ top: 10, left: 20, width: 30, height: 40 }], + timeoutMs: 1_000, + }, + { finish, query }, + ); + + await expect(request.refreshRects?.()).resolves.toEqual([ + { top: 80, left: 90, width: 140, height: 60 }, + ]); + request.onContinue(" done "); + request.onCancel(); + + expect(query).toHaveBeenCalledOnce(); + expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done "); + expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled"); + }); + + it("ignores rectangle replies for a replaced request", async () => { + const request = createHelpRequestData( + { + requestId: "help-1", + prompt: "complete the challenge", + selectors: [], + timeoutMs: 1_000, + }, + { + finish: vi.fn(), + query: vi.fn(async () => ({ + active: true, + request: { + requestId: "help-2", + prompt: "new challenge", + selectors: [], + rects: [{ top: 1, left: 2, width: 3, height: 4 }], + timeoutMs: 1_000, + }, + })), + }, + ); + + await expect(request.refreshRects?.()).resolves.toBeUndefined(); + }); +}); diff --git a/apps/extension/src/content/help-request.ts b/apps/extension/src/content/help-request.ts new file mode 100644 index 0000000..f5fa9f7 --- /dev/null +++ b/apps/extension/src/content/help-request.ts @@ -0,0 +1,31 @@ +import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge"; +import type { HelpRequestData } from "./HelpRequestOverlay"; + +export interface HelpRequestDataDeps { + finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void; + query(): Promise; +} + +/** Build the content-side representation shared by live delivery and recovery. */ +export function createHelpRequestData( + message: Omit, + deps: HelpRequestDataDeps, +): HelpRequestData { + return { + id: message.requestId, + prompt: message.prompt, + ...(message.title ? { title: message.title } : {}), + ...(message.displayMode ? { displayMode: message.displayMode } : {}), + selectors: message.selectors, + rects: message.rects, + refreshRects: async () => { + const response = await deps.query(); + return response?.active && response.request?.requestId === message.requestId + ? response.request.rects + : undefined; + }, + onContinue: (note: string) => + deps.finish(message.requestId, "continued", note.trim() ? note : undefined), + onCancel: () => deps.finish(message.requestId, "cancelled"), + }; +} diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 9a659d1..7552a09 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -7,6 +7,7 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay"; import { ControlOverlay } from "@/content/ControlOverlay"; import { createCaptureSuppressController } from "@/content/capture-suppress"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; +import { createHelpRequestData } from "@/content/help-request"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { RecordOverlay } from "@/content/RecordOverlay"; @@ -344,17 +345,7 @@ export default defineContentScript({ if (isHelpRequestMessage(message)) { const helpMsg = message as HelpRequestMessage; - const previousHelp = overlays.setAgentHelpRequest({ - id: helpMsg.requestId, - prompt: helpMsg.prompt, - ...(helpMsg.title ? { title: helpMsg.title } : {}), - ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), - selectors: helpMsg.selectors, - rects: helpMsg.rects, - onContinue: (note: string) => - void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), - onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), - }); + const previousHelp = mountHelpRequest(helpMsg); if (previousHelp && previousHelp.id !== helpMsg.requestId) { void sendHelpFinish(previousHelp.id, "cancelled"); } @@ -407,26 +398,18 @@ export default defineContentScript({ }); } - function mountHelpRequest(helpMsg: Omit): void { - overlays.setAgentHelpRequest({ - id: helpMsg.requestId, - prompt: helpMsg.prompt, - ...(helpMsg.title ? { title: helpMsg.title } : {}), - ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), - selectors: helpMsg.selectors, - rects: helpMsg.rects, - refreshRects: async () => { - const response = (await chrome.runtime.sendMessage({ - type: HELP_QUERY, - })) as HelpQueryResponse | undefined; - return response?.active && response.request?.requestId === helpMsg.requestId - ? response.request.rects - : undefined; - }, - onContinue: (note: string) => - void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), - onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), - }); + function mountHelpRequest(helpMsg: Omit) { + return overlays.setAgentHelpRequest( + createHelpRequestData(helpMsg, { + finish: (requestId, outcome, note) => { + void sendHelpFinish(requestId, outcome, note); + }, + query: () => + chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise< + HelpQueryResponse | undefined + >, + }), + ); } async function queryActiveHelpWithRetry(): Promise { diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index f4d13d2..c7f82b4 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -172,6 +172,58 @@ describe("frame geometry projection", () => { }); }); + it("reuses one frame graph while scrolling and projecting live geometry", async () => { + const getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph, + }; + + await expect( + resolveNodeGeometry( + cdp, + 4, + { + target: { tabId: 4, sessionId: "child-session" }, + frameId: "child", + backendNodeId: 101, + }, + { scrollIntoView: true }, + ), + ).resolves.not.toMatchObject({ code: expect.any(String) }); + expect(getFrameGraph).toHaveBeenCalledOnce(); + }); + it("clips live geometry to same-target iframe ancestors", async () => { const cdp: CdpRunner = { send: vi.fn(async (_tabId, method) => { diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index dcfb5e1..ba2be42 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -188,6 +188,24 @@ async function scrollFrameOwners( return null; } +async function scrollElementWithFrameGraph( + cdp: CdpRunner, + tabId: number, + target: CdpTarget, + backendNodeId: number, + frameId: string | undefined, + graph: CdpFrameGraph | null, +): Promise { + if (frameId) { + if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); + const error = await scrollFrameOwners(cdp, tabId, graph, frameId); + if (error) return error; + } else if (target.sessionId) { + return geometryError("an OOPIF node address requires frameId"); + } + return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId); +} + export async function scrollElementAndFramesIntoView( cdp: CdpRunner, tabId: number, @@ -195,18 +213,8 @@ export async function scrollElementAndFramesIntoView( backendNodeId: number, frameId?: string, ): Promise { - if (frameId) { - const graph = await loadFrameGraph(cdp, tabId); - if (graph) { - const error = await scrollFrameOwners(cdp, tabId, graph, frameId); - if (error) return error; - } else { - return geometryError(`could not resolve frame graph for ${frameId}`); - } - } else if (target.sessionId) { - return geometryError("an OOPIF node address requires frameId"); - } - return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId); + const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; + return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); } function largestRegion(regions: Region): Polygon | null { @@ -226,13 +234,22 @@ export async function resolveNodeGeometry( options: { scrollIntoView?: boolean } = {}, ): Promise { try { + if (address.target.sessionId && !address.frameId) { + return geometryError("an OOPIF node address requires frameId"); + } + const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; + if (address.frameId && !graph) { + return geometryError(`could not resolve frame graph for ${address.frameId}`); + } + if (options.scrollIntoView) { - const scrollError = await scrollElementAndFramesIntoView( + const scrollError = await scrollElementWithFrameGraph( cdp, tabId, address.target, address.backendNodeId, address.frameId, + graph, ); if (scrollError) return scrollError; } @@ -244,14 +261,6 @@ export async function resolveNodeGeometry( ); if (isRpcError(localRegion)) return localRegion; - const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; - if (address.target.sessionId && !address.frameId) { - return geometryError("an OOPIF node address requires frameId"); - } - if (address.frameId && !graph) { - return geometryError(`could not resolve frame graph for ${address.frameId}`); - } - let topVisibleRegions: Region; if (address.frameId && graph) { const projection = await resolveFrameProjection(cdp, graph, address.frameId);