diff --git a/apps/extension/src/session-manager/__tests__/surface-capture-store.test.ts b/apps/extension/src/session-manager/__tests__/surface-capture-store.test.ts new file mode 100644 index 0000000..fdd3364 --- /dev/null +++ b/apps/extension/src/session-manager/__tests__/surface-capture-store.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store"; + +function input(): SurfaceCaptureInput { + return { + sessionId: "aa11", + tabId: 7, + navigationIdentity: "navigation", + surface: { + ref: "e3", + frameId: "main", + backendNodeId: 99, + observationGeneration: 2, + }, + topViewportRect: { x: 10, y: 20, w: 100, h: 50 }, + imageWidth: 200, + imageHeight: 100, + viewportSignature: "viewport", + frameProjectionSignature: "frame", + }; +} + +describe("SurfaceCaptureStore", () => { + it("creates metadata-only, short-lived captures and consumes them once", () => { + let now = 1_000; + const store = new SurfaceCaptureStore({ + now: () => now, + ttlMs: 500, + createId: () => "sc_test", + }); + const capture = store.create(input()); + + expect(capture).toMatchObject({ + id: "sc_test", + createdAt: 1_000, + expiresAt: 1_500, + consumed: false, + }); + expect(capture).not.toHaveProperty("imageBase64"); + expect(store.consume("sc_test")).toMatchObject({ ok: true }); + expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" }); + + now = 1_600; + expect(store.size()).toBe(0); + }); + + it("rejects expired and unknown captures", () => { + let now = 1_000; + const store = new SurfaceCaptureStore({ + now: () => now, + ttlMs: 10, + createId: () => "sc_expired", + }); + store.create(input()); + now = 1_010; + + expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" }); + expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" }); + }); + + it("clears every capture on session cleanup", () => { + const store = new SurfaceCaptureStore({ createId: () => "sc_clear" }); + store.create(input()); + store.clear(); + expect(store.size()).toBe(0); + }); +}); diff --git a/apps/extension/src/session-manager/manager.ts b/apps/extension/src/session-manager/manager.ts index 2fef0d7..e7c6d9f 100644 --- a/apps/extension/src/session-manager/manager.ts +++ b/apps/extension/src/session-manager/manager.ts @@ -1,10 +1,12 @@ import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window"; import { RefStore } from "./ref-store"; +import { SurfaceCaptureStore } from "./surface-capture-store"; export interface SessionContext { sessionId: string; agentWindowId: number; refStore: RefStore; + surfaceCaptures: SurfaceCaptureStore; borrowedTabs: Map; createdAtMs: number; } @@ -186,6 +188,7 @@ export class SessionManager { sessionId, agentWindowId: windowId, refStore: new RefStore(), + surfaceCaptures: new SurfaceCaptureStore({ now: this.now }), borrowedTabs: new Map(), createdAtMs: this.now(), }; diff --git a/apps/extension/src/session-manager/surface-capture-store.ts b/apps/extension/src/session-manager/surface-capture-store.ts new file mode 100644 index 0000000..10985e0 --- /dev/null +++ b/apps/extension/src/session-manager/surface-capture-store.ts @@ -0,0 +1,100 @@ +import type { Rect } from "@browser-skill/vom"; + +export const SURFACE_CAPTURE_TTL_MS = 30_000; + +export interface SurfaceCapture { + id: string; + sessionId: string; + tabId: number; + navigationIdentity: string; + surface: { + ref: string; + frameId?: string; + backendNodeId: number; + observationGeneration: number; + }; + topViewportRect: Rect; + imageWidth: number; + imageHeight: number; + viewportSignature: string; + frameProjectionSignature: string; + createdAt: number; + expiresAt: number; + consumed: boolean; +} + +export type SurfaceCaptureInput = Omit< + SurfaceCapture, + "id" | "createdAt" | "expiresAt" | "consumed" +>; + +export type SurfaceCaptureConsumeResult = + | { ok: true; capture: SurfaceCapture } + | { ok: false; reason: "not_found" | "expired" | "consumed" }; + +export interface SurfaceCaptureStoreOptions { + now?: () => number; + ttlMs?: number; + createId?: () => string; +} + +function randomCaptureId(): string { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`; +} + +/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */ +export class SurfaceCaptureStore { + private readonly captures = new Map(); + private readonly now: () => number; + private readonly ttlMs: number; + private readonly createId: () => string; + + constructor(options: SurfaceCaptureStoreOptions = {}) { + this.now = options.now ?? Date.now; + this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS; + this.createId = options.createId ?? randomCaptureId; + } + + create(input: SurfaceCaptureInput): SurfaceCapture { + this.purgeExpired(); + const createdAt = this.now(); + const capture: SurfaceCapture = { + ...input, + id: this.createId(), + createdAt, + expiresAt: createdAt + this.ttlMs, + consumed: false, + }; + this.captures.set(capture.id, capture); + return capture; + } + + consume(id: string): SurfaceCaptureConsumeResult { + const capture = this.captures.get(id); + if (!capture) return { ok: false, reason: "not_found" }; + if (capture.expiresAt <= this.now()) { + this.captures.delete(id); + return { ok: false, reason: "expired" }; + } + if (capture.consumed) return { ok: false, reason: "consumed" }; + capture.consumed = true; + return { ok: true, capture }; + } + + clear(): void { + this.captures.clear(); + } + + size(): number { + this.purgeExpired(); + return this.captures.size; + } + + private purgeExpired(): void { + const now = this.now(); + for (const [id, capture] of this.captures) { + if (capture.expiresAt <= now) this.captures.delete(id); + } + } +} diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index cc3b070..0e3a5a2 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -67,6 +67,9 @@ function makeFakeCdp(handlers: Record unknown>) { if (!handler && method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; } + if (!handler && method === "Page.getFrameTree") { + return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } }; + } if (!handler) throw new Error(`unexpected CDP call ${method}`); return handler(params); }); @@ -334,6 +337,11 @@ describe("handleScreenshot", () => { }; expect(clip.clip).toMatchObject({ width: 4096, height: 4096 }); expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096))); + expect(res.capture).toMatchObject({ + surface_ref: "@e5", + coordinate_space: "capture-image-pixel", + }); + expect(ctx.surfaceCaptures.size()).toBe(1); }); it("crops a visual surface screenshot to its observation-time visible region", async () => { @@ -368,6 +376,7 @@ describe("handleScreenshot", () => { clip?: { x: number; y: number; width: number; height: number }; }; expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 }); + expect(res.capture?.id).toMatch(/^sc_/); }); it("returns not_found for unknown ref", async () => { diff --git a/apps/extension/src/tools/__tests__/session.test.ts b/apps/extension/src/tools/__tests__/session.test.ts index 118c18e..14fafdd 100644 --- a/apps/extension/src/tools/__tests__/session.test.ts +++ b/apps/extension/src/tools/__tests__/session.test.ts @@ -252,12 +252,23 @@ describe("handleSessionStop with auto-return", () => { expect(aw.remove).not.toHaveBeenCalled(); }); - it("clears the RefStore before window teardown", async () => { + it("clears refs and Surface captures before window teardown", async () => { const aw = fakeAgentWindow([100]); const sm = new SessionManager({ agentWindow: aw }); const ctx = await sm.start("aa11"); // Insert a fake ref so we can verify clear() ran. ctx.refStore.set("e1", 123, { tabId: 7 }); + ctx.surfaceCaptures.create({ + sessionId: "aa11", + tabId: 7, + navigationIdentity: "navigation", + surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 }, + topViewportRect: { x: 0, y: 0, w: 100, h: 50 }, + imageWidth: 100, + imageHeight: 50, + viewportSignature: "viewport", + frameProjectionSignature: "frame", + }); const state: FakeState = { tabs: new Map(), windowsClosed: new Set(), @@ -266,5 +277,6 @@ describe("handleSessionStop with auto-return", () => { const { tabs, windows } = makeApis(state); await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } }); expect(ctx.refStore.isEmpty()).toBe(true); + expect(ctx.surfaceCaptures.size()).toBe(0); }); }); diff --git a/apps/extension/src/tools/__tests__/snapshot-ref.test.ts b/apps/extension/src/tools/__tests__/snapshot-ref.test.ts index 5c7282f..32b21dd 100644 --- a/apps/extension/src/tools/__tests__/snapshot-ref.test.ts +++ b/apps/extension/src/tools/__tests__/snapshot-ref.test.ts @@ -26,12 +26,14 @@ describe("lookupSnapshotRef", () => { refKey: "e3", kind: "dom", capabilities: ["interact", "screenshot"], + generation: 0, }); expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({ backendNodeId: 1234, refKey: "e3", kind: "dom", capabilities: ["interact", "screenshot"], + generation: 0, }); }); @@ -68,6 +70,7 @@ describe("resolveSnapshotRef", () => { cdpSessionId: "child-session", kind: "dom" as const, capabilities: ["interact", "screenshot"] as const, + generation: 0, }; expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected); expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected); @@ -115,6 +118,7 @@ describe("resolveSnapshotRef", () => { refKey: "e3", kind: "dom", capabilities: ["interact", "screenshot"], + generation: 0, }); }); diff --git a/apps/extension/src/tools/__tests__/surface-coordinate.test.ts b/apps/extension/src/tools/__tests__/surface-coordinate.test.ts new file mode 100644 index 0000000..609ea8f --- /dev/null +++ b/apps/extension/src/tools/__tests__/surface-coordinate.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { + mapImagePointToViewport, + pointInRegion, + sameRect, + surfaceVisibleRect, +} from "../surface-coordinate"; + +describe("surface coordinate mapping", () => { + it("maps capture image pixels through the actual screenshot dimensions", () => { + expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({ + x: 200, + y: 100, + }); + }); + + it("rejects invalid image coordinates and dimensions", () => { + const rect = { x: 0, y: 0, w: 100, h: 50 }; + expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull(); + expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull(); + expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull(); + expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull(); + expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull(); + expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull(); + }); + + it("intersects live and observed visible regions and compares them strictly", () => { + const clipped = surfaceVisibleRect( + { x: 0, y: 0, width: 200, height: 100 }, + { x: 25, y: 30, w: 50, h: 40 }, + ); + expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 }); + expect(sameRect(clipped as NonNullable, { x: 25, y: 30, w: 50, h: 40 })).toBe( + true, + ); + expect(sameRect(clipped as NonNullable, { x: 26, y: 30, w: 50, h: 40 })).toBe( + false, + ); + }); + + it("checks the projected visible region rather than only its bounding box", () => { + const triangle = [ + [ + { x: 0, y: 0 }, + { x: 100, y: 0 }, + { x: 0, y: 100 }, + ], + ]; + expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true); + expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false); + }); +}); diff --git a/apps/extension/src/tools/__tests__/surface-point-action.test.ts b/apps/extension/src/tools/__tests__/surface-point-action.test.ts new file mode 100644 index 0000000..ce5bc80 --- /dev/null +++ b/apps/extension/src/tools/__tests__/surface-point-action.test.ts @@ -0,0 +1,313 @@ +import { describe, expect, it, vi } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import type { CdpRunner } from "../shared"; +import { captureSurfaceEnvironment, handleSurfacePointClick } from "../surface-point-action"; + +function fakeAgentWindow() { + return { + create: vi.fn(async () => 100), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }; +} + +function fakeBrowser() { + let loaderId = "loader-1"; + let pageY = 0; + let quad = [10, 20, 110, 20, 110, 60, 10, 60]; + let dispatchFails = false; + const sent: Array<{ method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + sent.push({ method, params }); + if (method === "Page.getFrameTree") { + return { + frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } }, + } as never; + } + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY }, + cssVisualViewport: { + clientWidth: 800, + clientHeight: 600, + pageX: 0, + pageY, + scale: 1, + }, + } as never; + } + if (method === "DOM.getContentQuads") return { quads: [quad] } as never; + if (method === "Input.dispatchMouseEvent") { + if (dispatchFails) throw new Error("input failed"); + return {} as never; + } + throw new Error(`unexpected CDP call ${method}`); + }) as CdpRunner["send"], + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + return { + cdp, + tabsApi, + sent, + setLoaderId: (value: string) => { + loaderId = value; + }, + setPageY: (value: number) => { + pageY = value; + }, + setQuad: (value: number[]) => { + quad = value; + }, + failDispatch: () => { + dispatchFails = true; + }, + }; +} + +async function setupCapture() { + const manager = new SessionManager({ agentWindow: fakeAgentWindow() }); + const context = await manager.start("aa11"); + context.refStore.set("e3", 99, { + tabId: 4, + kind: "surface", + visibleRect: { x: 10, y: 20, w: 100, h: 40 }, + }); + const browser = fakeBrowser(); + const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined); + if ("code" in environment) throw new Error(environment.message); + const entry = context.refStore.resolveEntry("e3"); + if (!entry) throw new Error("missing ref"); + const capture = context.surfaceCaptures.create({ + sessionId: "aa11", + tabId: 4, + navigationIdentity: environment.navigationIdentity, + surface: { + ref: "e3", + backendNodeId: 99, + observationGeneration: entry.generation, + }, + topViewportRect: { x: 10, y: 20, w: 100, h: 40 }, + imageWidth: 200, + imageHeight: 80, + viewportSignature: environment.viewportSignature, + frameProjectionSignature: environment.frameProjectionSignature, + }); + return { manager, context, browser, capture }; +} + +function pointParams(captureId: string) { + return { + session_id: "aa11", + ref: "@e3", + capture_id: captureId, + image_x: 50, + image_y: 20, + }; +} + +describe("Surface screenshot-bound point click", () => { + it("maps one fresh capture coordinate and dispatches exactly one trusted click", async () => { + const { manager, browser, capture } = await setupCapture(); + const result = await handleSurfacePointClick(manager, pointParams(capture.id), browser); + + if ("code" in result) throw new Error(JSON.stringify(result)); + expect(result).toMatchObject({ + used_ref: "e3", + capture_id: capture.id, + image_x: 50, + image_y: 20, + x: 35, + y: 30, + }); + expect(browser.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength( + 3, + ); + + const repeated = await handleSurfacePointClick(manager, pointParams(capture.id), browser); + expect(repeated).toMatchObject({ + code: "permission_denied", + data: { reason: "surface_capture_consumed" }, + }); + }); + + it("rejects a new observation generation before sending input", async () => { + const { manager, context, browser, capture } = await setupCapture(); + context.refStore.replace([ + [ + "e3", + { + backendNodeId: 99, + tabId: 4, + kind: "surface" as const, + visibleRect: { x: 10, y: 20, w: 100, h: 40 }, + }, + ], + ]); + + const result = await handleSurfacePointClick(manager, pointParams(capture.id), browser); + expect(result).toMatchObject({ + code: "permission_denied", + data: { reason: "surface_capture_stale" }, + }); + expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false); + }); + + it("rejects navigation, scroll, and visible geometry changes", async () => { + for (const mutate of [ + (browser: ReturnType) => browser.setLoaderId("loader-2"), + (browser: ReturnType) => browser.setPageY(20), + (browser: ReturnType) => + browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]), + ]) { + const { manager, browser, capture } = await setupCapture(); + mutate(browser); + const result = await handleSurfacePointClick(manager, pointParams(capture.id), browser); + expect(result).toMatchObject({ + code: "permission_denied", + data: { reason: "surface_capture_stale" }, + }); + expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false); + } + }); + + it("consumes the capture when coordinates or input dispatch fail", async () => { + const invalid = await setupCapture(); + const outside = await handleSurfacePointClick( + invalid.manager, + { ...pointParams(invalid.capture.id), image_x: 200 }, + invalid.browser, + ); + expect(outside).toMatchObject({ + code: "invalid_params", + data: { reason: "surface_coordinate_invalid" }, + }); + expect( + await handleSurfacePointClick( + invalid.manager, + pointParams(invalid.capture.id), + invalid.browser, + ), + ).toMatchObject({ data: { reason: "surface_capture_consumed" } }); + + const failed = await setupCapture(); + failed.browser.failDispatch(); + expect( + await handleSurfacePointClick(failed.manager, pointParams(failed.capture.id), failed.browser), + ).toMatchObject({ code: "cdp_failed" }); + expect( + await handleSurfacePointClick(failed.manager, pointParams(failed.capture.id), failed.browser), + ).toMatchObject({ data: { reason: "surface_capture_consumed" } }); + }); + + it("maps image coordinates through an OOPIF frame projection", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow() }); + const context = await manager.start("aa11"); + context.refStore.set("e3", 99, { + tabId: 4, + frameId: "child", + cdpSessionId: "child-session", + kind: "surface", + visibleRect: { x: 120, y: 110, w: 100, h: 40 }, + }); + const graph = { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 77, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + }; + const sent: Array<{ method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + sent.push({ method, params }); + if (method === "Page.getFrameTree") { + return { + frameTree: { + frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" }, + }, + } as never; + } + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 }, + cssVisualViewport: { + clientWidth: 800, + clientHeight: 600, + pageX: 0, + pageY: 0, + scale: 1, + }, + } as never; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never; + } + if (method === "Input.dispatchMouseEvent") return {} as never; + throw new Error(`unexpected root CDP call ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "DOM.getContentQuads") { + return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never; + } + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => graph), + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn(async () => ({ id: 4, windowId: 100, active: true }) as chrome.tabs.Tab), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + const environment = await captureSurfaceEnvironment(cdp, 4, "child"); + if ("code" in environment) throw new Error(environment.message); + const entry = context.refStore.resolveEntry("e3"); + if (!entry) throw new Error("missing ref"); + const capture = context.surfaceCaptures.create({ + sessionId: "aa11", + tabId: 4, + navigationIdentity: environment.navigationIdentity, + surface: { + ref: "e3", + frameId: "child", + backendNodeId: 99, + observationGeneration: entry.generation, + }, + topViewportRect: { x: 120, y: 110, w: 100, h: 40 }, + imageWidth: 200, + imageHeight: 80, + viewportSignature: environment.viewportSignature, + frameProjectionSignature: environment.frameProjectionSignature, + }); + + const result = await handleSurfacePointClick( + manager, + { + session_id: "aa11", + ref: "@e3", + capture_id: capture.id, + image_x: 100, + image_y: 40, + }, + { cdp, tabsApi }, + ); + + if ("code" in result) throw new Error(JSON.stringify(result)); + expect(result).toMatchObject({ x: 170, y: 130 }); + expect(sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3); + }); +}); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 350dc95..0f29ec9 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -43,6 +43,7 @@ import { resolveTargetTab, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; +import { handleSurfacePointClick } from "./surface-point-action"; export interface InteractionDeps { cdp: CdpRunner; @@ -224,6 +225,11 @@ export async function handleClick( params: ClickParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { + const hasSurfacePointParams = + params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined; + if (hasSurfacePointParams) { + return handleSurfacePointClick(manager, params, deps); + } const ctxOrErr = lookupSession(manager, params, "click"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index f529048..728e2ba 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -48,6 +48,8 @@ import { type ToolEffect, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; +import { surfaceVisibleRect } from "./surface-coordinate"; +import { captureSurfaceEnvironment } from "./surface-point-action"; import { type CapturedNode, type CapturedViewModel, @@ -194,7 +196,9 @@ async function captureElementScreenshot( visualSurface = false, observedVisibleRect?: Rect, signal?: AbortSignal, -): Promise<{ image_base64: string; width: number; height: number } | RpcError> { +): Promise< + { image_base64: string; width: number; height: number; topViewportRect: Rect } | RpcError +> { if (signal?.aborted) return cancelled("screenshot"); const geometry = await resolveNodeGeometry( cdp, @@ -204,24 +208,19 @@ async function captureElementScreenshot( ); if (isRpcError(geometry)) return geometry; if (signal?.aborted) return cancelled("screenshot"); - const liveRect = geometry.topBounds; - const rect = - visualSurface && observedVisibleRect - ? (() => { - const x = Math.max(liveRect.x, observedVisibleRect.x); - const y = Math.max(liveRect.y, observedVisibleRect.y); - const right = Math.min( - liveRect.x + liveRect.width, - observedVisibleRect.x + observedVisibleRect.w, - ); - const bottom = Math.min( - liveRect.y + liveRect.height, - observedVisibleRect.y + observedVisibleRect.h, - ); - return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null; - })() - : liveRect; - if (!rect) return { code: "permission_denied", message: "surface is no longer visible" }; + const topViewportRect = surfaceVisibleRect( + geometry.topBounds, + visualSurface ? observedVisibleRect : undefined, + ); + if (!topViewportRect) { + return { code: "permission_denied", message: "surface is no longer visible" }; + } + const rect = { + x: topViewportRect.x, + y: topViewportRect.y, + width: topViewportRect.w, + height: topViewportRect.h, + }; const scale = visualSurface ? Math.min( 1, @@ -251,7 +250,7 @@ async function captureElementScreenshot( width: Math.round(rect.width), height: Math.round(rect.height), }; - return { image_base64, width: dims.width, height: dims.height }; + return { image_base64, width: dims.width, height: dims.height, topViewportRect }; } catch (err) { return { code: "cdp_failed", @@ -374,12 +373,46 @@ export async function handleScreenshot( ); if (isRpcError(captured)) return captured; if (signal?.aborted) return cancelled("screenshot"); + const capture = + node.kind === "surface" + ? await (async () => { + const environment = await captureSurfaceEnvironment(cdp, target.tabId, node.frameId); + if (isRpcError(environment)) return environment; + return ctx.surfaceCaptures.create({ + sessionId: ctx.sessionId, + tabId: target.tabId, + navigationIdentity: environment.navigationIdentity, + surface: { + ref: node.refKey, + ...(node.frameId ? { frameId: node.frameId } : {}), + backendNodeId: node.backendNodeId, + observationGeneration: node.generation, + }, + topViewportRect: captured.topViewportRect, + imageWidth: captured.width, + imageHeight: captured.height, + viewportSignature: environment.viewportSignature, + frameProjectionSignature: environment.frameProjectionSignature, + }); + })() + : null; + if (capture && isRpcError(capture)) return capture; return withShotDialogs({ image_base64: captured.image_base64, width: captured.width, height: captured.height, format: "png", tab_id: target.tabId, + ...(capture + ? { + capture: { + id: capture.id, + surface_ref: `@${capture.surface.ref}`, + coordinate_space: "capture-image-pixel" as const, + expires_at: capture.expiresAt, + }, + } + : {}), }); } diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts index 9bd1b27..b4a9e9b 100644 --- a/apps/extension/src/tools/session.ts +++ b/apps/extension/src/tools/session.ts @@ -243,6 +243,7 @@ export async function handleSessionStop( // Step 2: clear the per-session RefStore (review M6/M7 parity). ctx.refStore.clear(); + ctx.surfaceCaptures.clear(); clearRecordingForSession(params.session_id); diff --git a/apps/extension/src/tools/snapshot-ref.ts b/apps/extension/src/tools/snapshot-ref.ts index 19e3a25..a1c5a10 100644 --- a/apps/extension/src/tools/snapshot-ref.ts +++ b/apps/extension/src/tools/snapshot-ref.ts @@ -16,6 +16,7 @@ export interface SnapshotRefLookup { visibleRect?: Rect; kind: RefTargetKind; capabilities: RefCapability[]; + generation: number; } function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) { @@ -44,6 +45,7 @@ export function lookupSnapshotRef( ...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}), kind: entry.kind, capabilities: entry.capabilities, + generation: entry.generation, }; } diff --git a/apps/extension/src/tools/surface-coordinate.ts b/apps/extension/src/tools/surface-coordinate.ts new file mode 100644 index 0000000..ced1a90 --- /dev/null +++ b/apps/extension/src/tools/surface-coordinate.ts @@ -0,0 +1,88 @@ +import type { Rect } from "@browser-skill/vom"; +import type { Point, Region, ViewportRect } from "./geometry"; + +const RECT_EPSILON = 0.01; + +export function surfaceVisibleRect( + liveRect: ViewportRect, + observedRect: Rect | undefined, +): Rect | null { + const observed = observedRect ?? { + x: liveRect.x, + y: liveRect.y, + w: liveRect.width, + h: liveRect.height, + }; + const x = Math.max(liveRect.x, observed.x); + const y = Math.max(liveRect.y, observed.y); + const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w); + const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h); + return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null; +} + +export function sameRect(a: Rect, b: Rect): boolean { + return ( + Math.abs(a.x - b.x) <= RECT_EPSILON && + Math.abs(a.y - b.y) <= RECT_EPSILON && + Math.abs(a.w - b.w) <= RECT_EPSILON && + Math.abs(a.h - b.h) <= RECT_EPSILON + ); +} + +export function mapImagePointToViewport( + rect: Rect, + imageWidth: number, + imageHeight: number, + imageX: number, + imageY: number, +): Point | null { + if ( + !Number.isFinite(imageX) || + !Number.isFinite(imageY) || + !Number.isSafeInteger(imageWidth) || + !Number.isSafeInteger(imageHeight) || + imageWidth <= 0 || + imageHeight <= 0 || + imageX < 0 || + imageY < 0 || + imageX >= imageWidth || + imageY >= imageHeight || + !Number.isFinite(rect.x) || + !Number.isFinite(rect.y) || + !Number.isFinite(rect.w) || + !Number.isFinite(rect.h) || + rect.w <= 0 || + rect.h <= 0 + ) { + return null; + } + return { + x: rect.x + (imageX / imageWidth) * rect.w, + y: rect.y + (imageY / imageHeight) * rect.h, + }; +} + +function pointInPolygon(point: Point, polygon: Point[]): boolean { + for (let index = 0; index < polygon.length; index += 1) { + const a = polygon[index]; + const b = polygon[(index + 1) % polygon.length]; + const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x); + const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9; + const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9; + if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true; + } + let inside = false; + for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) { + const a = polygon[index]; + const b = polygon[previous]; + const intersects = + a.y > point.y !== b.y > point.y && + point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x; + if (intersects) inside = !inside; + } + return inside; +} + +export function pointInRegion(point: Point, region: Region): boolean { + return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon)); +} diff --git a/apps/extension/src/tools/surface-point-action.ts b/apps/extension/src/tools/surface-point-action.ts new file mode 100644 index 0000000..e61b661 --- /dev/null +++ b/apps/extension/src/tools/surface-point-action.ts @@ -0,0 +1,320 @@ +import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; +import type { SessionManager } from "@/session-manager/manager"; +import type { ClickParams, ClickResult, RpcError } from "@/transport/types"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { rpcError } from "./errors"; +import { resolveNodeGeometry } from "./frame-geometry"; +import { + type CdpRunner, + type ChromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + normaliseRef, + resolveTargetTab, +} from "./shared"; +import { resolveSnapshotRef } from "./snapshot-ref"; +import { + mapImagePointToViewport, + pointInRegion, + sameRect, + surfaceVisibleRect, +} from "./surface-coordinate"; + +export interface SurfaceCaptureEnvironment { + navigationIdentity: string; + viewportSignature: string; + frameProjectionSignature: string; +} + +export interface SurfacePointActionDeps { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + signal?: AbortSignal; + bypassOverlay?: (tabId: number, enabled: boolean) => Promise; +} + +function stale(message: string): RpcError { + return rpcError("permission_denied", "surface_capture_stale", message); +} + +function finiteObject(value: unknown): unknown { + if (Array.isArray(value)) return value.map(finiteObject); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value) + .filter( + ([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)), + ) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, entry]) => [key, finiteObject(entry)]), + ); +} + +function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null { + if (!frameId) return "top"; + const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame])); + const path: Array<{ + frameId: string; + parentFrameId?: string; + ownerBackendNodeId?: number; + targetSessionId?: string; + }> = []; + const seen = new Set(); + let current = frames.get(frameId); + while (current) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + path.push({ + frameId: current.frameId, + ...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}), + ...(current.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: current.ownerBackendNodeId } + : {}), + ...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}), + }); + if (!current.parentFrameId) return JSON.stringify(path); + const parent = frames.get(current.parentFrameId); + if (!parent) return null; + current = parent; + } + return null; +} + +export async function captureSurfaceEnvironment( + cdp: CdpRunner, + tabId: number, + frameId: string | undefined, +): Promise { + try { + const [tree, metrics, graph] = await Promise.all([ + cdp.send<{ + frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } }; + }>(tabId, "Page.getFrameTree", {}), + cdp.send>(tabId, "Page.getLayoutMetrics", {}), + frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null), + ]); + const root = tree.frameTree?.frame; + if (!root?.id || !root.loaderId) { + return { code: "cdp_failed", message: "could not identify the current navigation" }; + } + const frameProjectionSignature = frameId + ? graph + ? framePathSignature(graph, frameId) + : null + : "top"; + if (!frameProjectionSignature) { + return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` }; + } + return { + navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }), + viewportSignature: JSON.stringify( + finiteObject({ + cssLayoutViewport: metrics.cssLayoutViewport, + cssVisualViewport: metrics.cssVisualViewport, + }), + ), + frameProjectionSignature, + }; + } catch (error) { + return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; + } +} + +function validatePointParams(params: ClickParams): RpcError | null { + if (!params.capture_id || typeof params.capture_id !== "string") { + return { code: "invalid_params", message: "surface point click requires capture_id" }; + } + if (typeof params.image_x !== "number" || !Number.isFinite(params.image_x)) { + return { code: "invalid_params", message: "image_x must be a finite number" }; + } + if (typeof params.image_y !== "number" || !Number.isFinite(params.image_y)) { + return { code: "invalid_params", message: "image_y must be a finite number" }; + } + if (!params.ref || params.selector) { + return { + code: "invalid_params", + message: "surface point click requires one Surface ref and does not accept a selector", + }; + } + if ((params.button ?? "left") !== "left" || (params.click_count ?? 1) !== 1) { + return { + code: "invalid_params", + message: "surface point click currently supports one left click only", + }; + } + if (params.modifiers && params.modifiers.length > 0) { + return { code: "invalid_params", message: "surface point click does not accept modifiers" }; + } + return null; +} + +function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError { + const code = reason === "not_found" ? "not_found" : "permission_denied"; + return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`); +} + +function aborted(signal: AbortSignal | undefined): RpcError | null { + return signal?.aborted ? { code: "cancelled", message: "surface point click aborted" } : null; +} + +export async function handleSurfacePointClick( + manager: SessionManager, + params: ClickParams, + deps: SurfacePointActionDeps, +): Promise { + const invalid = validatePointParams(params); + if (invalid) return invalid; + const ctxOrError = lookupSession(manager, params, "click"); + if (isRpcError(ctxOrError)) return ctxOrError; + const ctx = ctxOrError; + const earlyAbort = aborted(deps.signal); + if (earlyAbort) return earlyAbort; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "click"); + if (denied) return denied; + + const consumed = ctx.surfaceCaptures.consume(params.capture_id as string); + if (!consumed.ok) return captureConsumeError(consumed.reason); + const capture = consumed.capture; + const postConsumeAbort = aborted(deps.signal); + if (postConsumeAbort) return postConsumeAbort; + + if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) { + return stale("surface capture belongs to a different session or tab"); + } + if (normaliseRef(params.ref as string) !== capture.surface.ref) { + return stale("surface ref does not match the capture"); + } + const node = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "screenshot"); + if (isRpcError(node)) return stale(node.message); + if ( + node.kind !== "surface" || + node.backendNodeId !== capture.surface.backendNodeId || + node.frameId !== capture.surface.frameId || + node.generation !== capture.surface.observationGeneration + ) { + return stale("surface observation generation or node identity changed"); + } + + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + try { + await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); + } catch (error) { + return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; + } + const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId); + if (isRpcError(environment)) return environment; + if ( + environment.navigationIdentity !== capture.navigationIdentity || + environment.viewportSignature !== capture.viewportSignature || + environment.frameProjectionSignature !== capture.frameProjectionSignature + ) { + return stale("navigation, viewport, zoom, scroll, or frame projection changed"); + } + + const geometry = await resolveNodeGeometry( + deps.cdp, + target.tabId, + { + target: { + tabId: target.tabId, + ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), + }, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: false }, + ); + if (isRpcError(geometry)) return stale(geometry.message); + const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect); + if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) { + return stale("surface visible region changed"); + } + const point = mapImagePointToViewport( + capture.topViewportRect, + capture.imageWidth, + capture.imageHeight, + params.image_x as number, + params.image_y as number, + ); + if (!point || !pointInRegion(point, geometry.topVisibleRegions)) { + return rpcError( + "invalid_params", + "surface_coordinate_invalid", + "image coordinate is outside the captured Surface", + ); + } + + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + let bypassEnabled = false; + let pressed = false; + const release = () => + deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseReleased", + x: point.x, + y: point.y, + button: "left", + clickCount: 1, + modifiers: 0, + }); + try { + if (deps.bypassOverlay) { + await deps.bypassOverlay(target.tabId, true); + bypassEnabled = true; + } + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + modifiers: 0, + }); + const beforePressAbort = aborted(deps.signal); + if (beforePressAbort) return beforePressAbort; + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mousePressed", + x: point.x, + y: point.y, + button: "left", + clickCount: 1, + modifiers: 0, + }); + pressed = true; + const afterPressAbort = aborted(deps.signal); + if (afterPressAbort) { + await release(); + pressed = false; + return afterPressAbort; + } + await release(); + pressed = false; + } catch (error) { + if (pressed) { + try { + await release(); + } catch (releaseError) { + console.debug("[bsk surface-point] best-effort mouse release failed", releaseError); + } + } + return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; + } finally { + if (bypassEnabled && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (error) { + console.debug("[bsk surface-point] overlay bypass disable failed", error); + } + } + } + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: capture.surface.ref, + x: point.x, + y: point.y, + capture_id: capture.id, + image_x: params.image_x, + image_y: params.image_y, + }); +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index f9e5ffb..9ec464a 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -34,6 +34,11 @@ export type RpcErrorReason = | "restricted_tab_url" | "borrow_conflict" | "screenshot_capture_failed" + | "surface_capture_not_found" + | "surface_capture_expired" + | "surface_capture_consumed" + | "surface_capture_stale" + | "surface_coordinate_invalid" | "cleanup_failed"; export interface RpcErrorData { @@ -299,9 +304,17 @@ export interface ScreenshotResult { height: number; format: string; tab_id: number; + capture?: SurfaceCaptureInfo; dialogs?: JavaScriptDialogInfo[]; } +export interface SurfaceCaptureInfo { + id: string; + surface_ref: string; + coordinate_space: "capture-image-pixel"; + expires_at: number; +} + export interface SnapshotParams { session_id: string; tab_id?: number; @@ -415,6 +428,9 @@ export interface ClickParams { click_count?: number; modifiers?: KeyModifier[]; timeout_ms?: number; + capture_id?: string; + image_x?: number; + image_y?: number; } export interface ClickResult { @@ -423,6 +439,9 @@ export interface ClickResult { used_selector?: string; x: number; y: number; + capture_id?: string; + image_x?: number; + image_y?: number; dialogs?: JavaScriptDialogInfo[]; } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e098730..4d5ce56 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -80,7 +80,15 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. -When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually inspect image output, use `bsk screenshot --ref @eN --session ` to obtain the visible crop. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs. +When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability. + +If you can actually inspect image output, use `bsk screenshot --ref @eN --session ` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command: + +```bash +bsk click @eN --capture --image-x --image-y --session +``` + +Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them. ## Observation priority @@ -176,7 +184,7 @@ bsk emulate --session --off | `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | | `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | +| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id | ### Console & network debugging (read-only; require `--session`) @@ -202,7 +210,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | +| `bsk click ` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` | | `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) |