feat(vom): canvas position calculation and projection

packages/dsh-plugin-browserskill/src/tools.ts

@@ -563,7 +563,8 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       name: "interact.click",
       description:
         "Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
-        "last browser_inspect observation, or a CSS selector.",
+        "last browser_inspect observation, or a CSS selector. A visual Surface requires the " +
+        "captureId and image coordinates returned by browser_inspect action=screenshot.",
       parameters: {
         target: {
           type: "string",
@@ -580,6 +581,18 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
           type: "integer",
           description: "Number of consecutive presses (double-click = 2).",
         },
+        captureId: {
+          type: "string",
+          description: "Short-lived Surface capture id returned by browser_screenshot.",
+        },
+        imageX: {
+          type: "number",
+          description: "X coordinate in capture-image pixels; requires captureId and imageY.",
+        },
+        imageY: {
+          type: "number",
+          description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
+        },
       },
       output: {
         schema: {
@@ -590,6 +603,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             tabId: { type: "integer", required: true },
             x: { type: "number", required: true },
             y: { type: "number", required: true },
+            captureId: { type: "string" },
           },
         },
         render: (_args, value) => [
@@ -601,17 +615,41 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
       },
       async execute(args, exec) {
         if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
+        const pointArgCount =
+          Number(args.captureId !== undefined) +
+          Number(args.imageX !== undefined) +
+          Number(args.imageY !== undefined);
+        if (pointArgCount !== 0 && pointArgCount !== 3) {
+          throw new Error("captureId, imageX, and imageY must be given together");
+        }
         const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
         const cmdArgs = ["click", "--session", sessionId];
         if (args.button !== undefined) cmdArgs.push("--button", args.button);
         if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
+        if (args.captureId !== undefined) {
+          cmdArgs.push(
+            "--capture",
+            args.captureId,
+            "--image-x",
+            String(args.imageX),
+            "--image-y",
+            String(args.imageY),
+          );
+        }
         cmdArgs.push(args.target);
         const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
           tab_id: number;
           x: number;
           y: number;
+          capture_id?: string;
+        };
+        return {
+          session: sessionId,
+          tabId: reply.tab_id,
+          x: reply.x,
+          y: reply.y,
+          ...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
         };
-        return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
       },
       presentCall: (args) => ({
         card: "terminal",
@@ -778,6 +816,8 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             width: { type: "integer", required: true },
             height: { type: "integer", required: true },
             byteSize: { type: "integer", required: true },
+            captureId: { type: "string" },
+            captureExpiresAt: { type: "integer" },
             image: {
               type: "object",
               additionalProperties: false,
@@ -801,7 +841,11 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             value.image !== undefined
               ? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
               : `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
-          const blocks: ContentBlock[] = [{ type: "text", text }];
+          const captureText =
+            value.captureId !== undefined
+              ? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
+              : "";
+          const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
           if (value.image !== undefined) {
             blocks.push({
               type: "image",
@@ -840,6 +884,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             height: number;
             path: string;
             byte_size: number;
+            capture?: { id: string; expires_at: number };
           };
           writtenPath = reply.path;
           const data = await readFile(reply.path);
@@ -853,6 +898,9 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
             width: reply.width,
             height: reply.height,
             byteSize: reply.byte_size,
+            ...(reply.capture !== undefined
+              ? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
+              : {}),
             ...(ref !== undefined
               ? {
                   image: {

packages/dsh-plugin-browserskill/tests/tools.test.ts

@@ -499,6 +499,41 @@ describe("interaction tools", () => {
     ]);
   });
 
+  it("interact.click forwards a complete Surface capture coordinate transaction", async () => {
+    const { tools, calls } = setup({
+      ...responses,
+      click: {
+        tab_id: 7,
+        used_ref: "e1",
+        x: 110,
+        y: 70,
+        capture_id: "sc_test",
+        image_x: 200,
+        image_y: 100,
+      },
+    });
+    await startSession(tools);
+    const click = tools.get("interact.click");
+    const value = await click?.execute(
+      { target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
+      makeExec(),
+    );
+
+    expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
+    expect(calls[1].args).toEqual([
+      "click",
+      "--session",
+      "s1",
+      "--capture",
+      "sc_test",
+      "--image-x",
+      "200",
+      "--image-y",
+      "100",
+      "@e1",
+    ]);
+  });
+
   it("interact.fill passes value and --no-clear", async () => {
     const { tools, calls } = setup(responses);
     await startSession(tools);
@@ -923,6 +958,32 @@ describe("inspect.screenshot", () => {
     expect(rendered?.every((block) => block.type === "text")).toBe(true);
   });
 
+  it("returns Surface capture metadata alongside the screenshot", async () => {
+    const path = pngFile();
+    const { tools } = setup({
+      "session start": START_REPLY("s1"),
+      screenshot: {
+        tab_id: 7,
+        width: 800,
+        height: 600,
+        format: "png",
+        path,
+        byte_size: 8,
+        capture: { id: "sc_test", expires_at: 1700000030000 },
+      },
+    });
+    await startSession(tools);
+    const screenshot = tools.get("browser_screenshot");
+    const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
+
+    expect(value).toMatchObject({
+      captureId: "sc_test",
+      captureExpiresAt: 1700000030000,
+    });
+    const rendered = screenshot?.output.render({ session: "s1" }, value as never);
+    expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
+  });
+
   it("commits the image through the attachment store on an image-capable route", async () => {
     const path = pngFile();
     const services = {