feat(vom): canvas position calculation and projection

This commit is contained in:
Ljy-0827 committed 2026-09-08 11:15:53 +08:00
1 parent aeece8d7e4
commit 000b374d0c
30 files changed
+1375 -20

No files matched your search

@@ -0,0 +1,67 @@
import { describe, expect, it } from "vitest";
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
function input(): SurfaceCaptureInput {
return {
sessionId: "aa11",
tabId: 7,
navigationIdentity: "navigation",
surface: {
ref: "e3",
frameId: "main",
backendNodeId: 99,
observationGeneration: 2,
},
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
imageWidth: 200,
imageHeight: 100,
viewportSignature: "viewport",
frameProjectionSignature: "frame",
};
}
describe("SurfaceCaptureStore", () => {
it("creates metadata-only, short-lived captures and consumes them once", () => {
let now = 1_000;
const store = new SurfaceCaptureStore({
now: () => now,
ttlMs: 500,
createId: () => "sc_test",
});
const capture = store.create(input());
expect(capture).toMatchObject({
id: "sc_test",
createdAt: 1_000,
expiresAt: 1_500,
consumed: false,
});
expect(capture).not.toHaveProperty("imageBase64");
expect(store.consume("sc_test")).toMatchObject({ ok: true });
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
now = 1_600;
expect(store.size()).toBe(0);
});
it("rejects expired and unknown captures", () => {
let now = 1_000;
const store = new SurfaceCaptureStore({
now: () => now,
ttlMs: 10,
createId: () => "sc_expired",
});
store.create(input());
now = 1_010;
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
});
it("clears every capture on session cleanup", () => {
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
store.create(input());
store.clear();
expect(store.size()).toBe(0);
});
});
@@ -1,10 +1,12 @@
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
import { RefStore } from "./ref-store";
import { SurfaceCaptureStore } from "./surface-capture-store";
export interface SessionContext {
sessionId: string;
agentWindowId: number;
refStore: RefStore;
surfaceCaptures: SurfaceCaptureStore;
borrowedTabs: Map<number, BorrowedTab>;
/**
* Tabs explicitly claimed by the agent because it created them. This
@@ -210,6 +212,7 @@ export class SessionManager {
sessionId,
agentWindowId: windowId,
refStore: new RefStore(),
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
borrowedTabs: new Map(),
// The home tab is the session's first explicit claim. Every other
// tab remains free until `tab_create` or `tab_borrow` identifies it
@@ -0,0 +1,100 @@
import type { Rect } from "@browser-skill/vom";
export const SURFACE_CAPTURE_TTL_MS = 30_000;
export interface SurfaceCapture {
id: string;
sessionId: string;
tabId: number;
navigationIdentity: string;
surface: {
ref: string;
frameId?: string;
backendNodeId: number;
observationGeneration: number;
};
topViewportRect: Rect;
imageWidth: number;
imageHeight: number;
viewportSignature: string;
frameProjectionSignature: string;
createdAt: number;
expiresAt: number;
consumed: boolean;
}
export type SurfaceCaptureInput = Omit<
SurfaceCapture,
"id" | "createdAt" | "expiresAt" | "consumed"
>;
export type SurfaceCaptureConsumeResult =
| { ok: true; capture: SurfaceCapture }
| { ok: false; reason: "not_found" | "expired" | "consumed" };
export interface SurfaceCaptureStoreOptions {
now?: () => number;
ttlMs?: number;
createId?: () => string;
}
function randomCaptureId(): string {
const bytes = crypto.getRandomValues(new Uint8Array(12));
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
}
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
export class SurfaceCaptureStore {
private readonly captures = new Map<string, SurfaceCapture>();
private readonly now: () => number;
private readonly ttlMs: number;
private readonly createId: () => string;
constructor(options: SurfaceCaptureStoreOptions = {}) {
this.now = options.now ?? Date.now;
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
this.createId = options.createId ?? randomCaptureId;
}
create(input: SurfaceCaptureInput): SurfaceCapture {
this.purgeExpired();
const createdAt = this.now();
const capture: SurfaceCapture = {
...input,
id: this.createId(),
createdAt,
expiresAt: createdAt + this.ttlMs,
consumed: false,
};
this.captures.set(capture.id, capture);
return capture;
}
consume(id: string): SurfaceCaptureConsumeResult {
const capture = this.captures.get(id);
if (!capture) return { ok: false, reason: "not_found" };
if (capture.expiresAt <= this.now()) {
this.captures.delete(id);
return { ok: false, reason: "expired" };
}
if (capture.consumed) return { ok: false, reason: "consumed" };
capture.consumed = true;
return { ok: true, capture };
}
clear(): void {
this.captures.clear();
}
size(): number {
this.purgeExpired();
return this.captures.size;
}
private purgeExpired(): void {
const now = this.now();
for (const [id, capture] of this.captures) {
if (capture.expiresAt <= now) this.captures.delete(id);
}
}
}