mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): canvas position calculation and projection
This commit is contained in:
1 parent
aeece8d7e4
commit
000b374d0c
30 files changed
+1375
-20
No files matched your search
@@ -0,0 +1,67 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
|
||||
|
||||
function input(): SurfaceCaptureInput {
|
||||
return {
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "main",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: 2,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 100,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
};
|
||||
}
|
||||
|
||||
describe("SurfaceCaptureStore", () => {
|
||||
it("creates metadata-only, short-lived captures and consumes them once", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 500,
|
||||
createId: () => "sc_test",
|
||||
});
|
||||
const capture = store.create(input());
|
||||
|
||||
expect(capture).toMatchObject({
|
||||
id: "sc_test",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 1_500,
|
||||
consumed: false,
|
||||
});
|
||||
expect(capture).not.toHaveProperty("imageBase64");
|
||||
expect(store.consume("sc_test")).toMatchObject({ ok: true });
|
||||
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
|
||||
|
||||
now = 1_600;
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects expired and unknown captures", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 10,
|
||||
createId: () => "sc_expired",
|
||||
});
|
||||
store.create(input());
|
||||
now = 1_010;
|
||||
|
||||
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
|
||||
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
|
||||
});
|
||||
|
||||
it("clears every capture on session cleanup", () => {
|
||||
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
|
||||
store.create(input());
|
||||
store.clear();
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,12 @@
|
||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||
import { RefStore } from "./ref-store";
|
||||
import { SurfaceCaptureStore } from "./surface-capture-store";
|
||||
|
||||
export interface SessionContext {
|
||||
sessionId: string;
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
surfaceCaptures: SurfaceCaptureStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
/**
|
||||
* Tabs explicitly claimed by the agent because it created them. This
|
||||
@@ -210,6 +212,7 @@ export class SessionManager {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
|
||||
borrowedTabs: new Map(),
|
||||
// The home tab is the session's first explicit claim. Every other
|
||||
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export const SURFACE_CAPTURE_TTL_MS = 30_000;
|
||||
|
||||
export interface SurfaceCapture {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
tabId: number;
|
||||
navigationIdentity: string;
|
||||
surface: {
|
||||
ref: string;
|
||||
frameId?: string;
|
||||
backendNodeId: number;
|
||||
observationGeneration: number;
|
||||
};
|
||||
topViewportRect: Rect;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
createdAt: number;
|
||||
expiresAt: number;
|
||||
consumed: boolean;
|
||||
}
|
||||
|
||||
export type SurfaceCaptureInput = Omit<
|
||||
SurfaceCapture,
|
||||
"id" | "createdAt" | "expiresAt" | "consumed"
|
||||
>;
|
||||
|
||||
export type SurfaceCaptureConsumeResult =
|
||||
| { ok: true; capture: SurfaceCapture }
|
||||
| { ok: false; reason: "not_found" | "expired" | "consumed" };
|
||||
|
||||
export interface SurfaceCaptureStoreOptions {
|
||||
now?: () => number;
|
||||
ttlMs?: number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
function randomCaptureId(): string {
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(12));
|
||||
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
||||
}
|
||||
|
||||
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
|
||||
export class SurfaceCaptureStore {
|
||||
private readonly captures = new Map<string, SurfaceCapture>();
|
||||
private readonly now: () => number;
|
||||
private readonly ttlMs: number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(options: SurfaceCaptureStoreOptions = {}) {
|
||||
this.now = options.now ?? Date.now;
|
||||
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
|
||||
this.createId = options.createId ?? randomCaptureId;
|
||||
}
|
||||
|
||||
create(input: SurfaceCaptureInput): SurfaceCapture {
|
||||
this.purgeExpired();
|
||||
const createdAt = this.now();
|
||||
const capture: SurfaceCapture = {
|
||||
...input,
|
||||
id: this.createId(),
|
||||
createdAt,
|
||||
expiresAt: createdAt + this.ttlMs,
|
||||
consumed: false,
|
||||
};
|
||||
this.captures.set(capture.id, capture);
|
||||
return capture;
|
||||
}
|
||||
|
||||
consume(id: string): SurfaceCaptureConsumeResult {
|
||||
const capture = this.captures.get(id);
|
||||
if (!capture) return { ok: false, reason: "not_found" };
|
||||
if (capture.expiresAt <= this.now()) {
|
||||
this.captures.delete(id);
|
||||
return { ok: false, reason: "expired" };
|
||||
}
|
||||
if (capture.consumed) return { ok: false, reason: "consumed" };
|
||||
capture.consumed = true;
|
||||
return { ok: true, capture };
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.captures.clear();
|
||||
}
|
||||
|
||||
size(): number {
|
||||
this.purgeExpired();
|
||||
return this.captures.size;
|
||||
}
|
||||
|
||||
private purgeExpired(): void {
|
||||
const now = this.now();
|
||||
for (const [id, capture] of this.captures) {
|
||||
if (capture.expiresAt <= now) this.captures.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user