mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(dsh-plugin-browserskill): serve observation thumbnails over the plugin's own route
The session-authorized client RPC (readAttachment) refuses images that no session log references — observation frames are plugin-owned runtime data, so every overlay thumbnail failed with ATTACHMENT_NOT_REFERENCED. The host now keeps the full attachment ref per frame and serves bytes through GET /bsk-observation/thumbnail/<id> (verified readImage path); the client loader is a plain fetch of that route.
This commit is contained in:
@@ -40,25 +40,14 @@ async function loadSessionImage(
|
||||
}
|
||||
|
||||
/**
|
||||
* Thumbnail loader for the overlay: the observation overlay is root-scoped,
|
||||
* so borrow the current (or first) bound session for the authorized
|
||||
* attachment read. Without any bound session the frame simply fails and the
|
||||
* overlay keeps the previous one.
|
||||
* Thumbnail loader for the overlay: frames are plugin-owned runtime data
|
||||
* (never referenced by a session log, so the session-authorized RPC refuses
|
||||
* them), served by the host over the plugin's own route.
|
||||
*/
|
||||
function overlayImageLoader(sessions: ISessions): (attachmentId: string) => Promise<string> {
|
||||
return async (attachmentId) => {
|
||||
const list = sessions.list.getSnapshot();
|
||||
const sessionId = list.current ?? list.ids[0];
|
||||
if (sessionId === undefined) throw new Error("no bound session for attachment read");
|
||||
const session = sessions.binding(sessionId)?.session;
|
||||
if (session === undefined) throw new Error(`session "${String(sessionId)}" is not bound`);
|
||||
const result = await session.readAttachment(attachmentId as ImageAttachmentRef["attachmentId"]);
|
||||
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
|
||||
const bytes = Uint8Array.from(result.value.data);
|
||||
return URL.createObjectURL(
|
||||
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
|
||||
);
|
||||
};
|
||||
async function overlayImageLoader(attachmentId: string): Promise<string> {
|
||||
const res = await fetch(`/bsk-observation/thumbnail/${encodeURIComponent(attachmentId)}`);
|
||||
if (!res.ok) throw new Error(`thumbnail fetch failed: ${res.status}`);
|
||||
return URL.createObjectURL(await res.blob());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -81,7 +70,7 @@ export function apply(ctx: ClientContext): void {
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: (url, init) => fetch(url, init),
|
||||
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
||||
loadImage: overlayImageLoader(sessions),
|
||||
loadImage: overlayImageLoader,
|
||||
});
|
||||
ctx.slots.inject("shell.overlay", () =>
|
||||
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
* the event allowlist lives in dsh-api-remotes), so the plugin serves its
|
||||
* observation channel through the documented `webServer` route seam instead:
|
||||
*
|
||||
* GET /bsk-observation/state → { sessions: SessionObservation[] }
|
||||
* GET /bsk-observation/state → { sessions, available }
|
||||
* GET /bsk-observation/events → SSE stream of ObservationEvent
|
||||
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
|
||||
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
|
||||
*
|
||||
* Routes exist only when a webServer service is mounted (web composition);
|
||||
* headless deployments skip registration entirely.
|
||||
@@ -112,6 +113,28 @@ export function registerObservationRoutes(
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "prefix",
|
||||
path: `${ROUTE_BASE}/thumbnail/`,
|
||||
handler: async (req, res) => {
|
||||
if (req.method !== "GET") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
const pathname = decodeURIComponent(new URL(req.url ?? "/", "http://x").pathname);
|
||||
const attachmentId = pathname.slice(`${ROUTE_BASE}/thumbnail/`.length);
|
||||
const frame = await observation.readThumbnail(attachmentId);
|
||||
if (frame === undefined) {
|
||||
sendJson(res, 404, { error: "unknown thumbnail" });
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, {
|
||||
"content-type": frame.mediaType,
|
||||
"cache-control": "no-cache",
|
||||
});
|
||||
res.end(Buffer.from(frame.data));
|
||||
},
|
||||
}),
|
||||
];
|
||||
return () => {
|
||||
for (const dispose of disposers) dispose();
|
||||
|
||||
@@ -58,7 +58,20 @@ interface AttachmentLike {
|
||||
data: Uint8Array;
|
||||
mediaType: string;
|
||||
name?: string;
|
||||
}): Promise<{ attachmentId: unknown }>;
|
||||
}): Promise<ImageAttachmentRefLike>;
|
||||
readImage(
|
||||
ref: ImageAttachmentRefLike,
|
||||
): Promise<{ data: Uint8Array; attachment: ImageAttachmentRefLike }>;
|
||||
}
|
||||
|
||||
/** The attachment reference fields the store verifies reads against. */
|
||||
export interface ImageAttachmentRefLike {
|
||||
attachmentId: unknown;
|
||||
mediaType: string;
|
||||
bytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
/** Injectable clock/scheduler bits for tests. */
|
||||
@@ -89,6 +102,8 @@ export class ObservationService {
|
||||
private readonly captureInFlight = new Set<string>();
|
||||
private readonly captureFailures = new Map<string, number>();
|
||||
private readonly lastActivity = new Map<string, number>();
|
||||
/** attachmentId → full reference (the store requires it for verified reads). */
|
||||
private readonly thumbRefs = new Map<string, ImageAttachmentRefLike>();
|
||||
/** Consecutive capture failures across ALL sessions (daemon-level signal). */
|
||||
private globalFailures = 0;
|
||||
private available = true;
|
||||
@@ -223,6 +238,27 @@ export class ObservationService {
|
||||
return this.deps.runner.killFor(target) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one captured thumbnail back through the store's verified path.
|
||||
* Powers the plugin's own HTTP thumbnail route — frames are plugin-owned
|
||||
* runtime data, never referenced by any session log, so the
|
||||
* session-authorized client RPC cannot serve them.
|
||||
*/
|
||||
async readThumbnail(
|
||||
attachmentId: string,
|
||||
): Promise<{ data: Uint8Array; mediaType: string } | undefined> {
|
||||
const ref = this.thumbRefs.get(attachmentId);
|
||||
if (ref === undefined) return undefined;
|
||||
const attachments = this.deps.ctx.get("attachments") as AttachmentLike | undefined;
|
||||
if (attachments === undefined) return undefined;
|
||||
try {
|
||||
const stored = await attachments.readImage(ref);
|
||||
return { data: stored.data, mediaType: ref.mediaType };
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Tear down all state and timers (plugin dispose). */
|
||||
dispose(): void {
|
||||
this.disposed = true;
|
||||
@@ -231,6 +267,7 @@ export class ObservationService {
|
||||
this.captureInFlight.clear();
|
||||
this.captureFailures.clear();
|
||||
this.lastActivity.clear();
|
||||
this.thumbRefs.clear();
|
||||
this.observations.clear();
|
||||
this.emit({ type: "reset" });
|
||||
this.listeners.clear();
|
||||
@@ -320,6 +357,7 @@ export class ObservationService {
|
||||
this.captureFailures.delete(sessionId);
|
||||
this.globalFailures = 0;
|
||||
this.setAvailable(true);
|
||||
this.thumbRefs.set(String(ref.attachmentId), ref);
|
||||
this.put({ ...entry, thumbnailAttachmentId: String(ref.attachmentId) });
|
||||
} catch {
|
||||
// Silent by design: keep the previous frame, count toward backoff.
|
||||
|
||||
@@ -124,7 +124,20 @@ function setup(opts: {
|
||||
const attachments =
|
||||
"attachments" in opts
|
||||
? opts.attachments
|
||||
: { saveImage: async () => ({ attachmentId: "att-1" }) };
|
||||
: {
|
||||
saveImage: async (input: { data: Uint8Array; name?: string }) => ({
|
||||
attachmentId: "att-1",
|
||||
mediaType: "image/png",
|
||||
bytes: input.data.byteLength,
|
||||
width: 4,
|
||||
height: 4,
|
||||
name: input.name,
|
||||
}),
|
||||
readImage: async (ref: { attachmentId: unknown }) => ({
|
||||
data: new Uint8Array([1, 2, 3]),
|
||||
attachment: ref,
|
||||
}),
|
||||
};
|
||||
const service = new ObservationService({
|
||||
ctx: fakeCtx(attachments),
|
||||
runner,
|
||||
@@ -462,6 +475,22 @@ describe("availability and dead sessions", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("thumbnail read-back", () => {
|
||||
it("serves captured frames through readThumbnail and 404s unknown ids", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId !== undefined);
|
||||
const frame = await service.readThumbnail("att-1");
|
||||
expect(frame?.mediaType).toBe("image/png");
|
||||
expect([...(frame?.data ?? [])]).toEqual([1, 2, 3]);
|
||||
expect(await service.readThumbnail("nope")).toBeUndefined();
|
||||
service.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("actionForLabel", () => {
|
||||
it("maps tool labels onto observation verbs", () => {
|
||||
expect(actionForLabel("navigate")).toBe("navigating");
|
||||
|
||||
Reference in New Issue
Block a user