fix(dsh-plugin-browserskill): serve observation thumbnails over the plugin's own route

The session-authorized client RPC (readAttachment) refuses images that
no session log references — observation frames are plugin-owned runtime
data, so every overlay thumbnail failed with ATTACHMENT_NOT_REFERENCED.
The host now keeps the full attachment ref per frame and serves bytes
through GET /bsk-observation/thumbnail/<id> (verified readImage path);
the client loader is a plain fetch of that route.
This commit is contained in:
BB-fat
2026-08-14 06:49:15 +00:00
parent bc5d9610a3
commit 461d204ca3
4 changed files with 101 additions and 22 deletions
@@ -40,25 +40,14 @@ async function loadSessionImage(
}
/**
* Thumbnail loader for the overlay: the observation overlay is root-scoped,
* so borrow the current (or first) bound session for the authorized
* attachment read. Without any bound session the frame simply fails and the
* overlay keeps the previous one.
* Thumbnail loader for the overlay: frames are plugin-owned runtime data
* (never referenced by a session log, so the session-authorized RPC refuses
* them), served by the host over the plugin's own route.
*/
function overlayImageLoader(sessions: ISessions): (attachmentId: string) => Promise<string> {
return async (attachmentId) => {
const list = sessions.list.getSnapshot();
const sessionId = list.current ?? list.ids[0];
if (sessionId === undefined) throw new Error("no bound session for attachment read");
const session = sessions.binding(sessionId)?.session;
if (session === undefined) throw new Error(`session "${String(sessionId)}" is not bound`);
const result = await session.readAttachment(attachmentId as ImageAttachmentRef["attachmentId"]);
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
const bytes = Uint8Array.from(result.value.data);
return URL.createObjectURL(
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
);
};
async function overlayImageLoader(attachmentId: string): Promise<string> {
const res = await fetch(`/bsk-observation/thumbnail/${encodeURIComponent(attachmentId)}`);
if (!res.ok) throw new Error(`thumbnail fetch failed: ${res.status}`);
return URL.createObjectURL(await res.blob());
}
/**
@@ -81,7 +70,7 @@ export function apply(ctx: ClientContext): void {
const store = new ObservationClientStore({
fetchFn: (url, init) => fetch(url, init),
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
loadImage: overlayImageLoader(sessions),
loadImage: overlayImageLoader,
});
ctx.slots.inject("shell.overlay", () =>
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
@@ -5,9 +5,10 @@
* the event allowlist lives in dsh-api-remotes), so the plugin serves its
* observation channel through the documented `webServer` route seam instead:
*
* GET /bsk-observation/state → { sessions: SessionObservation[] }
* GET /bsk-observation/state → { sessions, available }
* GET /bsk-observation/events → SSE stream of ObservationEvent
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
*
* Routes exist only when a webServer service is mounted (web composition);
* headless deployments skip registration entirely.
@@ -112,6 +113,28 @@ export function registerObservationRoutes(
});
},
}),
webServer.register({
kind: "prefix",
path: `${ROUTE_BASE}/thumbnail/`,
handler: async (req, res) => {
if (req.method !== "GET") {
sendJson(res, 405, { error: "method not allowed" });
return;
}
const pathname = decodeURIComponent(new URL(req.url ?? "/", "http://x").pathname);
const attachmentId = pathname.slice(`${ROUTE_BASE}/thumbnail/`.length);
const frame = await observation.readThumbnail(attachmentId);
if (frame === undefined) {
sendJson(res, 404, { error: "unknown thumbnail" });
return;
}
res.writeHead(200, {
"content-type": frame.mediaType,
"cache-control": "no-cache",
});
res.end(Buffer.from(frame.data));
},
}),
];
return () => {
for (const dispose of disposers) dispose();
@@ -58,7 +58,20 @@ interface AttachmentLike {
data: Uint8Array;
mediaType: string;
name?: string;
}): Promise<{ attachmentId: unknown }>;
}): Promise<ImageAttachmentRefLike>;
readImage(
ref: ImageAttachmentRefLike,
): Promise<{ data: Uint8Array; attachment: ImageAttachmentRefLike }>;
}
/** The attachment reference fields the store verifies reads against. */
export interface ImageAttachmentRefLike {
attachmentId: unknown;
mediaType: string;
bytes: number;
width: number;
height: number;
name?: string;
}
/** Injectable clock/scheduler bits for tests. */
@@ -89,6 +102,8 @@ export class ObservationService {
private readonly captureInFlight = new Set<string>();
private readonly captureFailures = new Map<string, number>();
private readonly lastActivity = new Map<string, number>();
/** attachmentId → full reference (the store requires it for verified reads). */
private readonly thumbRefs = new Map<string, ImageAttachmentRefLike>();
/** Consecutive capture failures across ALL sessions (daemon-level signal). */
private globalFailures = 0;
private available = true;
@@ -223,6 +238,27 @@ export class ObservationService {
return this.deps.runner.killFor(target) > 0;
}
/**
* Read one captured thumbnail back through the store's verified path.
* Powers the plugin's own HTTP thumbnail route — frames are plugin-owned
* runtime data, never referenced by any session log, so the
* session-authorized client RPC cannot serve them.
*/
async readThumbnail(
attachmentId: string,
): Promise<{ data: Uint8Array; mediaType: string } | undefined> {
const ref = this.thumbRefs.get(attachmentId);
if (ref === undefined) return undefined;
const attachments = this.deps.ctx.get("attachments") as AttachmentLike | undefined;
if (attachments === undefined) return undefined;
try {
const stored = await attachments.readImage(ref);
return { data: stored.data, mediaType: ref.mediaType };
} catch {
return undefined;
}
}
/** Tear down all state and timers (plugin dispose). */
dispose(): void {
this.disposed = true;
@@ -231,6 +267,7 @@ export class ObservationService {
this.captureInFlight.clear();
this.captureFailures.clear();
this.lastActivity.clear();
this.thumbRefs.clear();
this.observations.clear();
this.emit({ type: "reset" });
this.listeners.clear();
@@ -320,6 +357,7 @@ export class ObservationService {
this.captureFailures.delete(sessionId);
this.globalFailures = 0;
this.setAvailable(true);
this.thumbRefs.set(String(ref.attachmentId), ref);
this.put({ ...entry, thumbnailAttachmentId: String(ref.attachmentId) });
} catch {
// Silent by design: keep the previous frame, count toward backoff.
@@ -124,7 +124,20 @@ function setup(opts: {
const attachments =
"attachments" in opts
? opts.attachments
: { saveImage: async () => ({ attachmentId: "att-1" }) };
: {
saveImage: async (input: { data: Uint8Array; name?: string }) => ({
attachmentId: "att-1",
mediaType: "image/png",
bytes: input.data.byteLength,
width: 4,
height: 4,
name: input.name,
}),
readImage: async (ref: { attachmentId: unknown }) => ({
data: new Uint8Array([1, 2, 3]),
attachment: ref,
}),
};
const service = new ObservationService({
ctx: fakeCtx(attachments),
runner,
@@ -462,6 +475,22 @@ describe("availability and dead sessions", () => {
});
});
describe("thumbnail read-back", () => {
it("serves captured frames through readThumbnail and 404s unknown ids", async () => {
const scheduler = fakeScheduler();
const runner = fakeRunner();
const { service } = setup({ runner, scheduler });
service.addSession("s1");
scheduler.runNext();
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId !== undefined);
const frame = await service.readThumbnail("att-1");
expect(frame?.mediaType).toBe("image/png");
expect([...(frame?.data ?? [])]).toEqual([1, 2, 3]);
expect(await service.readThumbnail("nope")).toBeUndefined();
service.dispose();
});
});
describe("actionForLabel", () => {
it("maps tool labels onto observation verbs", () => {
expect(actionForLabel("navigate")).toBe("navigating");