mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(dsh-plugin-browserskill): serve observation thumbnails over the plugin's own route
The session-authorized client RPC (readAttachment) refuses images that no session log references — observation frames are plugin-owned runtime data, so every overlay thumbnail failed with ATTACHMENT_NOT_REFERENCED. The host now keeps the full attachment ref per frame and serves bytes through GET /bsk-observation/thumbnail/<id> (verified readImage path); the client loader is a plain fetch of that route.
This commit is contained in:
@@ -40,25 +40,14 @@ async function loadSessionImage(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Thumbnail loader for the overlay: the observation overlay is root-scoped,
|
* Thumbnail loader for the overlay: frames are plugin-owned runtime data
|
||||||
* so borrow the current (or first) bound session for the authorized
|
* (never referenced by a session log, so the session-authorized RPC refuses
|
||||||
* attachment read. Without any bound session the frame simply fails and the
|
* them), served by the host over the plugin's own route.
|
||||||
* overlay keeps the previous one.
|
|
||||||
*/
|
*/
|
||||||
function overlayImageLoader(sessions: ISessions): (attachmentId: string) => Promise<string> {
|
async function overlayImageLoader(attachmentId: string): Promise<string> {
|
||||||
return async (attachmentId) => {
|
const res = await fetch(`/bsk-observation/thumbnail/${encodeURIComponent(attachmentId)}`);
|
||||||
const list = sessions.list.getSnapshot();
|
if (!res.ok) throw new Error(`thumbnail fetch failed: ${res.status}`);
|
||||||
const sessionId = list.current ?? list.ids[0];
|
return URL.createObjectURL(await res.blob());
|
||||||
if (sessionId === undefined) throw new Error("no bound session for attachment read");
|
|
||||||
const session = sessions.binding(sessionId)?.session;
|
|
||||||
if (session === undefined) throw new Error(`session "${String(sessionId)}" is not bound`);
|
|
||||||
const result = await session.readAttachment(attachmentId as ImageAttachmentRef["attachmentId"]);
|
|
||||||
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
|
|
||||||
const bytes = Uint8Array.from(result.value.data);
|
|
||||||
return URL.createObjectURL(
|
|
||||||
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
|
|
||||||
);
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -81,7 +70,7 @@ export function apply(ctx: ClientContext): void {
|
|||||||
const store = new ObservationClientStore({
|
const store = new ObservationClientStore({
|
||||||
fetchFn: (url, init) => fetch(url, init),
|
fetchFn: (url, init) => fetch(url, init),
|
||||||
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
||||||
loadImage: overlayImageLoader(sessions),
|
loadImage: overlayImageLoader,
|
||||||
});
|
});
|
||||||
ctx.slots.inject("shell.overlay", () =>
|
ctx.slots.inject("shell.overlay", () =>
|
||||||
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
||||||
|
|||||||
@@ -5,9 +5,10 @@
|
|||||||
* the event allowlist lives in dsh-api-remotes), so the plugin serves its
|
* the event allowlist lives in dsh-api-remotes), so the plugin serves its
|
||||||
* observation channel through the documented `webServer` route seam instead:
|
* observation channel through the documented `webServer` route seam instead:
|
||||||
*
|
*
|
||||||
* GET /bsk-observation/state → { sessions: SessionObservation[] }
|
* GET /bsk-observation/state → { sessions, available }
|
||||||
* GET /bsk-observation/events → SSE stream of ObservationEvent
|
* GET /bsk-observation/events → SSE stream of ObservationEvent
|
||||||
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
|
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
|
||||||
|
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
|
||||||
*
|
*
|
||||||
* Routes exist only when a webServer service is mounted (web composition);
|
* Routes exist only when a webServer service is mounted (web composition);
|
||||||
* headless deployments skip registration entirely.
|
* headless deployments skip registration entirely.
|
||||||
@@ -112,6 +113,28 @@ export function registerObservationRoutes(
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
webServer.register({
|
||||||
|
kind: "prefix",
|
||||||
|
path: `${ROUTE_BASE}/thumbnail/`,
|
||||||
|
handler: async (req, res) => {
|
||||||
|
if (req.method !== "GET") {
|
||||||
|
sendJson(res, 405, { error: "method not allowed" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const pathname = decodeURIComponent(new URL(req.url ?? "/", "http://x").pathname);
|
||||||
|
const attachmentId = pathname.slice(`${ROUTE_BASE}/thumbnail/`.length);
|
||||||
|
const frame = await observation.readThumbnail(attachmentId);
|
||||||
|
if (frame === undefined) {
|
||||||
|
sendJson(res, 404, { error: "unknown thumbnail" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.writeHead(200, {
|
||||||
|
"content-type": frame.mediaType,
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
});
|
||||||
|
res.end(Buffer.from(frame.data));
|
||||||
|
},
|
||||||
|
}),
|
||||||
];
|
];
|
||||||
return () => {
|
return () => {
|
||||||
for (const dispose of disposers) dispose();
|
for (const dispose of disposers) dispose();
|
||||||
|
|||||||
@@ -58,7 +58,20 @@ interface AttachmentLike {
|
|||||||
data: Uint8Array;
|
data: Uint8Array;
|
||||||
mediaType: string;
|
mediaType: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
}): Promise<{ attachmentId: unknown }>;
|
}): Promise<ImageAttachmentRefLike>;
|
||||||
|
readImage(
|
||||||
|
ref: ImageAttachmentRefLike,
|
||||||
|
): Promise<{ data: Uint8Array; attachment: ImageAttachmentRefLike }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The attachment reference fields the store verifies reads against. */
|
||||||
|
export interface ImageAttachmentRefLike {
|
||||||
|
attachmentId: unknown;
|
||||||
|
mediaType: string;
|
||||||
|
bytes: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
name?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Injectable clock/scheduler bits for tests. */
|
/** Injectable clock/scheduler bits for tests. */
|
||||||
@@ -89,6 +102,8 @@ export class ObservationService {
|
|||||||
private readonly captureInFlight = new Set<string>();
|
private readonly captureInFlight = new Set<string>();
|
||||||
private readonly captureFailures = new Map<string, number>();
|
private readonly captureFailures = new Map<string, number>();
|
||||||
private readonly lastActivity = new Map<string, number>();
|
private readonly lastActivity = new Map<string, number>();
|
||||||
|
/** attachmentId → full reference (the store requires it for verified reads). */
|
||||||
|
private readonly thumbRefs = new Map<string, ImageAttachmentRefLike>();
|
||||||
/** Consecutive capture failures across ALL sessions (daemon-level signal). */
|
/** Consecutive capture failures across ALL sessions (daemon-level signal). */
|
||||||
private globalFailures = 0;
|
private globalFailures = 0;
|
||||||
private available = true;
|
private available = true;
|
||||||
@@ -223,6 +238,27 @@ export class ObservationService {
|
|||||||
return this.deps.runner.killFor(target) > 0;
|
return this.deps.runner.killFor(target) > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read one captured thumbnail back through the store's verified path.
|
||||||
|
* Powers the plugin's own HTTP thumbnail route — frames are plugin-owned
|
||||||
|
* runtime data, never referenced by any session log, so the
|
||||||
|
* session-authorized client RPC cannot serve them.
|
||||||
|
*/
|
||||||
|
async readThumbnail(
|
||||||
|
attachmentId: string,
|
||||||
|
): Promise<{ data: Uint8Array; mediaType: string } | undefined> {
|
||||||
|
const ref = this.thumbRefs.get(attachmentId);
|
||||||
|
if (ref === undefined) return undefined;
|
||||||
|
const attachments = this.deps.ctx.get("attachments") as AttachmentLike | undefined;
|
||||||
|
if (attachments === undefined) return undefined;
|
||||||
|
try {
|
||||||
|
const stored = await attachments.readImage(ref);
|
||||||
|
return { data: stored.data, mediaType: ref.mediaType };
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Tear down all state and timers (plugin dispose). */
|
/** Tear down all state and timers (plugin dispose). */
|
||||||
dispose(): void {
|
dispose(): void {
|
||||||
this.disposed = true;
|
this.disposed = true;
|
||||||
@@ -231,6 +267,7 @@ export class ObservationService {
|
|||||||
this.captureInFlight.clear();
|
this.captureInFlight.clear();
|
||||||
this.captureFailures.clear();
|
this.captureFailures.clear();
|
||||||
this.lastActivity.clear();
|
this.lastActivity.clear();
|
||||||
|
this.thumbRefs.clear();
|
||||||
this.observations.clear();
|
this.observations.clear();
|
||||||
this.emit({ type: "reset" });
|
this.emit({ type: "reset" });
|
||||||
this.listeners.clear();
|
this.listeners.clear();
|
||||||
@@ -320,6 +357,7 @@ export class ObservationService {
|
|||||||
this.captureFailures.delete(sessionId);
|
this.captureFailures.delete(sessionId);
|
||||||
this.globalFailures = 0;
|
this.globalFailures = 0;
|
||||||
this.setAvailable(true);
|
this.setAvailable(true);
|
||||||
|
this.thumbRefs.set(String(ref.attachmentId), ref);
|
||||||
this.put({ ...entry, thumbnailAttachmentId: String(ref.attachmentId) });
|
this.put({ ...entry, thumbnailAttachmentId: String(ref.attachmentId) });
|
||||||
} catch {
|
} catch {
|
||||||
// Silent by design: keep the previous frame, count toward backoff.
|
// Silent by design: keep the previous frame, count toward backoff.
|
||||||
|
|||||||
@@ -124,7 +124,20 @@ function setup(opts: {
|
|||||||
const attachments =
|
const attachments =
|
||||||
"attachments" in opts
|
"attachments" in opts
|
||||||
? opts.attachments
|
? opts.attachments
|
||||||
: { saveImage: async () => ({ attachmentId: "att-1" }) };
|
: {
|
||||||
|
saveImage: async (input: { data: Uint8Array; name?: string }) => ({
|
||||||
|
attachmentId: "att-1",
|
||||||
|
mediaType: "image/png",
|
||||||
|
bytes: input.data.byteLength,
|
||||||
|
width: 4,
|
||||||
|
height: 4,
|
||||||
|
name: input.name,
|
||||||
|
}),
|
||||||
|
readImage: async (ref: { attachmentId: unknown }) => ({
|
||||||
|
data: new Uint8Array([1, 2, 3]),
|
||||||
|
attachment: ref,
|
||||||
|
}),
|
||||||
|
};
|
||||||
const service = new ObservationService({
|
const service = new ObservationService({
|
||||||
ctx: fakeCtx(attachments),
|
ctx: fakeCtx(attachments),
|
||||||
runner,
|
runner,
|
||||||
@@ -462,6 +475,22 @@ describe("availability and dead sessions", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("thumbnail read-back", () => {
|
||||||
|
it("serves captured frames through readThumbnail and 404s unknown ids", async () => {
|
||||||
|
const scheduler = fakeScheduler();
|
||||||
|
const runner = fakeRunner();
|
||||||
|
const { service } = setup({ runner, scheduler });
|
||||||
|
service.addSession("s1");
|
||||||
|
scheduler.runNext();
|
||||||
|
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId !== undefined);
|
||||||
|
const frame = await service.readThumbnail("att-1");
|
||||||
|
expect(frame?.mediaType).toBe("image/png");
|
||||||
|
expect([...(frame?.data ?? [])]).toEqual([1, 2, 3]);
|
||||||
|
expect(await service.readThumbnail("nope")).toBeUndefined();
|
||||||
|
service.dispose();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("actionForLabel", () => {
|
describe("actionForLabel", () => {
|
||||||
it("maps tool labels onto observation verbs", () => {
|
it("maps tool labels onto observation verbs", () => {
|
||||||
expect(actionForLabel("navigate")).toBe("navigating");
|
expect(actionForLabel("navigate")).toBe("navigating");
|
||||||
|
|||||||
Reference in New Issue
Block a user