mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin-browserskill): observation overlay — floating card, interrupt, and PiP (Phase 1-B)
Client half of the PiP observation window, wired to the host ObservationService over the /bsk-observation HTTP+SSE seam: - ObservationClientStore: initial state fetch + SSE increments + on-demand thumbnail blob loading (session-authorized readAttachment, same path as the toolview) + interrupt POST. All I/O injectable. - ObservationOverlay registered into the shell.overlay list seat (the sanctioned frame-wide surface; root is off-limits). Hidden with no owned sessions; appears on first start; vanishes when all stop. - Focus view: status row (green/grey/red dot + session + action + mm:ss ticking), breathing thumbnail (new frames fade in, failures keep the last frame with a warning badge, pre-navigate placeholder), and the action area. - Interrupt: single click, no confirm; greys into 'Interrupting…' and back; disabled when nothing is in flight; one-time tooltip explains the semantics (stops the current action only; the run continues). - Card: drag-move via the header, corner-handle resize (min 240x180, max 80% viewport), both clamped and kept for the page lifetime; collapsible status capsule. - PiP: Pop out (user gesture) opens Document PiP with the card's current size, portals the same content in, and clones the document's style nodes; pagehide falls back to the card with state intact; unsupported browsers hide the button; nothing auto-pops. Tests (+17, 89 total): store (fetch/SSE apply/malformed frames/ thumbnail lifecycle/interrupt wire/stop cleanup) and overlay (hidden-> visible lifecycle, status row, thumbnail via loader, interrupt disabled/active/hint-once, capsule, resize clamps, move clamps, PiP unsupported, PiP pop/fallback). Styling follows web-styling (dsw-alias tokens, CSS Modules, focus-visible, reduced-motion).
This commit is contained in:
@@ -1,21 +1,20 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: registers the `browser_screenshot`
|
||||
* keyed toolview (`tool.call.toolview` slot) so screenshot results render the
|
||||
* captured image inline in the Web UI instead of a bare JSON block. Every
|
||||
* other browser_* tool keeps the stock terminal card.
|
||||
*
|
||||
* The image bytes never enter the session log: the view resolves the durable
|
||||
* attachment reference through the client session's authorized
|
||||
* `readAttachment` RPC and hands a blob URL to the shared MessageImage atom.
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt) floating
|
||||
* over the shell via the `shell.overlay` seat.
|
||||
*/
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
|
||||
import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { createElement } from "react";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
/** Required services: the slot registry plus session-scoped attachment reads. */
|
||||
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
|
||||
export const inject = ["slots", "sessions"];
|
||||
|
||||
/** Resolve one durable image attachment into a browser blob URL. */
|
||||
@@ -41,7 +40,29 @@ async function loadSessionImage(
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed view with its session-bound loader.
|
||||
* Thumbnail loader for the overlay: the observation overlay is root-scoped,
|
||||
* so borrow the current (or first) bound session for the authorized
|
||||
* attachment read. Without any bound session the frame simply fails and the
|
||||
* overlay keeps the previous one.
|
||||
*/
|
||||
function overlayImageLoader(sessions: ISessions): (attachmentId: string) => Promise<string> {
|
||||
return async (attachmentId) => {
|
||||
const list = sessions.list.getSnapshot();
|
||||
const sessionId = list.current ?? list.ids[0];
|
||||
if (sessionId === undefined) throw new Error("no bound session for attachment read");
|
||||
const session = sessions.binding(sessionId)?.session;
|
||||
if (session === undefined) throw new Error(`session "${String(sessionId)}" is not bound`);
|
||||
const result = await session.readAttachment(attachmentId as ImageAttachmentRef["attachmentId"]);
|
||||
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
|
||||
const bytes = Uint8Array.from(result.value.data);
|
||||
return URL.createObjectURL(
|
||||
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed toolview and the observation overlay.
|
||||
* @param ctx - client root context.
|
||||
*/
|
||||
export function apply(ctx: ClientContext): void {
|
||||
@@ -57,4 +78,14 @@ export function apply(ctx: ClientContext): void {
|
||||
}),
|
||||
),
|
||||
);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: (url, init) => fetch(url, init),
|
||||
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
||||
loadImage: overlayImageLoader(sessions),
|
||||
});
|
||||
ctx.slots.inject("shell.overlay", () =>
|
||||
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
||||
createElement(ObservationOverlay, { store }),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user