mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin-browserskill): observation overlay — floating card, interrupt, and PiP (Phase 1-B)
Client half of the PiP observation window, wired to the host ObservationService over the /bsk-observation HTTP+SSE seam: - ObservationClientStore: initial state fetch + SSE increments + on-demand thumbnail blob loading (session-authorized readAttachment, same path as the toolview) + interrupt POST. All I/O injectable. - ObservationOverlay registered into the shell.overlay list seat (the sanctioned frame-wide surface; root is off-limits). Hidden with no owned sessions; appears on first start; vanishes when all stop. - Focus view: status row (green/grey/red dot + session + action + mm:ss ticking), breathing thumbnail (new frames fade in, failures keep the last frame with a warning badge, pre-navigate placeholder), and the action area. - Interrupt: single click, no confirm; greys into 'Interrupting…' and back; disabled when nothing is in flight; one-time tooltip explains the semantics (stops the current action only; the run continues). - Card: drag-move via the header, corner-handle resize (min 240x180, max 80% viewport), both clamped and kept for the page lifetime; collapsible status capsule. - PiP: Pop out (user gesture) opens Document PiP with the card's current size, portals the same content in, and clones the document's style nodes; pagehide falls back to the card with state intact; unsupported browsers hide the button; nothing auto-pops. Tests (+17, 89 total): store (fetch/SSE apply/malformed frames/ thumbnail lifecycle/interrupt wire/stop cleanup) and overlay (hidden-> visible lifecycle, status row, thumbnail via loader, interrupt disabled/active/hint-once, capsule, resize clamps, move clamps, PiP unsupported, PiP pop/fallback). Styling follows web-styling (dsw-alias tokens, CSS Modules, focus-visible, reduced-motion).
This commit is contained in:
BB-fat
@@ -0,0 +1,258 @@
|
||||
/* Observation overlay: floating card + collapsed capsule. Semantic tokens
|
||||
only (--dsw-alias-*); the shell overlay layer is click-through, so the card
|
||||
and capsule opt back into pointer events themselves. */
|
||||
|
||||
.card {
|
||||
position: fixed;
|
||||
z-index: 40;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
pointer-events: auto;
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 8px 28px rgb(0 0 0 / 18%);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
font-size: 12px;
|
||||
color: var(--dsw-alias-label-primary);
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 8px;
|
||||
cursor: move;
|
||||
user-select: none;
|
||||
touch-action: none;
|
||||
border-bottom: 1px solid var(--dsw-alias-border-secondary);
|
||||
}
|
||||
|
||||
.dot {
|
||||
flex: none;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--dsw-alias-label-tertiary);
|
||||
}
|
||||
|
||||
.dot[data-state="active"] {
|
||||
background: var(--dsw-alias-label-success);
|
||||
animation: bsk-obs-pulse 1.6s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.dot[data-state="error"] {
|
||||
background: var(--dsw-alias-label-error);
|
||||
}
|
||||
@keyframes bsk-obs-pulse {
|
||||
0%,
|
||||
100% {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
50% {
|
||||
opacity: 0.35;
|
||||
}
|
||||
}
|
||||
|
||||
.status-text {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.icon-button {
|
||||
flex: none;
|
||||
border: none;
|
||||
background: none;
|
||||
padding: 0 4px;
|
||||
font: inherit;
|
||||
color: var(--dsw-alias-label-tertiary);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.icon-button:hover {
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
}
|
||||
|
||||
.stage {
|
||||
position: relative;
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: var(--dsw-alias-bg-base);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
object-fit: contain;
|
||||
animation: bsk-obs-fade-in 240ms ease-out;
|
||||
}
|
||||
@keyframes bsk-obs-fade-in {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
color: var(--dsw-alias-label-tertiary);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.badge {
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
right: 6px;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 50%;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
color: var(--dsw-alias-label-warning);
|
||||
border: 1px solid var(--dsw-alias-label-warning);
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
}
|
||||
|
||||
.actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
padding: 6px 8px;
|
||||
border-top: 1px solid var(--dsw-alias-border-secondary);
|
||||
}
|
||||
|
||||
.interrupt-wrap {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
}
|
||||
|
||||
.interrupt {
|
||||
border: 1px solid var(--dsw-alias-label-error);
|
||||
background: none;
|
||||
color: var(--dsw-alias-label-error);
|
||||
border-radius: 6px;
|
||||
padding: 3px 12px;
|
||||
font: inherit;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.interrupt:disabled {
|
||||
opacity: 0.45;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.interrupt:focus-visible {
|
||||
outline: 2px solid var(--dsw-alias-border-focus);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.interrupt:not(:disabled):hover {
|
||||
background: var(--dsw-alias-bg-danger-secondary, rgb(220 60 60 / 12%));
|
||||
}
|
||||
|
||||
.hint {
|
||||
position: absolute;
|
||||
bottom: calc(100% + 6px);
|
||||
left: 0;
|
||||
width: 220px;
|
||||
padding: 6px 8px;
|
||||
border-radius: 6px;
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
box-shadow: 0 4px 14px rgb(0 0 0 / 16%);
|
||||
z-index: 41;
|
||||
}
|
||||
|
||||
.popout {
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
background: none;
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
border-radius: 6px;
|
||||
padding: 3px 10px;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.popout:hover {
|
||||
color: var(--dsw-alias-label-primary);
|
||||
}
|
||||
|
||||
.resize-handle {
|
||||
position: absolute;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
cursor: nwse-resize;
|
||||
touch-action: none;
|
||||
}
|
||||
|
||||
.resize-handle::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
right: 3px;
|
||||
bottom: 3px;
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-right: 2px solid var(--dsw-alias-label-tertiary);
|
||||
border-bottom: 2px solid var(--dsw-alias-label-tertiary);
|
||||
}
|
||||
|
||||
.capsule {
|
||||
position: fixed;
|
||||
right: 16px;
|
||||
bottom: 16px;
|
||||
z-index: 40;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 12px;
|
||||
pointer-events: auto;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
box-shadow: 0 4px 14px rgb(0 0 0 / 16%);
|
||||
font-size: 12px;
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.popout:focus-visible,
|
||||
.icon-button:focus-visible,
|
||||
.capsule:focus-visible {
|
||||
outline: 2px solid var(--dsw-alias-border-focus);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.dot[data-state="active"] {
|
||||
animation: none;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
// Observation overlay: the default in-app carrier for live session watching.
|
||||
// Expanded = draggable/resizable floating card (bottom-right); collapsed = a
|
||||
// status capsule; "pop out" upgrades the same content to a native Document
|
||||
// PiP window (user gesture required by the browser). Pure client state —
|
||||
// position/size/collapse live for the page lifetime only.
|
||||
|
||||
import {
|
||||
type PointerEvent as ReactPointerEvent,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
useSyncExternalStore,
|
||||
} from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import type { SessionObservation } from "../observation";
|
||||
import css from "./ObservationOverlay.module.css";
|
||||
import type { ObservationClientStore } from "./observation-store";
|
||||
|
||||
/** Minimal Document PiP surface (TS lib.dom lacks it). */
|
||||
interface DocumentPip {
|
||||
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
|
||||
}
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface Size {
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SIZE: Size = { w: 320, h: 240 };
|
||||
const MIN_SIZE: Size = { w: 240, h: 180 };
|
||||
const EDGE_MARGIN = 16;
|
||||
|
||||
function pipApi(): DocumentPip | undefined {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
|
||||
}
|
||||
|
||||
function clampSize(size: Size, viewport: Size): Size {
|
||||
const maxW = viewport.w * 0.8;
|
||||
const maxH = viewport.h * 0.8;
|
||||
return {
|
||||
w: Math.min(Math.max(size.w, MIN_SIZE.w), maxW),
|
||||
h: Math.min(Math.max(size.h, MIN_SIZE.h), maxH),
|
||||
};
|
||||
}
|
||||
|
||||
function clampPos(pos: Point, size: Size, viewport: Size): Point {
|
||||
return {
|
||||
x: Math.min(Math.max(pos.x, 0), Math.max(0, viewport.w - size.w)),
|
||||
y: Math.min(Math.max(pos.y, 0), Math.max(0, viewport.h - size.h)),
|
||||
};
|
||||
}
|
||||
|
||||
function formatElapsed(sinceMs: number, nowMs: number): string {
|
||||
const total = Math.max(0, Math.floor((nowMs - sinceMs) / 1000));
|
||||
const mm = String(Math.floor(total / 60)).padStart(2, "0");
|
||||
const ss = String(total % 60).padStart(2, "0");
|
||||
return `${mm}:${ss}`;
|
||||
}
|
||||
|
||||
/** Focus = most recently active session (action or latest state change). */
|
||||
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
|
||||
if (sessions.length === 0) return undefined;
|
||||
return sessions.reduce((a, b) => (a.since >= b.since ? a : b));
|
||||
}
|
||||
|
||||
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
|
||||
if (obs.lastError !== undefined && obs.action === "idle") return "error";
|
||||
return obs.action === "idle" ? "idle" : "active";
|
||||
}
|
||||
|
||||
/** The floating card / PiP shared content. */
|
||||
function OverlayBody(props: {
|
||||
store: ObservationClientStore;
|
||||
focus: SessionObservation | undefined;
|
||||
now: number;
|
||||
onPopOut?: (() => void) | undefined;
|
||||
onCollapse?: (() => void) | undefined;
|
||||
inPip: boolean;
|
||||
onHeaderPointerDown?: (event: ReactPointerEvent<HTMLDivElement>) => void;
|
||||
}) {
|
||||
const { store, focus, now, onPopOut, onCollapse, inPip, onHeaderPointerDown } = props;
|
||||
const [interrupting, setInterrupting] = useState(false);
|
||||
const [hintOpen, setHintOpen] = useState(false);
|
||||
const [hintSeen, setHintSeen] = useState(false);
|
||||
|
||||
const thumbId = focus?.thumbnailAttachmentId;
|
||||
useEffect(() => {
|
||||
store.ensureThumbnail(thumbId);
|
||||
}, [store, thumbId]);
|
||||
const thumb = thumbId !== undefined ? store.getSnapshot().thumbnails[thumbId] : undefined;
|
||||
|
||||
const canInterrupt = !interrupting && focus !== undefined && focus.action !== "idle";
|
||||
const onInterrupt = (): void => {
|
||||
if (!canInterrupt || focus === undefined) return;
|
||||
setHintSeen(true);
|
||||
setHintOpen(false);
|
||||
setInterrupting(true);
|
||||
void store.interrupt(focus.sessionId).finally(() => setInterrupting(false));
|
||||
};
|
||||
|
||||
const actionText =
|
||||
focus === undefined ? "no session" : focus.action === "idle" ? "idle" : focus.action;
|
||||
const elapsed = focus !== undefined ? formatElapsed(focus.since, now) : "00:00";
|
||||
const state = focus !== undefined ? statusOf(focus) : "idle";
|
||||
|
||||
return (
|
||||
<div className={css.body} data-state={state} data-in-pip={inPip || undefined}>
|
||||
<div
|
||||
className={css.header}
|
||||
data-testid="obs-header"
|
||||
onPointerDown={onHeaderPointerDown}
|
||||
role="presentation"
|
||||
>
|
||||
<span className={css.dot} data-state={state} aria-hidden />
|
||||
<span className={css["status-text"]}>
|
||||
{focus === undefined
|
||||
? "browser sessions"
|
||||
: `${focus.sessionId} · ${actionText} · ${elapsed}`}
|
||||
</span>
|
||||
{onCollapse !== undefined ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css["icon-button"]}
|
||||
aria-label="Collapse"
|
||||
onClick={onCollapse}
|
||||
>
|
||||
—
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
<div className={css.stage}>
|
||||
{thumb?.status === "ready" && thumb.url !== undefined ? (
|
||||
<img
|
||||
key={thumb.url}
|
||||
className={css.thumb}
|
||||
src={thumb.url}
|
||||
alt={`session ${focus?.sessionId ?? ""} view`}
|
||||
/>
|
||||
) : (
|
||||
<div className={css.placeholder}>
|
||||
{thumb?.status === "error" ? "frame unavailable" : "waiting for page"}
|
||||
</div>
|
||||
)}
|
||||
{thumb?.status === "error" ? (
|
||||
<span className={css.badge} aria-label="thumbnail failed">
|
||||
!
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<div className={css.actions}>
|
||||
<span className={css["interrupt-wrap"]}>
|
||||
<button
|
||||
type="button"
|
||||
className={css.interrupt}
|
||||
disabled={!canInterrupt}
|
||||
aria-label="Interrupt the current browser action"
|
||||
onPointerEnter={() => {
|
||||
if (!hintSeen) setHintOpen(true);
|
||||
}}
|
||||
onPointerLeave={() => setHintOpen(false)}
|
||||
onClick={onInterrupt}
|
||||
>
|
||||
{interrupting ? "Interrupting…" : "Interrupt"}
|
||||
</button>
|
||||
{hintOpen && !hintSeen ? (
|
||||
<span className={css.hint} role="tooltip">
|
||||
Interrupt stops only the current browser action — the agent run continues (use the
|
||||
chat Stop button to halt the run).
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
{onPopOut !== undefined ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css.popout}
|
||||
aria-label="Pop out into a mini window"
|
||||
onClick={onPopOut}
|
||||
>
|
||||
Pop out
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Clone the host document's style/link nodes into a PiP window. */
|
||||
function cloneStylesInto(pipWindow: Window): void {
|
||||
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
|
||||
pipWindow.document.head.appendChild(node.cloneNode(true));
|
||||
}
|
||||
}
|
||||
|
||||
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
|
||||
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
|
||||
useEffect(() => {
|
||||
store.start();
|
||||
return () => store.stop();
|
||||
}, [store]);
|
||||
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [pos, setPos] = useState<Point | null>(null);
|
||||
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
|
||||
const [pipWindow, setPipWindow] = useState<Window | null>(null);
|
||||
const dragRef = useRef<{
|
||||
kind: "move" | "resize";
|
||||
startX: number;
|
||||
startY: number;
|
||||
base: Point & Size;
|
||||
} | null>(null);
|
||||
|
||||
// Elapsed-time ticker: 1s while anything is active.
|
||||
const focus = focusOf(snapshot.sessions);
|
||||
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
useEffect(() => {
|
||||
if (!anyActive) return;
|
||||
const timer = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => clearInterval(timer);
|
||||
}, [anyActive]);
|
||||
|
||||
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
|
||||
|
||||
const onPointerMove = useCallback((event: PointerEvent) => {
|
||||
const drag = dragRef.current;
|
||||
if (drag === null) return;
|
||||
const dx = event.clientX - drag.startX;
|
||||
const dy = event.clientY - drag.startY;
|
||||
if (drag.kind === "move") {
|
||||
setPos(
|
||||
clampPos(
|
||||
{ x: drag.base.x + dx, y: drag.base.y + dy },
|
||||
{ w: drag.base.w, h: drag.base.h },
|
||||
viewport(),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
setSize(clampSize({ w: drag.base.w + dx, h: drag.base.h + dy }, viewport()));
|
||||
}
|
||||
}, []);
|
||||
const onPointerUp = useCallback(() => {
|
||||
dragRef.current = null;
|
||||
document.removeEventListener("pointermove", onPointerMove);
|
||||
document.removeEventListener("pointerup", onPointerUp);
|
||||
}, [onPointerMove]);
|
||||
|
||||
const beginDrag = (kind: "move" | "resize") => (event: ReactPointerEvent) => {
|
||||
event.preventDefault();
|
||||
const rect = (
|
||||
event.currentTarget.closest("[data-obs-card]") as HTMLElement | null
|
||||
)?.getBoundingClientRect();
|
||||
const base = {
|
||||
x: rect?.left ?? pos?.x ?? viewport().w - size.w - EDGE_MARGIN,
|
||||
y: rect?.top ?? pos?.y ?? viewport().h - size.h - EDGE_MARGIN,
|
||||
w: rect?.width ?? size.w,
|
||||
h: rect?.height ?? size.h,
|
||||
};
|
||||
dragRef.current = { kind, startX: event.clientX, startY: event.clientY, base };
|
||||
if (kind === "move") setPos({ x: base.x, y: base.y });
|
||||
document.addEventListener("pointermove", onPointerMove);
|
||||
document.addEventListener("pointerup", onPointerUp);
|
||||
};
|
||||
|
||||
const popOut = async (): Promise<void> => {
|
||||
const pip = pipApi();
|
||||
if (pip === undefined) return;
|
||||
const win = await pip.requestWindow({ width: size.w, height: size.h });
|
||||
cloneStylesInto(win);
|
||||
win.addEventListener("pagehide", () => setPipWindow(null));
|
||||
setPipWindow(win);
|
||||
};
|
||||
|
||||
// Hidden while no owned session exists (and no PiP is up).
|
||||
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
|
||||
|
||||
const body = (
|
||||
<OverlayBody
|
||||
store={store}
|
||||
focus={focus}
|
||||
now={now}
|
||||
inPip={pipWindow !== null}
|
||||
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
|
||||
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
|
||||
onHeaderPointerDown={pipWindow === null ? beginDrag("move") : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
if (pipWindow !== null) {
|
||||
return createPortal(body, pipWindow.document.body);
|
||||
}
|
||||
|
||||
if (collapsed) {
|
||||
const state = focus !== undefined ? statusOf(focus) : "idle";
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
className={css.capsule}
|
||||
data-state={state}
|
||||
aria-label="Expand browser observation overlay"
|
||||
onClick={() => setCollapsed(false)}
|
||||
>
|
||||
<span className={css.dot} data-state={state} aria-hidden />
|
||||
<span className={css["capsule-text"]}>
|
||||
{snapshot.sessions.length} session{snapshot.sessions.length === 1 ? "" : "s"}
|
||||
{focus !== undefined && focus.action !== "idle"
|
||||
? ` · ${focus.action} · ${formatElapsed(focus.since, now)}`
|
||||
: ""}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
const style: React.CSSProperties =
|
||||
pos !== null
|
||||
? { left: pos.x, top: pos.y, width: size.w, height: size.h }
|
||||
: { right: EDGE_MARGIN, bottom: EDGE_MARGIN, width: size.w, height: size.h };
|
||||
return (
|
||||
<div className={css.card} style={style} data-obs-card data-testid="obs-card">
|
||||
{body}
|
||||
<div
|
||||
className={css["resize-handle"]}
|
||||
data-testid="obs-resize"
|
||||
aria-label="Resize overlay"
|
||||
role="separator"
|
||||
onPointerDown={beginDrag("resize")}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,21 +1,20 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: registers the `browser_screenshot`
|
||||
* keyed toolview (`tool.call.toolview` slot) so screenshot results render the
|
||||
* captured image inline in the Web UI instead of a bare JSON block. Every
|
||||
* other browser_* tool keeps the stock terminal card.
|
||||
*
|
||||
* The image bytes never enter the session log: the view resolves the durable
|
||||
* attachment reference through the client session's authorized
|
||||
* `readAttachment` RPC and hands a blob URL to the shared MessageImage atom.
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt) floating
|
||||
* over the shell via the `shell.overlay` seat.
|
||||
*/
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
|
||||
import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { createElement } from "react";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
/** Required services: the slot registry plus session-scoped attachment reads. */
|
||||
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
|
||||
export const inject = ["slots", "sessions"];
|
||||
|
||||
/** Resolve one durable image attachment into a browser blob URL. */
|
||||
@@ -41,7 +40,29 @@ async function loadSessionImage(
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed view with its session-bound loader.
|
||||
* Thumbnail loader for the overlay: the observation overlay is root-scoped,
|
||||
* so borrow the current (or first) bound session for the authorized
|
||||
* attachment read. Without any bound session the frame simply fails and the
|
||||
* overlay keeps the previous one.
|
||||
*/
|
||||
function overlayImageLoader(sessions: ISessions): (attachmentId: string) => Promise<string> {
|
||||
return async (attachmentId) => {
|
||||
const list = sessions.list.getSnapshot();
|
||||
const sessionId = list.current ?? list.ids[0];
|
||||
if (sessionId === undefined) throw new Error("no bound session for attachment read");
|
||||
const session = sessions.binding(sessionId)?.session;
|
||||
if (session === undefined) throw new Error(`session "${String(sessionId)}" is not bound`);
|
||||
const result = await session.readAttachment(attachmentId as ImageAttachmentRef["attachmentId"]);
|
||||
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
|
||||
const bytes = Uint8Array.from(result.value.data);
|
||||
return URL.createObjectURL(
|
||||
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed toolview and the observation overlay.
|
||||
* @param ctx - client root context.
|
||||
*/
|
||||
export function apply(ctx: ClientContext): void {
|
||||
@@ -57,4 +78,14 @@ export function apply(ctx: ClientContext): void {
|
||||
}),
|
||||
),
|
||||
);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: (url, init) => fetch(url, init),
|
||||
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
||||
loadImage: overlayImageLoader(sessions),
|
||||
});
|
||||
ctx.slots.inject("shell.overlay", () =>
|
||||
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
||||
createElement(ObservationOverlay, { store }),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Client-side data layer for the observation overlay: initial state fetch,
|
||||
* SSE increment stream, on-demand thumbnail blob loading through the
|
||||
* session-authorized readAttachment path, and the interrupt call. All I/O is
|
||||
* injected so tests never touch a network.
|
||||
*/
|
||||
|
||||
import type { ObservationEvent, SessionObservation } from "../observation";
|
||||
|
||||
export interface EventSourceLike {
|
||||
onmessage: ((event: { data: string }) => void) | null;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface ObservationClientDeps {
|
||||
fetchFn: (
|
||||
url: string,
|
||||
init?: { method?: string; body?: string },
|
||||
) => Promise<{
|
||||
ok: boolean;
|
||||
json(): Promise<unknown>;
|
||||
}>;
|
||||
eventSourceFactory: (url: string) => EventSourceLike;
|
||||
/** Resolve one attachment id to a displayable blob URL. */
|
||||
loadImage: (attachmentId: string) => Promise<string>;
|
||||
}
|
||||
|
||||
export interface ThumbnailState {
|
||||
status: "loading" | "ready" | "error";
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface OverlaySnapshot {
|
||||
readonly sessions: readonly SessionObservation[];
|
||||
readonly connected: boolean;
|
||||
readonly thumbnails: Readonly<Record<string, ThumbnailState>>;
|
||||
}
|
||||
|
||||
const STATE_URL = "/bsk-observation/state";
|
||||
const EVENTS_URL = "/bsk-observation/events";
|
||||
const INTERRUPT_URL = "/bsk-observation/interrupt";
|
||||
|
||||
export class ObservationClientStore {
|
||||
private sessions = new Map<string, SessionObservation>();
|
||||
private thumbs = new Map<string, ThumbnailState>();
|
||||
private listeners = new Set<() => void>();
|
||||
private events: EventSourceLike | undefined;
|
||||
private snapshot: OverlaySnapshot = { sessions: [], connected: false, thumbnails: {} };
|
||||
private started = false;
|
||||
|
||||
constructor(private readonly deps: ObservationClientDeps) {}
|
||||
|
||||
subscribe = (listener: () => void): (() => void) => {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
};
|
||||
|
||||
getSnapshot = (): OverlaySnapshot => this.snapshot;
|
||||
|
||||
private publish(): void {
|
||||
this.snapshot = {
|
||||
sessions: [...this.sessions.values()],
|
||||
connected: this.events !== undefined,
|
||||
thumbnails: Object.fromEntries(this.thumbs),
|
||||
};
|
||||
for (const listener of [...this.listeners]) listener();
|
||||
}
|
||||
|
||||
/** Initial fetch + SSE subscription. Idempotent. */
|
||||
start(): void {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
void this.deps
|
||||
.fetchFn(STATE_URL)
|
||||
.then(async (res) => {
|
||||
if (!res.ok) return;
|
||||
const body = (await res.json()) as { sessions?: SessionObservation[] };
|
||||
this.sessions = new Map((body.sessions ?? []).map((s) => [s.sessionId, s]));
|
||||
this.publish();
|
||||
})
|
||||
.catch(() => {});
|
||||
const events = this.deps.eventSourceFactory(EVENTS_URL);
|
||||
events.onmessage = (message) => {
|
||||
let event: ObservationEvent;
|
||||
try {
|
||||
event = JSON.parse(message.data) as ObservationEvent;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
this.apply(event);
|
||||
};
|
||||
this.events = events;
|
||||
this.publish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.events?.close();
|
||||
this.events = undefined;
|
||||
this.started = false;
|
||||
for (const thumb of this.thumbs.values()) {
|
||||
if (thumb.url !== undefined && typeof URL.revokeObjectURL === "function") {
|
||||
URL.revokeObjectURL(thumb.url);
|
||||
}
|
||||
}
|
||||
this.thumbs.clear();
|
||||
this.sessions.clear();
|
||||
this.publish();
|
||||
}
|
||||
|
||||
private apply(event: ObservationEvent): void {
|
||||
if (event.type === "reset") {
|
||||
this.sessions.clear();
|
||||
} else if (event.type === "remove" && event.session !== undefined) {
|
||||
this.sessions.delete(event.session.sessionId);
|
||||
} else if (event.type === "upsert" && event.session !== undefined) {
|
||||
this.sessions.set(event.session.sessionId, event.session);
|
||||
}
|
||||
this.publish();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a thumbnail load is in flight for one attachment reference. New
|
||||
* frames replace the old URL; failures keep the last good frame (the caller
|
||||
* renders `status: 'error'` as a small badge over the old image).
|
||||
*/
|
||||
ensureThumbnail(attachmentId: string | undefined): void {
|
||||
if (attachmentId === undefined || this.thumbs.has(attachmentId)) return;
|
||||
this.thumbs.set(attachmentId, { status: "loading" });
|
||||
this.publish();
|
||||
this.deps.loadImage(attachmentId).then(
|
||||
(url) => {
|
||||
this.thumbs.set(attachmentId, { status: "ready", url });
|
||||
this.publish();
|
||||
},
|
||||
() => {
|
||||
this.thumbs.set(attachmentId, { status: "error" });
|
||||
this.publish();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Interrupt the current or named session's in-flight call. */
|
||||
async interrupt(sessionId?: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await this.deps.fetchFn(INTERRUPT_URL, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(sessionId === undefined ? {} : { sessionId }),
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
const body = (await res.json()) as { interrupted?: boolean };
|
||||
return body.interrupted === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user