mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin-browserskill): observation overlay — floating card, interrupt, and PiP (Phase 1-B)
Client half of the PiP observation window, wired to the host ObservationService over the /bsk-observation HTTP+SSE seam: - ObservationClientStore: initial state fetch + SSE increments + on-demand thumbnail blob loading (session-authorized readAttachment, same path as the toolview) + interrupt POST. All I/O injectable. - ObservationOverlay registered into the shell.overlay list seat (the sanctioned frame-wide surface; root is off-limits). Hidden with no owned sessions; appears on first start; vanishes when all stop. - Focus view: status row (green/grey/red dot + session + action + mm:ss ticking), breathing thumbnail (new frames fade in, failures keep the last frame with a warning badge, pre-navigate placeholder), and the action area. - Interrupt: single click, no confirm; greys into 'Interrupting…' and back; disabled when nothing is in flight; one-time tooltip explains the semantics (stops the current action only; the run continues). - Card: drag-move via the header, corner-handle resize (min 240x180, max 80% viewport), both clamped and kept for the page lifetime; collapsible status capsule. - PiP: Pop out (user gesture) opens Document PiP with the card's current size, portals the same content in, and clones the document's style nodes; pagehide falls back to the card with state intact; unsupported browsers hide the button; nothing auto-pops. Tests (+17, 89 total): store (fetch/SSE apply/malformed frames/ thumbnail lifecycle/interrupt wire/stop cleanup) and overlay (hidden-> visible lifecycle, status row, thumbnail via loader, interrupt disabled/active/hint-once, capsule, resize clamps, move clamps, PiP unsupported, PiP pop/fallback). Styling follows web-styling (dsw-alias tokens, CSS Modules, focus-visible, reduced-motion).
This commit is contained in:
BB-fat
@@ -28,7 +28,8 @@
|
||||
"platform": "web",
|
||||
"inject": [
|
||||
"@deepseek-ai/dsh-client-runtime",
|
||||
"@deepseek-ai/dsh-client-ui-tool"
|
||||
"@deepseek-ai/dsh-client-ui-tool",
|
||||
"@deepseek-ai/dsh-client-ui-layout"
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -75,6 +76,7 @@
|
||||
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-layout": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
||||
@@ -83,6 +85,7 @@
|
||||
"@testing-library/react": "^16.1.0",
|
||||
"@types/node": "^22.10.0",
|
||||
"@types/react": "~18.3.1",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"happy-dom": "^15.11.7",
|
||||
"lightningcss": "^1.30.0",
|
||||
"react": "^18.2.0",
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
/* Observation overlay: floating card + collapsed capsule. Semantic tokens
|
||||
only (--dsw-alias-*); the shell overlay layer is click-through, so the card
|
||||
and capsule opt back into pointer events themselves. */
|
||||
|
||||
.card {
|
||||
position: fixed;
|
||||
z-index: 40;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
pointer-events: auto;
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 8px 28px rgb(0 0 0 / 18%);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
font-size: 12px;
|
||||
color: var(--dsw-alias-label-primary);
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 8px;
|
||||
cursor: move;
|
||||
user-select: none;
|
||||
touch-action: none;
|
||||
border-bottom: 1px solid var(--dsw-alias-border-secondary);
|
||||
}
|
||||
|
||||
.dot {
|
||||
flex: none;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--dsw-alias-label-tertiary);
|
||||
}
|
||||
|
||||
.dot[data-state="active"] {
|
||||
background: var(--dsw-alias-label-success);
|
||||
animation: bsk-obs-pulse 1.6s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.dot[data-state="error"] {
|
||||
background: var(--dsw-alias-label-error);
|
||||
}
|
||||
@keyframes bsk-obs-pulse {
|
||||
0%,
|
||||
100% {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
50% {
|
||||
opacity: 0.35;
|
||||
}
|
||||
}
|
||||
|
||||
.status-text {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.icon-button {
|
||||
flex: none;
|
||||
border: none;
|
||||
background: none;
|
||||
padding: 0 4px;
|
||||
font: inherit;
|
||||
color: var(--dsw-alias-label-tertiary);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.icon-button:hover {
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
}
|
||||
|
||||
.stage {
|
||||
position: relative;
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: var(--dsw-alias-bg-base);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
object-fit: contain;
|
||||
animation: bsk-obs-fade-in 240ms ease-out;
|
||||
}
|
||||
@keyframes bsk-obs-fade-in {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
color: var(--dsw-alias-label-tertiary);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.badge {
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
right: 6px;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 50%;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
color: var(--dsw-alias-label-warning);
|
||||
border: 1px solid var(--dsw-alias-label-warning);
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
}
|
||||
|
||||
.actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
padding: 6px 8px;
|
||||
border-top: 1px solid var(--dsw-alias-border-secondary);
|
||||
}
|
||||
|
||||
.interrupt-wrap {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
}
|
||||
|
||||
.interrupt {
|
||||
border: 1px solid var(--dsw-alias-label-error);
|
||||
background: none;
|
||||
color: var(--dsw-alias-label-error);
|
||||
border-radius: 6px;
|
||||
padding: 3px 12px;
|
||||
font: inherit;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.interrupt:disabled {
|
||||
opacity: 0.45;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.interrupt:focus-visible {
|
||||
outline: 2px solid var(--dsw-alias-border-focus);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.interrupt:not(:disabled):hover {
|
||||
background: var(--dsw-alias-bg-danger-secondary, rgb(220 60 60 / 12%));
|
||||
}
|
||||
|
||||
.hint {
|
||||
position: absolute;
|
||||
bottom: calc(100% + 6px);
|
||||
left: 0;
|
||||
width: 220px;
|
||||
padding: 6px 8px;
|
||||
border-radius: 6px;
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
box-shadow: 0 4px 14px rgb(0 0 0 / 16%);
|
||||
z-index: 41;
|
||||
}
|
||||
|
||||
.popout {
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
background: none;
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
border-radius: 6px;
|
||||
padding: 3px 10px;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.popout:hover {
|
||||
color: var(--dsw-alias-label-primary);
|
||||
}
|
||||
|
||||
.resize-handle {
|
||||
position: absolute;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
cursor: nwse-resize;
|
||||
touch-action: none;
|
||||
}
|
||||
|
||||
.resize-handle::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
right: 3px;
|
||||
bottom: 3px;
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-right: 2px solid var(--dsw-alias-label-tertiary);
|
||||
border-bottom: 2px solid var(--dsw-alias-label-tertiary);
|
||||
}
|
||||
|
||||
.capsule {
|
||||
position: fixed;
|
||||
right: 16px;
|
||||
bottom: 16px;
|
||||
z-index: 40;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 12px;
|
||||
pointer-events: auto;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--dsw-alias-border-secondary);
|
||||
background: var(--dsw-alias-bg-elevated, var(--dsw-alias-bg-base));
|
||||
box-shadow: 0 4px 14px rgb(0 0 0 / 16%);
|
||||
font-size: 12px;
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.popout:focus-visible,
|
||||
.icon-button:focus-visible,
|
||||
.capsule:focus-visible {
|
||||
outline: 2px solid var(--dsw-alias-border-focus);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.dot[data-state="active"] {
|
||||
animation: none;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
// Observation overlay: the default in-app carrier for live session watching.
|
||||
// Expanded = draggable/resizable floating card (bottom-right); collapsed = a
|
||||
// status capsule; "pop out" upgrades the same content to a native Document
|
||||
// PiP window (user gesture required by the browser). Pure client state —
|
||||
// position/size/collapse live for the page lifetime only.
|
||||
|
||||
import {
|
||||
type PointerEvent as ReactPointerEvent,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
useSyncExternalStore,
|
||||
} from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import type { SessionObservation } from "../observation";
|
||||
import css from "./ObservationOverlay.module.css";
|
||||
import type { ObservationClientStore } from "./observation-store";
|
||||
|
||||
/** Minimal Document PiP surface (TS lib.dom lacks it). */
|
||||
interface DocumentPip {
|
||||
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
|
||||
}
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface Size {
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SIZE: Size = { w: 320, h: 240 };
|
||||
const MIN_SIZE: Size = { w: 240, h: 180 };
|
||||
const EDGE_MARGIN = 16;
|
||||
|
||||
function pipApi(): DocumentPip | undefined {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
|
||||
}
|
||||
|
||||
function clampSize(size: Size, viewport: Size): Size {
|
||||
const maxW = viewport.w * 0.8;
|
||||
const maxH = viewport.h * 0.8;
|
||||
return {
|
||||
w: Math.min(Math.max(size.w, MIN_SIZE.w), maxW),
|
||||
h: Math.min(Math.max(size.h, MIN_SIZE.h), maxH),
|
||||
};
|
||||
}
|
||||
|
||||
function clampPos(pos: Point, size: Size, viewport: Size): Point {
|
||||
return {
|
||||
x: Math.min(Math.max(pos.x, 0), Math.max(0, viewport.w - size.w)),
|
||||
y: Math.min(Math.max(pos.y, 0), Math.max(0, viewport.h - size.h)),
|
||||
};
|
||||
}
|
||||
|
||||
function formatElapsed(sinceMs: number, nowMs: number): string {
|
||||
const total = Math.max(0, Math.floor((nowMs - sinceMs) / 1000));
|
||||
const mm = String(Math.floor(total / 60)).padStart(2, "0");
|
||||
const ss = String(total % 60).padStart(2, "0");
|
||||
return `${mm}:${ss}`;
|
||||
}
|
||||
|
||||
/** Focus = most recently active session (action or latest state change). */
|
||||
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
|
||||
if (sessions.length === 0) return undefined;
|
||||
return sessions.reduce((a, b) => (a.since >= b.since ? a : b));
|
||||
}
|
||||
|
||||
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
|
||||
if (obs.lastError !== undefined && obs.action === "idle") return "error";
|
||||
return obs.action === "idle" ? "idle" : "active";
|
||||
}
|
||||
|
||||
/** The floating card / PiP shared content. */
|
||||
function OverlayBody(props: {
|
||||
store: ObservationClientStore;
|
||||
focus: SessionObservation | undefined;
|
||||
now: number;
|
||||
onPopOut?: (() => void) | undefined;
|
||||
onCollapse?: (() => void) | undefined;
|
||||
inPip: boolean;
|
||||
onHeaderPointerDown?: (event: ReactPointerEvent<HTMLDivElement>) => void;
|
||||
}) {
|
||||
const { store, focus, now, onPopOut, onCollapse, inPip, onHeaderPointerDown } = props;
|
||||
const [interrupting, setInterrupting] = useState(false);
|
||||
const [hintOpen, setHintOpen] = useState(false);
|
||||
const [hintSeen, setHintSeen] = useState(false);
|
||||
|
||||
const thumbId = focus?.thumbnailAttachmentId;
|
||||
useEffect(() => {
|
||||
store.ensureThumbnail(thumbId);
|
||||
}, [store, thumbId]);
|
||||
const thumb = thumbId !== undefined ? store.getSnapshot().thumbnails[thumbId] : undefined;
|
||||
|
||||
const canInterrupt = !interrupting && focus !== undefined && focus.action !== "idle";
|
||||
const onInterrupt = (): void => {
|
||||
if (!canInterrupt || focus === undefined) return;
|
||||
setHintSeen(true);
|
||||
setHintOpen(false);
|
||||
setInterrupting(true);
|
||||
void store.interrupt(focus.sessionId).finally(() => setInterrupting(false));
|
||||
};
|
||||
|
||||
const actionText =
|
||||
focus === undefined ? "no session" : focus.action === "idle" ? "idle" : focus.action;
|
||||
const elapsed = focus !== undefined ? formatElapsed(focus.since, now) : "00:00";
|
||||
const state = focus !== undefined ? statusOf(focus) : "idle";
|
||||
|
||||
return (
|
||||
<div className={css.body} data-state={state} data-in-pip={inPip || undefined}>
|
||||
<div
|
||||
className={css.header}
|
||||
data-testid="obs-header"
|
||||
onPointerDown={onHeaderPointerDown}
|
||||
role="presentation"
|
||||
>
|
||||
<span className={css.dot} data-state={state} aria-hidden />
|
||||
<span className={css["status-text"]}>
|
||||
{focus === undefined
|
||||
? "browser sessions"
|
||||
: `${focus.sessionId} · ${actionText} · ${elapsed}`}
|
||||
</span>
|
||||
{onCollapse !== undefined ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css["icon-button"]}
|
||||
aria-label="Collapse"
|
||||
onClick={onCollapse}
|
||||
>
|
||||
—
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
<div className={css.stage}>
|
||||
{thumb?.status === "ready" && thumb.url !== undefined ? (
|
||||
<img
|
||||
key={thumb.url}
|
||||
className={css.thumb}
|
||||
src={thumb.url}
|
||||
alt={`session ${focus?.sessionId ?? ""} view`}
|
||||
/>
|
||||
) : (
|
||||
<div className={css.placeholder}>
|
||||
{thumb?.status === "error" ? "frame unavailable" : "waiting for page"}
|
||||
</div>
|
||||
)}
|
||||
{thumb?.status === "error" ? (
|
||||
<span className={css.badge} aria-label="thumbnail failed">
|
||||
!
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<div className={css.actions}>
|
||||
<span className={css["interrupt-wrap"]}>
|
||||
<button
|
||||
type="button"
|
||||
className={css.interrupt}
|
||||
disabled={!canInterrupt}
|
||||
aria-label="Interrupt the current browser action"
|
||||
onPointerEnter={() => {
|
||||
if (!hintSeen) setHintOpen(true);
|
||||
}}
|
||||
onPointerLeave={() => setHintOpen(false)}
|
||||
onClick={onInterrupt}
|
||||
>
|
||||
{interrupting ? "Interrupting…" : "Interrupt"}
|
||||
</button>
|
||||
{hintOpen && !hintSeen ? (
|
||||
<span className={css.hint} role="tooltip">
|
||||
Interrupt stops only the current browser action — the agent run continues (use the
|
||||
chat Stop button to halt the run).
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
{onPopOut !== undefined ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css.popout}
|
||||
aria-label="Pop out into a mini window"
|
||||
onClick={onPopOut}
|
||||
>
|
||||
Pop out
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Clone the host document's style/link nodes into a PiP window. */
|
||||
function cloneStylesInto(pipWindow: Window): void {
|
||||
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
|
||||
pipWindow.document.head.appendChild(node.cloneNode(true));
|
||||
}
|
||||
}
|
||||
|
||||
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
|
||||
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
|
||||
useEffect(() => {
|
||||
store.start();
|
||||
return () => store.stop();
|
||||
}, [store]);
|
||||
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [pos, setPos] = useState<Point | null>(null);
|
||||
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
|
||||
const [pipWindow, setPipWindow] = useState<Window | null>(null);
|
||||
const dragRef = useRef<{
|
||||
kind: "move" | "resize";
|
||||
startX: number;
|
||||
startY: number;
|
||||
base: Point & Size;
|
||||
} | null>(null);
|
||||
|
||||
// Elapsed-time ticker: 1s while anything is active.
|
||||
const focus = focusOf(snapshot.sessions);
|
||||
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
useEffect(() => {
|
||||
if (!anyActive) return;
|
||||
const timer = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => clearInterval(timer);
|
||||
}, [anyActive]);
|
||||
|
||||
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
|
||||
|
||||
const onPointerMove = useCallback((event: PointerEvent) => {
|
||||
const drag = dragRef.current;
|
||||
if (drag === null) return;
|
||||
const dx = event.clientX - drag.startX;
|
||||
const dy = event.clientY - drag.startY;
|
||||
if (drag.kind === "move") {
|
||||
setPos(
|
||||
clampPos(
|
||||
{ x: drag.base.x + dx, y: drag.base.y + dy },
|
||||
{ w: drag.base.w, h: drag.base.h },
|
||||
viewport(),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
setSize(clampSize({ w: drag.base.w + dx, h: drag.base.h + dy }, viewport()));
|
||||
}
|
||||
}, []);
|
||||
const onPointerUp = useCallback(() => {
|
||||
dragRef.current = null;
|
||||
document.removeEventListener("pointermove", onPointerMove);
|
||||
document.removeEventListener("pointerup", onPointerUp);
|
||||
}, [onPointerMove]);
|
||||
|
||||
const beginDrag = (kind: "move" | "resize") => (event: ReactPointerEvent) => {
|
||||
event.preventDefault();
|
||||
const rect = (
|
||||
event.currentTarget.closest("[data-obs-card]") as HTMLElement | null
|
||||
)?.getBoundingClientRect();
|
||||
const base = {
|
||||
x: rect?.left ?? pos?.x ?? viewport().w - size.w - EDGE_MARGIN,
|
||||
y: rect?.top ?? pos?.y ?? viewport().h - size.h - EDGE_MARGIN,
|
||||
w: rect?.width ?? size.w,
|
||||
h: rect?.height ?? size.h,
|
||||
};
|
||||
dragRef.current = { kind, startX: event.clientX, startY: event.clientY, base };
|
||||
if (kind === "move") setPos({ x: base.x, y: base.y });
|
||||
document.addEventListener("pointermove", onPointerMove);
|
||||
document.addEventListener("pointerup", onPointerUp);
|
||||
};
|
||||
|
||||
const popOut = async (): Promise<void> => {
|
||||
const pip = pipApi();
|
||||
if (pip === undefined) return;
|
||||
const win = await pip.requestWindow({ width: size.w, height: size.h });
|
||||
cloneStylesInto(win);
|
||||
win.addEventListener("pagehide", () => setPipWindow(null));
|
||||
setPipWindow(win);
|
||||
};
|
||||
|
||||
// Hidden while no owned session exists (and no PiP is up).
|
||||
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
|
||||
|
||||
const body = (
|
||||
<OverlayBody
|
||||
store={store}
|
||||
focus={focus}
|
||||
now={now}
|
||||
inPip={pipWindow !== null}
|
||||
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
|
||||
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
|
||||
onHeaderPointerDown={pipWindow === null ? beginDrag("move") : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
if (pipWindow !== null) {
|
||||
return createPortal(body, pipWindow.document.body);
|
||||
}
|
||||
|
||||
if (collapsed) {
|
||||
const state = focus !== undefined ? statusOf(focus) : "idle";
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
className={css.capsule}
|
||||
data-state={state}
|
||||
aria-label="Expand browser observation overlay"
|
||||
onClick={() => setCollapsed(false)}
|
||||
>
|
||||
<span className={css.dot} data-state={state} aria-hidden />
|
||||
<span className={css["capsule-text"]}>
|
||||
{snapshot.sessions.length} session{snapshot.sessions.length === 1 ? "" : "s"}
|
||||
{focus !== undefined && focus.action !== "idle"
|
||||
? ` · ${focus.action} · ${formatElapsed(focus.since, now)}`
|
||||
: ""}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
const style: React.CSSProperties =
|
||||
pos !== null
|
||||
? { left: pos.x, top: pos.y, width: size.w, height: size.h }
|
||||
: { right: EDGE_MARGIN, bottom: EDGE_MARGIN, width: size.w, height: size.h };
|
||||
return (
|
||||
<div className={css.card} style={style} data-obs-card data-testid="obs-card">
|
||||
{body}
|
||||
<div
|
||||
className={css["resize-handle"]}
|
||||
data-testid="obs-resize"
|
||||
aria-label="Resize overlay"
|
||||
role="separator"
|
||||
onPointerDown={beginDrag("resize")}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,21 +1,20 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: registers the `browser_screenshot`
|
||||
* keyed toolview (`tool.call.toolview` slot) so screenshot results render the
|
||||
* captured image inline in the Web UI instead of a bare JSON block. Every
|
||||
* other browser_* tool keeps the stock terminal card.
|
||||
*
|
||||
* The image bytes never enter the session log: the view resolves the durable
|
||||
* attachment reference through the client session's authorized
|
||||
* `readAttachment` RPC and hands a blob URL to the shared MessageImage atom.
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt) floating
|
||||
* over the shell via the `shell.overlay` seat.
|
||||
*/
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
|
||||
import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { createElement } from "react";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
/** Required services: the slot registry plus session-scoped attachment reads. */
|
||||
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
|
||||
export const inject = ["slots", "sessions"];
|
||||
|
||||
/** Resolve one durable image attachment into a browser blob URL. */
|
||||
@@ -41,7 +40,29 @@ async function loadSessionImage(
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed view with its session-bound loader.
|
||||
* Thumbnail loader for the overlay: the observation overlay is root-scoped,
|
||||
* so borrow the current (or first) bound session for the authorized
|
||||
* attachment read. Without any bound session the frame simply fails and the
|
||||
* overlay keeps the previous one.
|
||||
*/
|
||||
function overlayImageLoader(sessions: ISessions): (attachmentId: string) => Promise<string> {
|
||||
return async (attachmentId) => {
|
||||
const list = sessions.list.getSnapshot();
|
||||
const sessionId = list.current ?? list.ids[0];
|
||||
if (sessionId === undefined) throw new Error("no bound session for attachment read");
|
||||
const session = sessions.binding(sessionId)?.session;
|
||||
if (session === undefined) throw new Error(`session "${String(sessionId)}" is not bound`);
|
||||
const result = await session.readAttachment(attachmentId as ImageAttachmentRef["attachmentId"]);
|
||||
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
|
||||
const bytes = Uint8Array.from(result.value.data);
|
||||
return URL.createObjectURL(
|
||||
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed toolview and the observation overlay.
|
||||
* @param ctx - client root context.
|
||||
*/
|
||||
export function apply(ctx: ClientContext): void {
|
||||
@@ -57,4 +78,14 @@ export function apply(ctx: ClientContext): void {
|
||||
}),
|
||||
),
|
||||
);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: (url, init) => fetch(url, init),
|
||||
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
||||
loadImage: overlayImageLoader(sessions),
|
||||
});
|
||||
ctx.slots.inject("shell.overlay", () =>
|
||||
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
||||
createElement(ObservationOverlay, { store }),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Client-side data layer for the observation overlay: initial state fetch,
|
||||
* SSE increment stream, on-demand thumbnail blob loading through the
|
||||
* session-authorized readAttachment path, and the interrupt call. All I/O is
|
||||
* injected so tests never touch a network.
|
||||
*/
|
||||
|
||||
import type { ObservationEvent, SessionObservation } from "../observation";
|
||||
|
||||
export interface EventSourceLike {
|
||||
onmessage: ((event: { data: string }) => void) | null;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface ObservationClientDeps {
|
||||
fetchFn: (
|
||||
url: string,
|
||||
init?: { method?: string; body?: string },
|
||||
) => Promise<{
|
||||
ok: boolean;
|
||||
json(): Promise<unknown>;
|
||||
}>;
|
||||
eventSourceFactory: (url: string) => EventSourceLike;
|
||||
/** Resolve one attachment id to a displayable blob URL. */
|
||||
loadImage: (attachmentId: string) => Promise<string>;
|
||||
}
|
||||
|
||||
export interface ThumbnailState {
|
||||
status: "loading" | "ready" | "error";
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface OverlaySnapshot {
|
||||
readonly sessions: readonly SessionObservation[];
|
||||
readonly connected: boolean;
|
||||
readonly thumbnails: Readonly<Record<string, ThumbnailState>>;
|
||||
}
|
||||
|
||||
const STATE_URL = "/bsk-observation/state";
|
||||
const EVENTS_URL = "/bsk-observation/events";
|
||||
const INTERRUPT_URL = "/bsk-observation/interrupt";
|
||||
|
||||
export class ObservationClientStore {
|
||||
private sessions = new Map<string, SessionObservation>();
|
||||
private thumbs = new Map<string, ThumbnailState>();
|
||||
private listeners = new Set<() => void>();
|
||||
private events: EventSourceLike | undefined;
|
||||
private snapshot: OverlaySnapshot = { sessions: [], connected: false, thumbnails: {} };
|
||||
private started = false;
|
||||
|
||||
constructor(private readonly deps: ObservationClientDeps) {}
|
||||
|
||||
subscribe = (listener: () => void): (() => void) => {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
};
|
||||
|
||||
getSnapshot = (): OverlaySnapshot => this.snapshot;
|
||||
|
||||
private publish(): void {
|
||||
this.snapshot = {
|
||||
sessions: [...this.sessions.values()],
|
||||
connected: this.events !== undefined,
|
||||
thumbnails: Object.fromEntries(this.thumbs),
|
||||
};
|
||||
for (const listener of [...this.listeners]) listener();
|
||||
}
|
||||
|
||||
/** Initial fetch + SSE subscription. Idempotent. */
|
||||
start(): void {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
void this.deps
|
||||
.fetchFn(STATE_URL)
|
||||
.then(async (res) => {
|
||||
if (!res.ok) return;
|
||||
const body = (await res.json()) as { sessions?: SessionObservation[] };
|
||||
this.sessions = new Map((body.sessions ?? []).map((s) => [s.sessionId, s]));
|
||||
this.publish();
|
||||
})
|
||||
.catch(() => {});
|
||||
const events = this.deps.eventSourceFactory(EVENTS_URL);
|
||||
events.onmessage = (message) => {
|
||||
let event: ObservationEvent;
|
||||
try {
|
||||
event = JSON.parse(message.data) as ObservationEvent;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
this.apply(event);
|
||||
};
|
||||
this.events = events;
|
||||
this.publish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.events?.close();
|
||||
this.events = undefined;
|
||||
this.started = false;
|
||||
for (const thumb of this.thumbs.values()) {
|
||||
if (thumb.url !== undefined && typeof URL.revokeObjectURL === "function") {
|
||||
URL.revokeObjectURL(thumb.url);
|
||||
}
|
||||
}
|
||||
this.thumbs.clear();
|
||||
this.sessions.clear();
|
||||
this.publish();
|
||||
}
|
||||
|
||||
private apply(event: ObservationEvent): void {
|
||||
if (event.type === "reset") {
|
||||
this.sessions.clear();
|
||||
} else if (event.type === "remove" && event.session !== undefined) {
|
||||
this.sessions.delete(event.session.sessionId);
|
||||
} else if (event.type === "upsert" && event.session !== undefined) {
|
||||
this.sessions.set(event.session.sessionId, event.session);
|
||||
}
|
||||
this.publish();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a thumbnail load is in flight for one attachment reference. New
|
||||
* frames replace the old URL; failures keep the last good frame (the caller
|
||||
* renders `status: 'error'` as a small badge over the old image).
|
||||
*/
|
||||
ensureThumbnail(attachmentId: string | undefined): void {
|
||||
if (attachmentId === undefined || this.thumbs.has(attachmentId)) return;
|
||||
this.thumbs.set(attachmentId, { status: "loading" });
|
||||
this.publish();
|
||||
this.deps.loadImage(attachmentId).then(
|
||||
(url) => {
|
||||
this.thumbs.set(attachmentId, { status: "ready", url });
|
||||
this.publish();
|
||||
},
|
||||
() => {
|
||||
this.thumbs.set(attachmentId, { status: "error" });
|
||||
this.publish();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Interrupt the current or named session's in-flight call. */
|
||||
async interrupt(sessionId?: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await this.deps.fetchFn(INTERRUPT_URL, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(sessionId === undefined ? {} : { sessionId }),
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
const body = (await res.json()) as { interrupted?: boolean };
|
||||
return body.interrupted === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
// @vitest-environment happy-dom
|
||||
// ObservationOverlay: visibility lifecycle, focus view, interrupt interaction,
|
||||
// one-time hint, drag move/resize with clamps, collapse capsule, and PiP
|
||||
// upgrade/fallback (mocked documentPictureInPicture). NOTE: use RTL's waitFor
|
||||
// (act-flushing), not vi.waitFor, when asserting store-driven UI updates.
|
||||
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { ObservationOverlay } from "../../src/client/ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import type { SessionObservation } from "../../src/observation";
|
||||
|
||||
const BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: Date.now() - 7000 };
|
||||
|
||||
interface Harness {
|
||||
store: ObservationClientStore;
|
||||
es: () => EventSourceLike;
|
||||
push: (sessions: SessionObservation[]) => void;
|
||||
fetches: { url: string; init?: { body?: string } }[];
|
||||
loadImage: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
function makeHarness(initial: SessionObservation[]): Harness {
|
||||
const fetches: Harness["fetches"] = [];
|
||||
let es: EventSourceLike | undefined;
|
||||
let current = initial;
|
||||
const loadImage = vi.fn(async (id: string) => `blob:${id}`);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: async (url: string, init?: { body?: string }) => {
|
||||
fetches.push({ url, init });
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: current }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
},
|
||||
eventSourceFactory: () => {
|
||||
es = { onmessage: null, close: vi.fn() };
|
||||
return es;
|
||||
},
|
||||
loadImage,
|
||||
});
|
||||
return {
|
||||
store,
|
||||
es: () => {
|
||||
if (es === undefined) throw new Error("no EventSource yet");
|
||||
return es;
|
||||
},
|
||||
push: (sessions) => {
|
||||
current = sessions;
|
||||
for (const s of sessions) {
|
||||
es?.onmessage?.({ data: JSON.stringify({ type: "upsert", session: s }) });
|
||||
}
|
||||
if (sessions.length === 0) es?.onmessage?.({ data: JSON.stringify({ type: "reset" }) });
|
||||
},
|
||||
fetches,
|
||||
loadImage,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe("ObservationOverlay", () => {
|
||||
beforeEach(() => {
|
||||
delete (window as unknown as Record<string, unknown>).documentPictureInPicture;
|
||||
});
|
||||
|
||||
it("renders nothing with no sessions, appears when one starts", async () => {
|
||||
const h = makeHarness([]);
|
||||
const { container } = render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.es).not.toThrow());
|
||||
expect(container.firstChild).toBeNull();
|
||||
h.push([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
await screen.findByText(/s1 · idle/);
|
||||
expect(screen.getByTestId("obs-card")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows the status row and a placeholder without a thumbnail", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking · 00:0/);
|
||||
expect(screen.getByText("waiting for page")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("loads and renders the thumbnail through the store", async () => {
|
||||
const h = makeHarness([{ ...BUSY, thumbnailAttachmentId: "att-9" }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.loadImage).toHaveBeenCalledWith("att-9"));
|
||||
await screen.findByRole("img", { name: "session s1 view" });
|
||||
});
|
||||
|
||||
it("interrupt: disabled while idle, active call posts and shows progress", async () => {
|
||||
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const button = await screen.findByRole("button", { name: /Interrupt the current/ });
|
||||
expect(button).toHaveProperty("disabled", true);
|
||||
h.push([BUSY]);
|
||||
await waitFor(() => expect(button).toHaveProperty("disabled", false));
|
||||
fireEvent.click(button);
|
||||
await waitFor(() =>
|
||||
expect(h.fetches.some((f) => f.url === "/bsk-observation/interrupt")).toBe(true),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the semantics hint once", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const button = await screen.findByRole("button", { name: /Interrupt the current/ });
|
||||
fireEvent.pointerEnter(button);
|
||||
await screen.findByRole("tooltip");
|
||||
fireEvent.pointerLeave(button);
|
||||
fireEvent.click(button);
|
||||
fireEvent.pointerEnter(button);
|
||||
expect(screen.queryByRole("tooltip")).toBeNull();
|
||||
});
|
||||
|
||||
it("collapses to a capsule and expands back", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Collapse" }));
|
||||
const capsule = await screen.findByRole("button", {
|
||||
name: "Expand browser observation overlay",
|
||||
});
|
||||
expect(capsule.textContent).toContain("clicking");
|
||||
fireEvent.click(capsule);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
});
|
||||
|
||||
it("resizes within min/max clamps", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const card = await screen.findByTestId("obs-card");
|
||||
expect(Number.parseFloat(card.style.width)).toBe(320);
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-resize"), { clientX: 100, clientY: 100 });
|
||||
fireEvent.pointerMove(document, { clientX: 40, clientY: 40 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(Number.parseFloat(card.style.width)).toBe(240); // min width clamp
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-resize"), { clientX: 100, clientY: 100 });
|
||||
fireEvent.pointerMove(document, { clientX: 5000, clientY: 5000 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(Number.parseFloat(card.style.width)).toBe(window.innerWidth * 0.8);
|
||||
});
|
||||
|
||||
it("moves the card with header drags, clamped to the viewport", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const card = await screen.findByTestId("obs-card");
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-header"), { clientX: 50, clientY: 50 });
|
||||
fireEvent.pointerMove(document, { clientX: -500, clientY: -500 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(card.style.left).toBe("0px");
|
||||
expect(card.style.top).toBe("0px");
|
||||
});
|
||||
|
||||
it("hides Pop out when Document PiP is unsupported", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
expect(screen.queryByRole("button", { name: /Pop out/ })).toBeNull();
|
||||
});
|
||||
|
||||
it("pops out with the card's current size and falls back on pagehide", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const pipDoc = document.implementation.createHTMLDocument("pip");
|
||||
const listeners = new Map<string, (() => void)[]>();
|
||||
const requestWindow = vi.fn(async (options?: { width?: number; height?: number }) => {
|
||||
void options;
|
||||
return {
|
||||
document: pipDoc,
|
||||
addEventListener: (name: string, fn: () => void) => {
|
||||
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
|
||||
},
|
||||
} as unknown as Window;
|
||||
});
|
||||
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const popout = await screen.findByRole("button", { name: /Pop out/ });
|
||||
fireEvent.click(popout);
|
||||
await waitFor(() => expect(requestWindow).toHaveBeenCalledWith({ width: 320, height: 240 }));
|
||||
// Content now renders inside the PiP document.
|
||||
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
|
||||
// Closing the PiP returns to the in-page card with state preserved.
|
||||
for (const fn of listeners.get("pagehide") ?? []) fn();
|
||||
await screen.findByTestId("obs-card");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
// ObservationClientStore: initial fetch + SSE application, thumbnail loading
|
||||
// lifecycle, interrupt wire shape. All I/O faked.
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import type { ObservationEvent, SessionObservation } from "../../src/observation";
|
||||
|
||||
const OBS_IDLE: SessionObservation = { sessionId: "s1", action: "idle", since: 1000 };
|
||||
const OBS_BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: 2000 };
|
||||
|
||||
function harness(state: SessionObservation[] = []) {
|
||||
const fetches: { url: string; init?: { method?: string; body?: string } }[] = [];
|
||||
const esInstances: EventSourceLike[] = [];
|
||||
const fetchFn = vi.fn(async (url: string, init?: { method?: string; body?: string }) => {
|
||||
fetches.push({ url, init });
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: state }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
});
|
||||
const eventSourceFactory = (url: string) => {
|
||||
const es: EventSourceLike = { onmessage: null, close: vi.fn() };
|
||||
esInstances.push(es);
|
||||
return es;
|
||||
};
|
||||
const loadImage = vi.fn(async (id: string) => `blob:url-${id}`);
|
||||
const store = new ObservationClientStore({ fetchFn, eventSourceFactory, loadImage });
|
||||
return { store, fetches, esInstances, loadImage };
|
||||
}
|
||||
|
||||
function emit(es: EventSourceLike, event: ObservationEvent): void {
|
||||
es.onmessage?.({ data: JSON.stringify(event) });
|
||||
}
|
||||
|
||||
describe("ObservationClientStore", () => {
|
||||
it("loads the initial state and applies SSE increments", async () => {
|
||||
const { store, esInstances } = harness([OBS_IDLE]);
|
||||
const seen: number[] = [];
|
||||
store.subscribe(() => seen.push(store.getSnapshot().sessions.length));
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
expect(esInstances).toHaveLength(1);
|
||||
emit(esInstances[0], { type: "upsert", session: OBS_BUSY });
|
||||
expect(store.getSnapshot().sessions[0].action).toBe("clicking");
|
||||
emit(esInstances[0], {
|
||||
type: "upsert",
|
||||
session: { sessionId: "s2", action: "idle", since: 3 },
|
||||
});
|
||||
expect(store.getSnapshot().sessions).toHaveLength(2);
|
||||
emit(esInstances[0], {
|
||||
type: "remove",
|
||||
session: { sessionId: "s2", action: "idle", since: 0 },
|
||||
});
|
||||
expect(store.getSnapshot().sessions).toHaveLength(1);
|
||||
emit(esInstances[0], { type: "reset" });
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
expect(seen.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("ignores malformed SSE frames", () => {
|
||||
const { store, esInstances } = harness([]);
|
||||
store.start();
|
||||
esInstances[0].onmessage?.({ data: "not json" });
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("loads each thumbnail once and reports ready", async () => {
|
||||
const { store, loadImage } = harness();
|
||||
store.ensureThumbnail("att-1");
|
||||
store.ensureThumbnail("att-1");
|
||||
expect(loadImage).toHaveBeenCalledTimes(1);
|
||||
await vi.waitFor(() =>
|
||||
expect(store.getSnapshot().thumbnails["att-1"]).toEqual({
|
||||
status: "ready",
|
||||
url: "blob:url-att-1",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("marks failed thumbnail loads as error", async () => {
|
||||
const { store, loadImage } = harness();
|
||||
loadImage.mockRejectedValueOnce(new Error("nope"));
|
||||
store.ensureThumbnail("att-bad");
|
||||
await vi.waitFor(() => expect(store.getSnapshot().thumbnails["att-bad"].status).toBe("error"));
|
||||
});
|
||||
|
||||
it("posts interrupt with and without a session id", async () => {
|
||||
const { store, fetches } = harness();
|
||||
expect(await store.interrupt()).toBe(true);
|
||||
expect(await store.interrupt("s1")).toBe(true);
|
||||
const calls = fetches.filter((f) => f.url === "/bsk-observation/interrupt");
|
||||
expect(JSON.parse(calls[0].init?.body ?? "")).toEqual({});
|
||||
expect(JSON.parse(calls[1].init?.body ?? "")).toEqual({ sessionId: "s1" });
|
||||
});
|
||||
|
||||
it("returns false when interrupt fails", async () => {
|
||||
const failing = new ObservationClientStore({
|
||||
fetchFn: async () => ({ ok: false, json: async () => ({}) }),
|
||||
eventSourceFactory: () => ({ onmessage: null, close: () => {} }),
|
||||
loadImage: async () => "blob:x",
|
||||
});
|
||||
expect(await failing.interrupt("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("stop closes the stream and clears state", async () => {
|
||||
const { store, esInstances } = harness([OBS_IDLE]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.stop();
|
||||
expect(esInstances[0].close).toHaveBeenCalled();
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -132,20 +132,23 @@ describe("ScreenshotToolView", () => {
|
||||
});
|
||||
|
||||
describe("client plugin registration", () => {
|
||||
it("registers only the browser_screenshot keyed toolview", () => {
|
||||
const registrations: { name: string; key?: string }[] = [];
|
||||
it("registers the browser_screenshot toolview and the shell observation overlay", () => {
|
||||
const registrations: { name: string; key?: string; id?: string }[] = [];
|
||||
const sessions = { binding: () => undefined };
|
||||
const ctx = {
|
||||
get: (key: string) => (key === "sessions" ? sessions : undefined),
|
||||
slots: {
|
||||
inject: (_name: string, fn: () => unknown) => fn(),
|
||||
register: (slot: { name: string; key?: string }, view: unknown) => {
|
||||
register: (slot: { name: string; key?: string; id?: string }, view: unknown) => {
|
||||
registrations.push(slot);
|
||||
expect(typeof view).toBe("function");
|
||||
},
|
||||
},
|
||||
};
|
||||
apply(ctx as never);
|
||||
expect(registrations).toEqual([{ name: "tool.call.toolview", key: "browser_screenshot" }]);
|
||||
expect(registrations).toEqual([
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "shell.overlay", id: "bsk-observation" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
Generated
+68
-5
@@ -99,6 +99,9 @@ importers:
|
||||
'@deepseek-ai/dsh-client-ui-attachment':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
'@deepseek-ai/dsh-client-ui-layout':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-client-runtime@0.1.0-rc.6(6ef9bea1ffb02b7be9333a18bfeaf17e))(@deepseek-ai/dsh-client-ui-slots@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-client-ui-theme@0.1.0-rc.6(83594937fb9eb063ee56c7bdd666e663))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(react@18.3.1)
|
||||
'@deepseek-ai/dsh-client-ui-primitives':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
@@ -116,13 +119,16 @@ importers:
|
||||
version: 3.18.1
|
||||
'@testing-library/react':
|
||||
specifier: ^16.1.0
|
||||
version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
'@types/node':
|
||||
specifier: ^22.10.0
|
||||
version: 22.20.1
|
||||
'@types/react':
|
||||
specifier: ~18.3.1
|
||||
version: 18.3.31
|
||||
'@types/react-dom':
|
||||
specifier: ^18.3.0
|
||||
version: 18.3.7(@types/react@18.3.31)
|
||||
happy-dom:
|
||||
specifier: ^15.11.7
|
||||
version: 15.11.7
|
||||
@@ -516,6 +522,16 @@ packages:
|
||||
'@deepseek-ai/dsh-invariants': ^0.1.0-rc.6
|
||||
react: ^18.2.0
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-layout@0.1.0-rc.6':
|
||||
resolution: {integrity: sha512-JpQ5rzySpK7molQBfMqrcBCGDWYQngqZmFuFqhcxecIc2n5mBYVP0F5UbGITgxo+D7sDOy78RA1PWqZ/5b/g0g==}
|
||||
peerDependencies:
|
||||
'@deepseek-ai/cordis': ^4.0.1
|
||||
'@deepseek-ai/dsh-client-runtime': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-ui-slots': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-ui-theme': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-invariants': ^0.1.0-rc.6
|
||||
react: ^18.2.0
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-primitives@0.1.0-rc.6':
|
||||
resolution: {integrity: sha512-vZ9/V5pXYbkumLEOHagEjzSWvVxX+CtXZ13+iNz3BnqDVsy0HYLgnrWwcLLPHqHLUmww9NlxXe9husoS7o/zyw==}
|
||||
peerDependencies:
|
||||
@@ -541,6 +557,21 @@ packages:
|
||||
'@deepseek-ai/cordis': ^4.0.1
|
||||
'@deepseek-ai/dsh-invariants': ^0.1.0-rc.6
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-theme@0.1.0-rc.6':
|
||||
resolution: {integrity: sha512-Wu+bvnuti/gLA+t5a2cWUMQJ5UCqxt6oEK+OJiJ68gN0ixs2skpaN0nFdFoY2exC5KByXrNlN1rRrD+FsZSBLA==}
|
||||
peerDependencies:
|
||||
'@deepseek-ai/cordis': ^4.0.1
|
||||
'@deepseek-ai/dsh-api-remotes': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-connection': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-locale': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-runtime': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-ui-primitives': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-ui-settings': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-client-ui-slots': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-host-webserver': ^0.1.0-rc.6
|
||||
'@deepseek-ai/dsh-invariants': ^0.1.0-rc.6
|
||||
react: ^18.2.0
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-tool@0.1.0-rc.6':
|
||||
resolution: {integrity: sha512-pjNksBDDwWdRTb+YiZ1DSXWPipfw7C89B4bydopjwBtacOmx+rZH8Z9i8iFpmMwD5IFVd9ZJzwRR44sYkiun5g==}
|
||||
peerDependencies:
|
||||
@@ -1682,6 +1713,11 @@ packages:
|
||||
'@types/prop-types@15.7.15':
|
||||
resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==}
|
||||
|
||||
'@types/react-dom@18.3.7':
|
||||
resolution: {integrity: sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==}
|
||||
peerDependencies:
|
||||
'@types/react': ^18.0.0
|
||||
|
||||
'@types/react-dom@19.2.3':
|
||||
resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==}
|
||||
peerDependencies:
|
||||
@@ -4417,6 +4453,15 @@ snapshots:
|
||||
clsx: 2.1.1
|
||||
react: 18.3.1
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-layout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-client-runtime@0.1.0-rc.6(6ef9bea1ffb02b7be9333a18bfeaf17e))(@deepseek-ai/dsh-client-ui-slots@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-client-ui-theme@0.1.0-rc.6(83594937fb9eb063ee56c7bdd666e663))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(react@18.3.1)':
|
||||
dependencies:
|
||||
'@deepseek-ai/cordis': 4.0.1(@deepseek-ai/cordis-plugin-include@1.0.6)(@deepseek-ai/cordis-plugin-loader@1.0.2)
|
||||
'@deepseek-ai/dsh-client-runtime': 0.1.0-rc.6(6ef9bea1ffb02b7be9333a18bfeaf17e)
|
||||
'@deepseek-ai/dsh-client-ui-slots': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
'@deepseek-ai/dsh-client-ui-theme': 0.1.0-rc.6(83594937fb9eb063ee56c7bdd666e663)
|
||||
'@deepseek-ai/dsh-invariants': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)
|
||||
react: 18.3.1
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-primitives@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))':
|
||||
dependencies:
|
||||
'@deepseek-ai/cordis': 4.0.1(@deepseek-ai/cordis-plugin-include@1.0.6)(@deepseek-ai/cordis-plugin-loader@1.0.2)
|
||||
@@ -4461,6 +4506,25 @@ snapshots:
|
||||
'@deepseek-ai/cordis': 4.0.1(@deepseek-ai/cordis-plugin-include@1.0.6)(@deepseek-ai/cordis-plugin-loader@1.0.2)
|
||||
'@deepseek-ai/dsh-invariants': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-theme@0.1.0-rc.6(83594937fb9eb063ee56c7bdd666e663)':
|
||||
dependencies:
|
||||
'@deepseek-ai/cordis': 4.0.1(@deepseek-ai/cordis-plugin-include@1.0.6)(@deepseek-ai/cordis-plugin-loader@1.0.2)
|
||||
'@deepseek-ai/dsh-api-remotes': 0.1.0-rc.6(83d9c5c9b8349236135a0a0b0d2ee7ec)
|
||||
'@deepseek-ai/dsh-client-connection': 0.1.0-rc.6(74b1e373a98551f8dcfec393a9dea855)
|
||||
'@deepseek-ai/dsh-client-locale': 0.1.0-rc.6(ce9105fc2b8666e57db2f9de071e4b25)
|
||||
'@deepseek-ai/dsh-client-runtime': 0.1.0-rc.6(6ef9bea1ffb02b7be9333a18bfeaf17e)
|
||||
'@deepseek-ai/dsh-client-ui-primitives': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
'@deepseek-ai/dsh-client-ui-settings': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-api-remotes@0.1.0-rc.6(83d9c5c9b8349236135a0a0b0d2ee7ec))(@deepseek-ai/dsh-client-connection@0.1.0-rc.6)(@deepseek-ai/dsh-client-runtime@0.1.0-rc.6(6ef9bea1ffb02b7be9333a18bfeaf17e))(@deepseek-ai/dsh-client-schema-form@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-client-ui-slots@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-settings@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/schemastery@3.18.1))(react@18.3.1)
|
||||
'@deepseek-ai/dsh-client-ui-slots': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
'@deepseek-ai/dsh-host-webserver': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
'@deepseek-ai/dsh-invariants': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)
|
||||
'@deepseek-ai/dsh-settings': 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/schemastery@3.18.1)
|
||||
'@deepseek-ai/schemastery': 3.18.1
|
||||
clsx: 2.1.1
|
||||
react: 18.3.1
|
||||
transitivePeerDependencies:
|
||||
- '@deepseek-ai/dsh-brand'
|
||||
|
||||
'@deepseek-ai/dsh-client-ui-tool@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-api-remotes@0.1.0-rc.6(83d9c5c9b8349236135a0a0b0d2ee7ec))(@deepseek-ai/dsh-client-locale@0.1.0-rc.6(ce9105fc2b8666e57db2f9de071e4b25))(@deepseek-ai/dsh-client-runtime@0.1.0-rc.6(6ef9bea1ffb02b7be9333a18bfeaf17e))(@deepseek-ai/dsh-client-ui-conversation@0.1.0-rc.6(e8451c10452151e540675b4b130aebe9))(@deepseek-ai/dsh-client-ui-primitives@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-client-ui-slots@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(react@18.3.1)':
|
||||
dependencies:
|
||||
'@deepseek-ai/cordis': 4.0.1(@deepseek-ai/cordis-plugin-include@1.0.6)(@deepseek-ai/cordis-plugin-loader@1.0.2)
|
||||
@@ -5331,7 +5395,7 @@ snapshots:
|
||||
picocolors: 1.1.1
|
||||
pretty-format: 27.5.1
|
||||
|
||||
'@testing-library/react@16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
|
||||
'@testing-library/react@16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
|
||||
dependencies:
|
||||
'@babel/runtime': 7.28.2
|
||||
'@testing-library/dom': 10.4.1
|
||||
@@ -5339,7 +5403,7 @@ snapshots:
|
||||
react-dom: 18.3.1(react@18.3.1)
|
||||
optionalDependencies:
|
||||
'@types/react': 18.3.31
|
||||
'@types/react-dom': 19.2.3(@types/react@18.3.31)
|
||||
'@types/react-dom': 18.3.7(@types/react@18.3.31)
|
||||
|
||||
'@testing-library/react@16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)':
|
||||
dependencies:
|
||||
@@ -5416,10 +5480,9 @@ snapshots:
|
||||
|
||||
'@types/prop-types@15.7.15': {}
|
||||
|
||||
'@types/react-dom@19.2.3(@types/react@18.3.31)':
|
||||
'@types/react-dom@18.3.7(@types/react@18.3.31)':
|
||||
dependencies:
|
||||
'@types/react': 18.3.31
|
||||
optional: true
|
||||
|
||||
'@types/react-dom@19.2.3(@types/react@19.2.14)':
|
||||
dependencies:
|
||||
|
||||
Reference in New Issue
Block a user