mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin-browserskill): observation overlay — floating card, interrupt, and PiP (Phase 1-B)
Client half of the PiP observation window, wired to the host ObservationService over the /bsk-observation HTTP+SSE seam: - ObservationClientStore: initial state fetch + SSE increments + on-demand thumbnail blob loading (session-authorized readAttachment, same path as the toolview) + interrupt POST. All I/O injectable. - ObservationOverlay registered into the shell.overlay list seat (the sanctioned frame-wide surface; root is off-limits). Hidden with no owned sessions; appears on first start; vanishes when all stop. - Focus view: status row (green/grey/red dot + session + action + mm:ss ticking), breathing thumbnail (new frames fade in, failures keep the last frame with a warning badge, pre-navigate placeholder), and the action area. - Interrupt: single click, no confirm; greys into 'Interrupting…' and back; disabled when nothing is in flight; one-time tooltip explains the semantics (stops the current action only; the run continues). - Card: drag-move via the header, corner-handle resize (min 240x180, max 80% viewport), both clamped and kept for the page lifetime; collapsible status capsule. - PiP: Pop out (user gesture) opens Document PiP with the card's current size, portals the same content in, and clones the document's style nodes; pagehide falls back to the card with state intact; unsupported browsers hide the button; nothing auto-pops. Tests (+17, 89 total): store (fetch/SSE apply/malformed frames/ thumbnail lifecycle/interrupt wire/stop cleanup) and overlay (hidden-> visible lifecycle, status row, thumbnail via loader, interrupt disabled/active/hint-once, capsule, resize clamps, move clamps, PiP unsupported, PiP pop/fallback). Styling follows web-styling (dsw-alias tokens, CSS Modules, focus-visible, reduced-motion).
This commit is contained in:
BB-fat
@@ -0,0 +1,186 @@
|
||||
// @vitest-environment happy-dom
|
||||
// ObservationOverlay: visibility lifecycle, focus view, interrupt interaction,
|
||||
// one-time hint, drag move/resize with clamps, collapse capsule, and PiP
|
||||
// upgrade/fallback (mocked documentPictureInPicture). NOTE: use RTL's waitFor
|
||||
// (act-flushing), not vi.waitFor, when asserting store-driven UI updates.
|
||||
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { ObservationOverlay } from "../../src/client/ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import type { SessionObservation } from "../../src/observation";
|
||||
|
||||
const BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: Date.now() - 7000 };
|
||||
|
||||
interface Harness {
|
||||
store: ObservationClientStore;
|
||||
es: () => EventSourceLike;
|
||||
push: (sessions: SessionObservation[]) => void;
|
||||
fetches: { url: string; init?: { body?: string } }[];
|
||||
loadImage: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
function makeHarness(initial: SessionObservation[]): Harness {
|
||||
const fetches: Harness["fetches"] = [];
|
||||
let es: EventSourceLike | undefined;
|
||||
let current = initial;
|
||||
const loadImage = vi.fn(async (id: string) => `blob:${id}`);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: async (url: string, init?: { body?: string }) => {
|
||||
fetches.push({ url, init });
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: current }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
},
|
||||
eventSourceFactory: () => {
|
||||
es = { onmessage: null, close: vi.fn() };
|
||||
return es;
|
||||
},
|
||||
loadImage,
|
||||
});
|
||||
return {
|
||||
store,
|
||||
es: () => {
|
||||
if (es === undefined) throw new Error("no EventSource yet");
|
||||
return es;
|
||||
},
|
||||
push: (sessions) => {
|
||||
current = sessions;
|
||||
for (const s of sessions) {
|
||||
es?.onmessage?.({ data: JSON.stringify({ type: "upsert", session: s }) });
|
||||
}
|
||||
if (sessions.length === 0) es?.onmessage?.({ data: JSON.stringify({ type: "reset" }) });
|
||||
},
|
||||
fetches,
|
||||
loadImage,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe("ObservationOverlay", () => {
|
||||
beforeEach(() => {
|
||||
delete (window as unknown as Record<string, unknown>).documentPictureInPicture;
|
||||
});
|
||||
|
||||
it("renders nothing with no sessions, appears when one starts", async () => {
|
||||
const h = makeHarness([]);
|
||||
const { container } = render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.es).not.toThrow());
|
||||
expect(container.firstChild).toBeNull();
|
||||
h.push([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
await screen.findByText(/s1 · idle/);
|
||||
expect(screen.getByTestId("obs-card")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows the status row and a placeholder without a thumbnail", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking · 00:0/);
|
||||
expect(screen.getByText("waiting for page")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("loads and renders the thumbnail through the store", async () => {
|
||||
const h = makeHarness([{ ...BUSY, thumbnailAttachmentId: "att-9" }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.loadImage).toHaveBeenCalledWith("att-9"));
|
||||
await screen.findByRole("img", { name: "session s1 view" });
|
||||
});
|
||||
|
||||
it("interrupt: disabled while idle, active call posts and shows progress", async () => {
|
||||
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const button = await screen.findByRole("button", { name: /Interrupt the current/ });
|
||||
expect(button).toHaveProperty("disabled", true);
|
||||
h.push([BUSY]);
|
||||
await waitFor(() => expect(button).toHaveProperty("disabled", false));
|
||||
fireEvent.click(button);
|
||||
await waitFor(() =>
|
||||
expect(h.fetches.some((f) => f.url === "/bsk-observation/interrupt")).toBe(true),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the semantics hint once", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const button = await screen.findByRole("button", { name: /Interrupt the current/ });
|
||||
fireEvent.pointerEnter(button);
|
||||
await screen.findByRole("tooltip");
|
||||
fireEvent.pointerLeave(button);
|
||||
fireEvent.click(button);
|
||||
fireEvent.pointerEnter(button);
|
||||
expect(screen.queryByRole("tooltip")).toBeNull();
|
||||
});
|
||||
|
||||
it("collapses to a capsule and expands back", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Collapse" }));
|
||||
const capsule = await screen.findByRole("button", {
|
||||
name: "Expand browser observation overlay",
|
||||
});
|
||||
expect(capsule.textContent).toContain("clicking");
|
||||
fireEvent.click(capsule);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
});
|
||||
|
||||
it("resizes within min/max clamps", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const card = await screen.findByTestId("obs-card");
|
||||
expect(Number.parseFloat(card.style.width)).toBe(320);
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-resize"), { clientX: 100, clientY: 100 });
|
||||
fireEvent.pointerMove(document, { clientX: 40, clientY: 40 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(Number.parseFloat(card.style.width)).toBe(240); // min width clamp
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-resize"), { clientX: 100, clientY: 100 });
|
||||
fireEvent.pointerMove(document, { clientX: 5000, clientY: 5000 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(Number.parseFloat(card.style.width)).toBe(window.innerWidth * 0.8);
|
||||
});
|
||||
|
||||
it("moves the card with header drags, clamped to the viewport", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const card = await screen.findByTestId("obs-card");
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-header"), { clientX: 50, clientY: 50 });
|
||||
fireEvent.pointerMove(document, { clientX: -500, clientY: -500 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(card.style.left).toBe("0px");
|
||||
expect(card.style.top).toBe("0px");
|
||||
});
|
||||
|
||||
it("hides Pop out when Document PiP is unsupported", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
expect(screen.queryByRole("button", { name: /Pop out/ })).toBeNull();
|
||||
});
|
||||
|
||||
it("pops out with the card's current size and falls back on pagehide", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const pipDoc = document.implementation.createHTMLDocument("pip");
|
||||
const listeners = new Map<string, (() => void)[]>();
|
||||
const requestWindow = vi.fn(async (options?: { width?: number; height?: number }) => {
|
||||
void options;
|
||||
return {
|
||||
document: pipDoc,
|
||||
addEventListener: (name: string, fn: () => void) => {
|
||||
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
|
||||
},
|
||||
} as unknown as Window;
|
||||
});
|
||||
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const popout = await screen.findByRole("button", { name: /Pop out/ });
|
||||
fireEvent.click(popout);
|
||||
await waitFor(() => expect(requestWindow).toHaveBeenCalledWith({ width: 320, height: 240 }));
|
||||
// Content now renders inside the PiP document.
|
||||
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
|
||||
// Closing the PiP returns to the in-page card with state preserved.
|
||||
for (const fn of listeners.get("pagehide") ?? []) fn();
|
||||
await screen.findByTestId("obs-card");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
// ObservationClientStore: initial fetch + SSE application, thumbnail loading
|
||||
// lifecycle, interrupt wire shape. All I/O faked.
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import type { ObservationEvent, SessionObservation } from "../../src/observation";
|
||||
|
||||
const OBS_IDLE: SessionObservation = { sessionId: "s1", action: "idle", since: 1000 };
|
||||
const OBS_BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: 2000 };
|
||||
|
||||
function harness(state: SessionObservation[] = []) {
|
||||
const fetches: { url: string; init?: { method?: string; body?: string } }[] = [];
|
||||
const esInstances: EventSourceLike[] = [];
|
||||
const fetchFn = vi.fn(async (url: string, init?: { method?: string; body?: string }) => {
|
||||
fetches.push({ url, init });
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: state }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
});
|
||||
const eventSourceFactory = (url: string) => {
|
||||
const es: EventSourceLike = { onmessage: null, close: vi.fn() };
|
||||
esInstances.push(es);
|
||||
return es;
|
||||
};
|
||||
const loadImage = vi.fn(async (id: string) => `blob:url-${id}`);
|
||||
const store = new ObservationClientStore({ fetchFn, eventSourceFactory, loadImage });
|
||||
return { store, fetches, esInstances, loadImage };
|
||||
}
|
||||
|
||||
function emit(es: EventSourceLike, event: ObservationEvent): void {
|
||||
es.onmessage?.({ data: JSON.stringify(event) });
|
||||
}
|
||||
|
||||
describe("ObservationClientStore", () => {
|
||||
it("loads the initial state and applies SSE increments", async () => {
|
||||
const { store, esInstances } = harness([OBS_IDLE]);
|
||||
const seen: number[] = [];
|
||||
store.subscribe(() => seen.push(store.getSnapshot().sessions.length));
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
expect(esInstances).toHaveLength(1);
|
||||
emit(esInstances[0], { type: "upsert", session: OBS_BUSY });
|
||||
expect(store.getSnapshot().sessions[0].action).toBe("clicking");
|
||||
emit(esInstances[0], {
|
||||
type: "upsert",
|
||||
session: { sessionId: "s2", action: "idle", since: 3 },
|
||||
});
|
||||
expect(store.getSnapshot().sessions).toHaveLength(2);
|
||||
emit(esInstances[0], {
|
||||
type: "remove",
|
||||
session: { sessionId: "s2", action: "idle", since: 0 },
|
||||
});
|
||||
expect(store.getSnapshot().sessions).toHaveLength(1);
|
||||
emit(esInstances[0], { type: "reset" });
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
expect(seen.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("ignores malformed SSE frames", () => {
|
||||
const { store, esInstances } = harness([]);
|
||||
store.start();
|
||||
esInstances[0].onmessage?.({ data: "not json" });
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("loads each thumbnail once and reports ready", async () => {
|
||||
const { store, loadImage } = harness();
|
||||
store.ensureThumbnail("att-1");
|
||||
store.ensureThumbnail("att-1");
|
||||
expect(loadImage).toHaveBeenCalledTimes(1);
|
||||
await vi.waitFor(() =>
|
||||
expect(store.getSnapshot().thumbnails["att-1"]).toEqual({
|
||||
status: "ready",
|
||||
url: "blob:url-att-1",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("marks failed thumbnail loads as error", async () => {
|
||||
const { store, loadImage } = harness();
|
||||
loadImage.mockRejectedValueOnce(new Error("nope"));
|
||||
store.ensureThumbnail("att-bad");
|
||||
await vi.waitFor(() => expect(store.getSnapshot().thumbnails["att-bad"].status).toBe("error"));
|
||||
});
|
||||
|
||||
it("posts interrupt with and without a session id", async () => {
|
||||
const { store, fetches } = harness();
|
||||
expect(await store.interrupt()).toBe(true);
|
||||
expect(await store.interrupt("s1")).toBe(true);
|
||||
const calls = fetches.filter((f) => f.url === "/bsk-observation/interrupt");
|
||||
expect(JSON.parse(calls[0].init?.body ?? "")).toEqual({});
|
||||
expect(JSON.parse(calls[1].init?.body ?? "")).toEqual({ sessionId: "s1" });
|
||||
});
|
||||
|
||||
it("returns false when interrupt fails", async () => {
|
||||
const failing = new ObservationClientStore({
|
||||
fetchFn: async () => ({ ok: false, json: async () => ({}) }),
|
||||
eventSourceFactory: () => ({ onmessage: null, close: () => {} }),
|
||||
loadImage: async () => "blob:x",
|
||||
});
|
||||
expect(await failing.interrupt("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("stop closes the stream and clears state", async () => {
|
||||
const { store, esInstances } = harness([OBS_IDLE]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.stop();
|
||||
expect(esInstances[0].close).toHaveBeenCalled();
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -132,20 +132,23 @@ describe("ScreenshotToolView", () => {
|
||||
});
|
||||
|
||||
describe("client plugin registration", () => {
|
||||
it("registers only the browser_screenshot keyed toolview", () => {
|
||||
const registrations: { name: string; key?: string }[] = [];
|
||||
it("registers the browser_screenshot toolview and the shell observation overlay", () => {
|
||||
const registrations: { name: string; key?: string; id?: string }[] = [];
|
||||
const sessions = { binding: () => undefined };
|
||||
const ctx = {
|
||||
get: (key: string) => (key === "sessions" ? sessions : undefined),
|
||||
slots: {
|
||||
inject: (_name: string, fn: () => unknown) => fn(),
|
||||
register: (slot: { name: string; key?: string }, view: unknown) => {
|
||||
register: (slot: { name: string; key?: string; id?: string }, view: unknown) => {
|
||||
registrations.push(slot);
|
||||
expect(typeof view).toBe("function");
|
||||
},
|
||||
},
|
||||
};
|
||||
apply(ctx as never);
|
||||
expect(registrations).toEqual([{ name: "tool.call.toolview", key: "browser_screenshot" }]);
|
||||
expect(registrations).toEqual([
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "shell.overlay", id: "bsk-observation" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user