mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin-browserskill): host-side ObservationService for the PiP overlay (Phase 1-A)
Per-session live observation for owned sessions only, feeding the upcoming client overlay: - State model: SessionObservation (sessionId/url/action/since/ thumbnailAttachmentId/lastError), one entry per owned session; added on start, removed on stop, cleared on dispose. - Instrumentation: the shared runBsk wrapper begins/ends an action (tool name mapped to a verb) around every model-facing call and tags the child with its session id; action end triggers an immediate thumbnail refresh. - Thumbnail loop: 1.5s cadence while active, 8s idle downclock, x3 failure backoff, silent frame retention on errors; captures run OUTSIDE the instrumentation (no action events, no current-pointer movement) and land in the attachment store by reference. - Interrupt: runner.killFor(tag) kills exactly the tagged in-flight bsk children — chat-Stop-equivalent semantics for the current or a named owned session; foreign ids and the no-in-flight case return false. - Remote seam: dsh 0.1's Typert Remote pipeline and forwarded-event allowlist are closed to out-of-tree packages, so state/events/ interrupt are served over the documented webServer route seam: GET /bsk-observation/state, GET /bsk-observation/events (SSE), POST /bsk-observation/interrupt; routes mount only when a webServer service exists (headless compositions skip them). - Config: observationEnabled (true), thumbnailIntervalMs (1500), idleIntervalMs (8000). Tests (+13, 72 total): state machine + event stream, cadence fast/idle/ backoff, headless no-store silence, observation-traffic isolation, interrupt routing (default/specified/none/foreign), HTTP route handlers, disabled-webServer no-op.
This commit is contained in:
@@ -12,6 +12,8 @@
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import Schema from "@deepseek-ai/schemastery";
|
||||
import { ObservationService } from "./observation";
|
||||
import { registerObservationRoutes } from "./observation-http";
|
||||
import { type BskRunner, createBskRunner } from "./runner";
|
||||
import { SessionRegistry } from "./sessions";
|
||||
import { type PluginConfig, registerTools } from "./tools";
|
||||
@@ -30,6 +32,15 @@ export const Config = Schema.object({
|
||||
maxSessions: Schema.number()
|
||||
.default(5)
|
||||
.description("Maximum number of concurrent browser sessions started through this plugin."),
|
||||
observationEnabled: Schema.boolean()
|
||||
.default(true)
|
||||
.description("Track per-session observation state (action/url/thumbnail) for the PiP overlay."),
|
||||
thumbnailIntervalMs: Schema.number()
|
||||
.default(1500)
|
||||
.description("Thumbnail refresh cadence for active sessions (milliseconds)."),
|
||||
idleIntervalMs: Schema.number()
|
||||
.default(8000)
|
||||
.description("Thumbnail refresh cadence for idle sessions; also the recent-activity window."),
|
||||
});
|
||||
|
||||
export type Config = PluginConfig;
|
||||
@@ -48,11 +59,25 @@ export function apply(
|
||||
bskPath: config.bskPath ?? "bsk",
|
||||
defaultTimeoutMs: config.defaultTimeoutMs ?? 120_000,
|
||||
maxSessions: config.maxSessions ?? 5,
|
||||
observationEnabled: config.observationEnabled ?? true,
|
||||
thumbnailIntervalMs: config.thumbnailIntervalMs ?? 1500,
|
||||
idleIntervalMs: config.idleIntervalMs ?? 8000,
|
||||
};
|
||||
const runner = options.runnerFactory?.(resolved.bskPath) ?? createBskRunner(resolved.bskPath);
|
||||
const registry = new SessionRegistry(resolved.maxSessions);
|
||||
const observation = new ObservationService({
|
||||
ctx,
|
||||
runner,
|
||||
registry,
|
||||
options: {
|
||||
enabled: resolved.observationEnabled,
|
||||
thumbnailIntervalMs: resolved.thumbnailIntervalMs,
|
||||
idleIntervalMs: resolved.idleIntervalMs,
|
||||
},
|
||||
});
|
||||
|
||||
registerTools({ ctx, runner, registry, config: resolved });
|
||||
registerTools({ ctx, runner, registry, config: resolved, observation });
|
||||
const removeRoutes = registerObservationRoutes(ctx, observation);
|
||||
|
||||
// Non-blocking install probe: warn early when bsk is missing instead of
|
||||
// failing the first tool call with a bare spawn error.
|
||||
@@ -73,6 +98,8 @@ export function apply(
|
||||
// are swallowed so one stale handle cannot abort the rest.
|
||||
ctx.effect(() => {
|
||||
return () => {
|
||||
removeRoutes();
|
||||
observation.dispose();
|
||||
runner.killAll();
|
||||
const stops = registry
|
||||
.ownedIds()
|
||||
@@ -84,6 +111,9 @@ export function apply(
|
||||
});
|
||||
}
|
||||
|
||||
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
|
||||
export { ObservationService } from "./observation";
|
||||
export { registerObservationRoutes } from "./observation-http";
|
||||
export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
|
||||
export { BskError, createBskRunner } from "./runner";
|
||||
export { SessionRegistry } from "./sessions";
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* HTTP/SSE exposure of the ObservationService for the client half. The Typert
|
||||
* Remote pipeline and the forwarded-event allowlist are both closed to
|
||||
* out-of-tree packages in dsh 0.1 (generation runs over dsh's own ts.Program;
|
||||
* the event allowlist lives in dsh-api-remotes), so the plugin serves its
|
||||
* observation channel through the documented `webServer` route seam instead:
|
||||
*
|
||||
* GET /bsk-observation/state → { sessions: SessionObservation[] }
|
||||
* GET /bsk-observation/events → SSE stream of ObservationEvent
|
||||
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
|
||||
*
|
||||
* Routes exist only when a webServer service is mounted (web composition);
|
||||
* headless deployments skip registration entirely.
|
||||
*/
|
||||
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ObservationService } from "./observation";
|
||||
|
||||
/** Structural view of the dsh-host-webserver route seam. */
|
||||
interface WebServerLike {
|
||||
register(route: {
|
||||
kind: "exact" | "prefix";
|
||||
path: string;
|
||||
handler: (req: IncomingMessage, res: ServerResponse) => void | Promise<void>;
|
||||
}): () => void;
|
||||
}
|
||||
|
||||
const ROUTE_BASE = "/bsk-observation";
|
||||
const SSE_HEARTBEAT_MS = 15_000;
|
||||
|
||||
function sendJson(res: ServerResponse, status: number, body: unknown): void {
|
||||
res.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the observation routes. No-op (with a console note) when the
|
||||
* composition has no web server.
|
||||
* @returns disposer removing the routes.
|
||||
*/
|
||||
export function registerObservationRoutes(
|
||||
ctx: Context,
|
||||
observation: ObservationService,
|
||||
): () => void {
|
||||
const webServer = ctx.get("webServer") as WebServerLike | undefined;
|
||||
if (webServer === undefined) {
|
||||
return () => {};
|
||||
}
|
||||
const disposers = [
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/state`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "GET") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, { sessions: observation.getState() });
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/events`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "GET") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, {
|
||||
"content-type": "text/event-stream",
|
||||
"cache-control": "no-cache",
|
||||
connection: "keep-alive",
|
||||
});
|
||||
const unsubscribe = observation.subscribe((event) => {
|
||||
res.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
});
|
||||
const heartbeat = setInterval(() => res.write(": heartbeat\n\n"), SSE_HEARTBEAT_MS);
|
||||
req.on("close", () => {
|
||||
clearInterval(heartbeat);
|
||||
unsubscribe();
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/interrupt`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "POST") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
let body = "";
|
||||
req.on("data", (chunk: Buffer | string) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
let sessionId: string | undefined;
|
||||
try {
|
||||
const parsed = JSON.parse(body || "{}") as { sessionId?: unknown };
|
||||
if (typeof parsed.sessionId === "string" && parsed.sessionId !== "") {
|
||||
sessionId = parsed.sessionId;
|
||||
}
|
||||
} catch {
|
||||
sendJson(res, 400, { error: "invalid JSON body" });
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, { interrupted: observation.interrupt(sessionId) });
|
||||
});
|
||||
},
|
||||
}),
|
||||
];
|
||||
return () => {
|
||||
for (const dispose of disposers) dispose();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
/**
|
||||
* ObservationService: per-owned-session live observation state for the PiP
|
||||
* overlay — current action, page url, and a breathing thumbnail carried by
|
||||
* attachment reference. Strict ownership boundary applies throughout: only
|
||||
* sessions in the plugin's SessionRegistry (owned by construction) ever get
|
||||
* an observation entry, and interrupt/kill paths can only reach children this
|
||||
* plugin spawned.
|
||||
*
|
||||
* Observation traffic isolation: thumbnail captures run through the runner
|
||||
* directly (never through the tool-level instrumentation), emit no action
|
||||
* events, and never move the registry's current pointer.
|
||||
*/
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { BskRunner } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
/** One owned session's live observation record (wire-stable shape). */
|
||||
export interface SessionObservation {
|
||||
sessionId: string;
|
||||
/** Last settled page URL, when any navigation completed. */
|
||||
url?: string;
|
||||
/** Current action ('idle' when nothing is in flight). */
|
||||
action: string;
|
||||
/** Epoch ms when the current action started (elapsed time is client-side). */
|
||||
since: number;
|
||||
/** Latest thumbnail, by attachment-store reference (never bytes on the wire). */
|
||||
thumbnailAttachmentId?: string;
|
||||
/** Summary of the most recent failed action (drives the red status dot). */
|
||||
lastError?: string;
|
||||
}
|
||||
|
||||
/** Incremental event carried to subscribers (SSE on the wire). */
|
||||
export interface ObservationEvent {
|
||||
type: "upsert" | "remove" | "reset";
|
||||
session?: SessionObservation;
|
||||
}
|
||||
|
||||
export interface ObservationOptions {
|
||||
enabled: boolean;
|
||||
/** Fast cadence while a session is active or recently was. */
|
||||
thumbnailIntervalMs: number;
|
||||
/** Slow cadence for idle sessions; also the "recently active" window. */
|
||||
idleIntervalMs: number;
|
||||
}
|
||||
|
||||
/** Structural view of the optional attachment seam (absent in headless compositions). */
|
||||
interface AttachmentLike {
|
||||
saveImage(input: {
|
||||
data: Uint8Array;
|
||||
mediaType: string;
|
||||
name?: string;
|
||||
}): Promise<{ attachmentId: unknown }>;
|
||||
}
|
||||
|
||||
/** Injectable clock/scheduler bits for tests. */
|
||||
export interface ObservationScheduler {
|
||||
setTimeout: (fn: () => void, ms: number) => unknown;
|
||||
clearTimeout: (handle: unknown) => void;
|
||||
now: () => number;
|
||||
}
|
||||
|
||||
const DEFAULT_SCHEDULER: ObservationScheduler = {
|
||||
setTimeout: (fn, ms) => {
|
||||
const timer = setTimeout(fn, ms);
|
||||
// Never hold the event loop open for a frame refresh.
|
||||
if (typeof timer === "object" && typeof timer.unref === "function") timer.unref();
|
||||
return timer;
|
||||
},
|
||||
clearTimeout: (h) => clearTimeout(h as Parameters<typeof clearTimeout>[0]),
|
||||
now: () => Date.now(),
|
||||
};
|
||||
|
||||
/** Max consecutive capture failures before a session drops to the idle cadence. */
|
||||
const FAILURE_BACKOFF_THRESHOLD = 3;
|
||||
|
||||
export class ObservationService {
|
||||
private readonly observations = new Map<string, SessionObservation>();
|
||||
private readonly listeners = new Set<(event: ObservationEvent) => void>();
|
||||
private readonly captureTimers = new Map<string, unknown>();
|
||||
private readonly captureInFlight = new Set<string>();
|
||||
private readonly captureFailures = new Map<string, number>();
|
||||
private readonly lastActivity = new Map<string, number>();
|
||||
private disposed = false;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
ctx: Context;
|
||||
runner: BskRunner;
|
||||
registry: SessionRegistry;
|
||||
options: ObservationOptions;
|
||||
scheduler?: ObservationScheduler;
|
||||
},
|
||||
) {}
|
||||
|
||||
private get scheduler(): ObservationScheduler {
|
||||
return this.deps.scheduler ?? DEFAULT_SCHEDULER;
|
||||
}
|
||||
|
||||
/** All current entries (client initial/resync snapshot). */
|
||||
getState(): SessionObservation[] {
|
||||
return [...this.observations.values()].map((entry) => ({ ...entry }));
|
||||
}
|
||||
|
||||
/** Subscribe to incremental changes; returns an unsubscribe function. */
|
||||
subscribe(listener: (event: ObservationEvent) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
private emit(event: ObservationEvent): void {
|
||||
for (const listener of [...this.listeners]) {
|
||||
try {
|
||||
listener(event);
|
||||
} catch {
|
||||
// A faulty subscriber must not starve the others.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private put(entry: SessionObservation): void {
|
||||
this.observations.set(entry.sessionId, entry);
|
||||
this.emit({ type: "upsert", session: { ...entry } });
|
||||
}
|
||||
|
||||
/** Register a fresh owned session (called from browser_session_start). */
|
||||
addSession(sessionId: string, url?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
this.put({
|
||||
sessionId,
|
||||
...(url !== undefined ? { url } : {}),
|
||||
action: "idle",
|
||||
since: this.scheduler.now(),
|
||||
});
|
||||
this.lastActivity.set(sessionId, this.scheduler.now());
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Drop a session (called from browser_session_stop). */
|
||||
removeSession(sessionId: string): void {
|
||||
this.cancelCapture(sessionId);
|
||||
this.captureFailures.delete(sessionId);
|
||||
this.lastActivity.delete(sessionId);
|
||||
if (this.observations.delete(sessionId)) {
|
||||
this.emit({
|
||||
type: "remove",
|
||||
session: { sessionId, action: "idle", since: this.scheduler.now() },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark an action starting on a session (tool entry instrumentation). */
|
||||
beginAction(sessionId: string, action: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined) return;
|
||||
const now = this.scheduler.now();
|
||||
this.lastActivity.set(sessionId, now);
|
||||
this.put({ ...entry, action, since: now });
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark the current action settling (tool exit instrumentation). Triggers an
|
||||
* immediate thumbnail refresh — action-driven first, timer as fallback.
|
||||
*/
|
||||
endAction(sessionId: string, error?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined) return;
|
||||
const now = this.scheduler.now();
|
||||
this.lastActivity.set(sessionId, now);
|
||||
this.put({
|
||||
...entry,
|
||||
action: "idle",
|
||||
since: now,
|
||||
...(error !== undefined ? { lastError: error } : {}),
|
||||
});
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Record the settled page URL (navigate success / start with url). */
|
||||
setUrl(sessionId: string, url: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined) return;
|
||||
this.put({ ...entry, url });
|
||||
}
|
||||
|
||||
/**
|
||||
* Interrupt the in-flight call of one session (default: the registry's
|
||||
* current). Kills exactly the bsk children this plugin spawned for that
|
||||
* session — same user-visible semantics as the chat Stop button (the in-flight
|
||||
* tool call fails; the agent flow may continue).
|
||||
* @returns whether an in-flight call was actually interrupted.
|
||||
*/
|
||||
interrupt(sessionId?: string): boolean {
|
||||
const target = sessionId ?? this.deps.registry.current();
|
||||
if (target === undefined || !this.deps.registry.isOwned(target)) return false;
|
||||
return this.deps.runner.killFor(target) > 0;
|
||||
}
|
||||
|
||||
/** Tear down all state and timers (plugin dispose). */
|
||||
dispose(): void {
|
||||
this.disposed = true;
|
||||
for (const sessionId of [...this.captureTimers.keys()]) this.cancelCapture(sessionId);
|
||||
this.captureTimers.clear();
|
||||
this.captureInFlight.clear();
|
||||
this.captureFailures.clear();
|
||||
this.lastActivity.clear();
|
||||
this.observations.clear();
|
||||
this.emit({ type: "reset" });
|
||||
this.listeners.clear();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Thumbnail loop
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private cancelCapture(sessionId: string): void {
|
||||
const timer = this.captureTimers.get(sessionId);
|
||||
if (timer !== undefined) {
|
||||
this.scheduler.clearTimeout(timer);
|
||||
this.captureTimers.delete(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Schedule the next capture for a session; `delayMs` 0 means "as soon as the event loop allows". */
|
||||
private scheduleCapture(sessionId: string, delayMs?: number): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
if (!this.observations.has(sessionId)) return;
|
||||
this.cancelCapture(sessionId);
|
||||
const now = this.scheduler.now();
|
||||
const lastSeen = this.lastActivity.get(sessionId) ?? 0;
|
||||
const failures = this.captureFailures.get(sessionId) ?? 0;
|
||||
const { thumbnailIntervalMs, idleIntervalMs } = this.deps.options;
|
||||
const cadence =
|
||||
now - lastSeen < idleIntervalMs && failures < FAILURE_BACKOFF_THRESHOLD
|
||||
? thumbnailIntervalMs
|
||||
: idleIntervalMs;
|
||||
const delay = delayMs ?? cadence;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
this.captureTimers.delete(sessionId);
|
||||
void this.capture(sessionId);
|
||||
}, delay);
|
||||
this.captureTimers.set(sessionId, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture one frame: `bsk screenshot --json` through the runner, bytes into
|
||||
* the attachment store, reference onto the observation. Runs OUTSIDE the
|
||||
* tool instrumentation on purpose — no action events, no registry writes.
|
||||
* Failures keep the previous frame and back off silently.
|
||||
*/
|
||||
private async capture(sessionId: string): Promise<void> {
|
||||
if (this.disposed || !this.observations.has(sessionId)) return;
|
||||
if (this.captureInFlight.has(sessionId)) return;
|
||||
const attachments = this.deps.ctx.get("attachments") as AttachmentLike | undefined;
|
||||
if (attachments === undefined) {
|
||||
// No attachment store (headless composition): thumbnails stay absent.
|
||||
return;
|
||||
}
|
||||
this.captureInFlight.add(sessionId);
|
||||
try {
|
||||
const outPath = join(tmpdir(), `bsk-obs-${sessionId}-${this.scheduler.now()}.png`);
|
||||
const result = await this.deps.runner.run(
|
||||
["screenshot", "--session", sessionId, "--out", outPath],
|
||||
{ timeoutMs: 15_000, tag: `observation:${sessionId}` },
|
||||
);
|
||||
if (result.code !== 0) throw new Error(`screenshot exited ${result.code}`);
|
||||
const reply = JSON.parse(result.stdout) as { path?: string };
|
||||
const data = await readFile(reply.path ?? outPath);
|
||||
const ref = await attachments.saveImage({
|
||||
data,
|
||||
mediaType: "image/png",
|
||||
name: `observation-${sessionId}.png`,
|
||||
});
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined) return;
|
||||
this.captureFailures.delete(sessionId);
|
||||
this.put({ ...entry, thumbnailAttachmentId: String(ref.attachmentId) });
|
||||
} catch {
|
||||
// Silent by design: keep the previous frame, count toward backoff.
|
||||
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
|
||||
} finally {
|
||||
this.captureInFlight.delete(sessionId);
|
||||
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Map a bsk command label onto its observation action verb. */
|
||||
export function actionForLabel(label: string): string {
|
||||
switch (label) {
|
||||
case "session start":
|
||||
return "starting";
|
||||
case "session stop":
|
||||
return "stopping";
|
||||
case "navigate":
|
||||
return "navigating";
|
||||
case "snapshot":
|
||||
return "snapshotting";
|
||||
case "observe":
|
||||
return "observing";
|
||||
case "click":
|
||||
return "clicking";
|
||||
case "fill":
|
||||
return "filling";
|
||||
case "press":
|
||||
return "pressing";
|
||||
case "screenshot":
|
||||
return "capturing";
|
||||
case "emulate":
|
||||
return "emulating";
|
||||
default:
|
||||
return label;
|
||||
}
|
||||
}
|
||||
@@ -53,17 +53,26 @@ export interface BskRunOptions {
|
||||
/** Minimal spawn signature so tests can substitute a fake child process. */
|
||||
export type SpawnImpl = (command: string, args: string[]) => ChildProcess;
|
||||
|
||||
export interface BskRunOptions {
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
/** Opaque routing tag (e.g. a session id) enabling per-tag kills. */
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
export interface BskRunner {
|
||||
/** Run `bsk <args...> --json` and collect its output. */
|
||||
run(args: string[], options?: BskRunOptions): Promise<BskRunResult>;
|
||||
/** Kill every in-flight child (used when the plugin unloads). */
|
||||
killAll(): void;
|
||||
/** Kill only the in-flight children carrying this tag; returns how many were killed. */
|
||||
killFor(tag: string): number;
|
||||
}
|
||||
|
||||
const KILL_GRACE_MS = 2000;
|
||||
|
||||
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
|
||||
const live = new Set<ChildProcess>();
|
||||
const live = new Map<ChildProcess, string | undefined>();
|
||||
|
||||
function killChild(child: ChildProcess): void {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
@@ -84,7 +93,7 @@ export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn):
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
live.add(child);
|
||||
live.set(child, options.tag);
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
@@ -134,7 +143,17 @@ export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn):
|
||||
});
|
||||
},
|
||||
killAll() {
|
||||
for (const child of live) killChild(child);
|
||||
for (const child of live.keys()) killChild(child);
|
||||
},
|
||||
killFor(tag: string) {
|
||||
let killed = 0;
|
||||
for (const [child, childTag] of live) {
|
||||
if (childTag === tag) {
|
||||
killChild(child);
|
||||
killed += 1;
|
||||
}
|
||||
}
|
||||
return killed;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ContentBlock } from "@deepseek-ai/dsh-llm";
|
||||
import { defineTool, type ToolResult, type ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { trySaveScreenshot } from "./image";
|
||||
import { actionForLabel, type ObservationService } from "./observation";
|
||||
import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
@@ -21,6 +22,9 @@ export interface PluginConfig {
|
||||
bskPath: string;
|
||||
defaultTimeoutMs: number;
|
||||
maxSessions: number;
|
||||
observationEnabled: boolean;
|
||||
thumbnailIntervalMs: number;
|
||||
idleIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface ToolDeps {
|
||||
@@ -28,6 +32,7 @@ export interface ToolDeps {
|
||||
runner: BskRunner;
|
||||
registry: SessionRegistry;
|
||||
config: PluginConfig;
|
||||
observation: ObservationService;
|
||||
}
|
||||
|
||||
/** Device presets supported by `bsk emulate --device`. */
|
||||
@@ -48,31 +53,52 @@ const SESSION_PARAM = {
|
||||
"Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
/** Run one bsk command and return its parsed JSON payload, or throw. */
|
||||
/**
|
||||
* Run one bsk command and return its parsed JSON payload, or throw.
|
||||
* `observeSession` marks the run as model-facing work on that session: it is
|
||||
* instrumented into the observation service (action begin/end) and tagged so
|
||||
* interrupt() can kill exactly this child. Observation traffic itself never
|
||||
* passes an observeSession.
|
||||
*/
|
||||
async function runBsk(
|
||||
deps: ToolDeps,
|
||||
exec: ToolRunContext,
|
||||
args: string[],
|
||||
label: string,
|
||||
observeSession?: string,
|
||||
): Promise<unknown> {
|
||||
let result;
|
||||
if (observeSession !== undefined) {
|
||||
deps.observation.beginAction(observeSession, actionForLabel(label));
|
||||
}
|
||||
let actionError: string | undefined;
|
||||
try {
|
||||
result = await deps.runner.run(args, {
|
||||
signal: exec.signal,
|
||||
timeoutMs: deps.config.defaultTimeoutMs,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isCommandNotFound(error)) {
|
||||
throw new Error(bskInstallMessage(deps.config.bskPath));
|
||||
let result;
|
||||
try {
|
||||
result = await deps.runner.run(args, {
|
||||
signal: exec.signal,
|
||||
timeoutMs: deps.config.defaultTimeoutMs,
|
||||
...(observeSession !== undefined ? { tag: observeSession } : {}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isCommandNotFound(error)) {
|
||||
throw new Error(bskInstallMessage(deps.config.bskPath));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (result.aborted) {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
throw error;
|
||||
}
|
||||
return parseBskJson(result, label);
|
||||
} catch (error) {
|
||||
actionError = error instanceof Error ? error.message.split("\n")[0] : String(error);
|
||||
throw error;
|
||||
} finally {
|
||||
if (observeSession !== undefined) {
|
||||
deps.observation.endAction(observeSession, actionError);
|
||||
}
|
||||
}
|
||||
if (result.aborted) {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
throw error;
|
||||
}
|
||||
return parseBskJson(result, label);
|
||||
}
|
||||
|
||||
/** Build the command line shown on the pending terminal card (pure). */
|
||||
@@ -184,6 +210,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
browserInstanceId: reply.browser_instance_id,
|
||||
startedAtMs: Date.now(),
|
||||
});
|
||||
deps.observation.addSession(reply.session_id, args.url);
|
||||
try {
|
||||
if (args.device !== undefined) {
|
||||
await runBsk(
|
||||
@@ -191,6 +218,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
exec,
|
||||
["emulate", "--session", reply.session_id, "--device", args.device],
|
||||
"emulate",
|
||||
reply.session_id,
|
||||
);
|
||||
}
|
||||
if (args.url !== undefined) {
|
||||
@@ -199,6 +227,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
exec,
|
||||
["navigate", "--session", reply.session_id, args.url],
|
||||
"navigate",
|
||||
reply.session_id,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -207,6 +236,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
.run(["session", "stop", reply.session_id], { timeoutMs: 30_000 })
|
||||
.catch(() => {});
|
||||
registry.remove(reply.session_id);
|
||||
deps.observation.removeSession(reply.session_id);
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
@@ -250,8 +280,9 @@ export function registerTools(deps: ToolDeps): void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolveForStop(args.session);
|
||||
await runBsk(deps, exec, ["session", "stop", sessionId], "session stop");
|
||||
await runBsk(deps, exec, ["session", "stop", sessionId], "session stop", sessionId);
|
||||
registry.remove(sessionId);
|
||||
deps.observation.removeSession(sessionId);
|
||||
return { stopped: sessionId };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
@@ -377,12 +408,13 @@ export function registerTools(deps: ToolDeps): void {
|
||||
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
cmdArgs.push(args.url);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "navigate")) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "navigate", sessionId)) as {
|
||||
tab_id: number;
|
||||
url: string;
|
||||
final_url?: string;
|
||||
reached: string;
|
||||
};
|
||||
deps.observation.setUrl(sessionId, reply.final_url ?? reply.url);
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
@@ -448,7 +480,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
const cmdArgs = [kind, "--session", sessionId];
|
||||
if (args.maxDepth !== undefined) cmdArgs.push("--max-depth", String(args.maxDepth));
|
||||
if (args.maxTokens !== undefined) cmdArgs.push("--max-tokens", String(args.maxTokens));
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, kind)) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, kind, sessionId)) as {
|
||||
text: string;
|
||||
ref_count: number;
|
||||
tab_id: number;
|
||||
@@ -522,7 +554,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click")) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
||||
tab_id: number;
|
||||
x: number;
|
||||
y: number;
|
||||
@@ -580,7 +612,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
const cmdArgs = ["fill", "--session", sessionId, "--value", args.value];
|
||||
if (args.noClear === true) cmdArgs.push("--no-clear");
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "fill")) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "fill", sessionId)) as {
|
||||
tab_id: number;
|
||||
value_length: number;
|
||||
};
|
||||
@@ -650,7 +682,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
}
|
||||
if (args.holdMs !== undefined) cmdArgs.push("--hold-ms", String(args.holdMs));
|
||||
cmdArgs.push(args.key);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "press")) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "press", sessionId)) as {
|
||||
tab_id: number;
|
||||
key: string;
|
||||
};
|
||||
@@ -735,7 +767,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
);
|
||||
const cmdArgs = ["screenshot", "--session", sessionId, "--out", outPath];
|
||||
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "screenshot")) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "screenshot", sessionId)) as {
|
||||
tab_id: number;
|
||||
width: number;
|
||||
height: number;
|
||||
@@ -839,7 +871,7 @@ export function registerTools(deps: ToolDeps): void {
|
||||
}
|
||||
if (args.mobile === true) cmdArgs.push("--mobile");
|
||||
}
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "emulate")) as {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "emulate", sessionId)) as {
|
||||
tab_id: number;
|
||||
cleared: boolean;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user