Compare commits

..
Author SHA1 Message Date
Ljy-0827 eeba301080 feat(vom): fixture test 2026-08-26 12:11:19 +08:00
Ljy-0827 5ca158a729 fix(vom): canvas recognition 2026-08-26 11:51:18 +08:00
Ljy-0827 ed91f41291 feat(vom): vom can distinguish important canvas area 2026-08-25 16:12:19 +08:00
Ljy-0827 b9fd6e46af fix(vom): fix repeat names 2026-08-24 14:52:16 +08:00
Ljy-0827 36135c1a2d fix(vom): bug fix 2026-08-24 14:31:35 +08:00
Ljy-0827 28d8a5f605 fix(vom): safety policy and rebase bug fix 2026-08-24 14:06:42 +08:00
Ljy-0827 54ce2f8dc3 feat(vom): fix name lost 2026-08-24 14:06:42 +08:00
Ljy-0827 b7e62d8cf1 feat(vom): function refactor 2026-08-24 14:06:42 +08:00
53 changed files with 1652 additions and 1599 deletions
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.1.11"
version = "0.1.10"
dependencies = [
"anyhow",
"bsk-protocol",
@@ -168,7 +168,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.1.11"
version = "0.1.10"
dependencies = [
"schemars",
"semver",
+1 -1
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.1.11"
version = "0.1.10"
edition = "2024"
rust-version = "1.85"
license = "MIT"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
"version": "0.1.7",
"version": "0.1.6",
"private": true,
"type": "module",
"scripts": {
@@ -13,6 +13,8 @@ describe("RefStore", () => {
backendNodeId: 42,
tabId: 7,
generation: 0,
kind: "dom",
capabilities: ["interact", "screenshot"],
});
expect(s.size()).toBe(1);
expect(s.isEmpty()).toBe(false);
@@ -48,4 +50,31 @@ describe("RefStore", () => {
cdpSessionId: "child-session",
});
});
it("preserves explicit capabilities for visual surface refs", () => {
const s = new RefStore();
s.set("e1", 42, {
tabId: 7,
kind: "surface",
capabilities: ["screenshot"],
});
expect(s.resolveEntry("e1")).toMatchObject({
kind: "surface",
capabilities: ["screenshot"],
});
});
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
const s = new RefStore();
s.set("e1", 42, { tabId: 7, kind: "surface" });
s.set("e2", 43, {
tabId: 7,
kind: "surface",
capabilities: ["interact", "screenshot"],
});
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
});
});
@@ -1,3 +1,5 @@
import type { Rect } from "@browser-skill/vom";
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
@@ -12,12 +14,17 @@
* inside the `SessionContext`.
*/
export type BackendNodeId = number;
export type RefCapability = "interact" | "screenshot";
export type RefTargetKind = "dom" | "surface";
export interface RefEntry {
backendNodeId: BackendNodeId;
tabId: number | null;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
generation: number;
}
@@ -28,8 +35,19 @@ export type RefInput =
tabId: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
};
function capabilitiesFor(
kind: RefTargetKind,
capabilities: RefCapability[] | undefined,
): RefCapability[] {
if (kind === "surface") return ["screenshot"];
return capabilities ?? ["interact", "screenshot"];
}
export class RefStore {
private readonly map = new Map<string, RefEntry>();
private generation = 0;
@@ -70,13 +88,20 @@ export class RefStore {
tabId?: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
} = {},
): void {
const kind = opts.kind ?? "dom";
this.map.set(normaliseRef(ref), {
backendNodeId: id,
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, opts.capabilities),
generation: this.generation,
});
}
@@ -94,14 +119,20 @@ export class RefStore {
return {
backendNodeId: input,
tabId: null,
kind: "dom",
capabilities: ["interact", "screenshot"],
generation: this.generation,
};
}
const kind = input.kind ?? "dom";
return {
backendNodeId: input.backendNodeId,
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, input.capabilities),
generation: this.generation,
};
}
@@ -123,6 +123,29 @@ describe("handleClick", () => {
expect(fake.cdp.send).not.toHaveBeenCalled();
});
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
const fake = makeFakeCdp({});
const res = await handleClick(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({
code: "permission_denied",
data: { reason: "ref_capability_denied", required_capability: "interact" },
});
expect(fake.cdp.send).not.toHaveBeenCalled();
});
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -300,6 +300,76 @@ describe("handleScreenshot", () => {
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
});
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
capabilities: ["screenshot"],
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { width: number; height: number; scale: number };
};
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
});
it("crops a visual surface screenshot to its observation-time visible region", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { x: number; y: number; width: number; height: number };
};
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
});
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -2693,6 +2763,85 @@ describe("handleSnapshot", () => {
);
});
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Canvas app" },
backendDOMNodeId: 100,
};
const strings = ["body", "canvas", "static", "auto"];
const i = (value: string) => strings.indexOf(value);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("canvas")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 30, 800, 500],
],
paintOrders: [0, 1],
},
},
],
};
}
throw new Error(`unexpected CDP method ${method}`);
});
const deps = {
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
conditionalSurfaceProbe: false,
};
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
backendNodeId: 200,
kind: "surface",
capabilities: ["screenshot"],
});
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
expect(snapshot.text).not.toContain("visual surface");
expect(snapshot.ref_count).toBe(0);
});
it("allows passive snapshots of explicit user-window tabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => {
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
});
@@ -53,6 +57,8 @@ describe("resolveSnapshotRef", () => {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
const expected = {
@@ -60,6 +66,8 @@ describe("resolveSnapshotRef", () => {
refKey: "e3",
frameId: "child-frame",
cdpSessionId: "child-session",
kind: "dom" as const,
capabilities: ["interact", "screenshot"] as const,
};
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => {
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
});
it("rejects an operation outside the ref's declared capabilities", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
code: "permission_denied",
message: "ref @e3 does not support interact",
data: {
reason: "ref_capability_denied",
required_capability: "interact",
kind: "surface",
},
});
});
});
+1 -1
View File
@@ -159,7 +159,7 @@ async function resolveBackendNode(
};
}
if (hasRef) {
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
if (isRpcError(resolved)) return resolved;
return {
backendNodeId: resolved.backendNodeId,
+61 -5
View File
@@ -8,6 +8,7 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type Rect,
renderVom,
type VomNode,
type VomOptions,
@@ -59,6 +60,11 @@ import {
type CaptureVomObservationResult,
projectRecordSafeObservation,
} from "./vom/record-safe-observation";
import {
clusterRenderedSurfaces,
discoverRenderedSurfaces,
projectRenderedSurfaces,
} from "./vom/rendered-surfaces";
import {
buildSemanticGraph,
buildSemanticVomScene,
@@ -90,6 +96,9 @@ export const normaliseRef = sharedNormaliseRef;
// screenshot — `tool.screenshot`
// ---------------------------------------------------------------------------
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
/**
* Strip the `data:image/...;base64,` prefix from a Chrome
* `captureVisibleTab` dataURL and return the raw base64 payload.
@@ -182,6 +191,8 @@ async function captureElementScreenshot(
target: CdpTarget,
backendNodeId: number,
frameId?: string,
visualSurface = false,
observedVisibleRect?: Rect,
signal?: AbortSignal,
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
if (signal?.aborted) return cancelled("screenshot");
@@ -189,11 +200,36 @@ async function captureElementScreenshot(
cdp,
tabId,
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
{ scrollIntoView: true },
{ scrollIntoView: !visualSurface },
);
if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds;
const liveRect = geometry.topBounds;
const rect =
visualSurface && observedVisibleRect
? (() => {
const x = Math.max(liveRect.x, observedVisibleRect.x);
const y = Math.max(liveRect.y, observedVisibleRect.y);
const right = Math.min(
liveRect.x + liveRect.width,
observedVisibleRect.x + observedVisibleRect.w,
);
const bottom = Math.min(
liveRect.y + liveRect.height,
observedVisibleRect.y + observedVisibleRect.h,
);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
})()
: liveRect;
if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
const scale = visualSurface
? Math.min(
1,
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
)
: 1;
try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
@@ -203,7 +239,7 @@ async function captureElementScreenshot(
y: rect.y,
width: rect.width,
height: rect.height,
scale: 1,
scale,
},
});
if (signal?.aborted) return cancelled("screenshot");
@@ -310,7 +346,7 @@ export async function handleScreenshot(
if (!deps.cdp) {
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
}
const node = resolveSnapshotRef(ctx, ref, target.tabId);
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
if (isRpcError(node)) return node;
if (signal?.aborted) return cancelled("screenshot");
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -330,6 +366,8 @@ export async function handleScreenshot(
nodeTarget,
node.backendNodeId,
node.frameId,
node.kind === "surface",
node.visibleRect,
signal,
),
deps.sendToTab,
@@ -1023,6 +1061,7 @@ async function captureForVom(
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
renderedSurfaceDiscovery?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
@@ -1064,7 +1103,20 @@ export async function captureVomObservation(
),
);
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
const rendered = renderVom(decoratedScene, {
const observedScene = options.renderedSurfaceDiscovery
? projectRenderedSurfaces(
decoratedScene,
normalizedDocuments,
clusterRenderedSurfaces(
discoverRenderedSurfaces(
normalizedDocuments,
captured.viewport,
captured.excludedBackendNodeIds,
),
),
)
: decoratedScene;
const rendered = renderVom(observedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
redactValues: options.redactValues,
@@ -1115,6 +1167,7 @@ async function handleVomObservation(
maxTokens: params.max_tokens,
activeRegionPolicy: true,
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
renderedSurfaceDiscovery: toolName === "observe",
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
signal,
});
@@ -1132,6 +1185,9 @@ async function handleVomObservation(
tabId: target.tabId,
...(ref.frameId ? { frameId: ref.frameId } : {}),
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
...(ref.kind ? { kind: ref.kind } : {}),
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
},
] as const;
}),
+17 -1
View File
@@ -2,8 +2,9 @@
// compound CDP node identity via the session RefStore, and build stable
// `ref_not_found` errors for hard-failure tool paths.
import type { Rect } from "@browser-skill/vom";
import type { SessionContext } from "@/session-manager/manager";
import { normaliseRef } from "@/session-manager/ref-store";
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
@@ -12,6 +13,9 @@ export interface SnapshotRefLookup {
refKey: string;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
}
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
@@ -37,6 +41,9 @@ export function lookupSnapshotRef(
refKey,
...(entry.frameId ? { frameId: entry.frameId } : {}),
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
kind: entry.kind,
capabilities: entry.capabilities,
};
}
@@ -49,6 +56,7 @@ export function resolveSnapshotRef(
ctx: SessionContext,
ref: string,
tabId: number,
requiredCapability?: RefCapability,
): SnapshotRefLookup | RpcError {
const looked = lookupSnapshotRef(ctx, ref, tabId);
if (looked === null) {
@@ -58,5 +66,13 @@ export function resolveSnapshotRef(
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
);
}
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
return rpcError(
"permission_denied",
"ref_capability_denied",
`ref ${ref} does not support ${requiredCapability}`,
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
);
}
return looked;
}
@@ -722,6 +722,44 @@ describe("captureViewModel", () => {
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
});
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
const i = (value: string) => S.indexOf(value);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
backendNodeId: [10, 11, 12, 13],
attributes: [[], [], [], []],
},
layout: {
nodeIndex: [1, 2, 3],
styles: [
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
],
bounds: [
[0, 0, 1000, 800],
[10, 10, 200, 100],
[10, 10, 200, 100],
],
paintOrders: [0, 1, 2],
},
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const canvas = nodes.find((node) => node.backendNodeId === 13);
expect(canvas?.rendered).toBe(true);
expect(canvas?.visuallySuppressed).toBe(true);
});
it("subtracts scroll offset to make rects viewport-relative", async () => {
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
@@ -0,0 +1,256 @@
import type { VomScene } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "../capture";
import type { CapturedFrameDocument } from "../frame-capture";
import {
clusterRenderedSurfaces,
discoverRenderedSurfaces,
projectRenderedSurfaces,
} from "../rendered-surfaces";
import type { SemanticAxNode } from "../semantic-graph";
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "canvas",
attrs: {},
rect: { x: 10, y: 20, w: 800, h: 500 },
localRect: { x: 10, y: 20, w: 800, h: 500 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
rendered: true,
...overrides,
};
}
function document(
domNodes: CapturedNode[],
axNodes: SemanticAxNode[] = [],
): CapturedFrameDocument<SemanticAxNode> {
return {
frameId: domNodes[0]?.frameId ?? "main",
contextScopeId: domNodes[0]?.frameId ?? "main",
target: { tabId: 4 },
domNodes,
axNodes,
};
}
describe("rendered surface discovery", () => {
it("finds prominent rendered canvases and excludes hidden descendants", () => {
const visible = domNode(2, { parentBackendNodeId: 1 });
const hidden = domNode(3, { parentBackendNodeId: 4 });
const surfaces = discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "main", rect: null, localRect: null }),
visible,
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
hidden,
]),
],
{ width: 1280, height: 720 },
);
expect(surfaces).toHaveLength(1);
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
});
it("keeps tiny canvases as individually addressable exact discoveries", () => {
const surfaces = discoverRenderedSurfaces(
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
{ width: 1280, height: 720 },
);
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("keeps a surface when the canvas only has partial native accessibility content", () => {
const axNodes = [
{
nodeId: "canvas",
backendDOMNodeId: 2,
role: { type: "role", value: "presentation" },
childIds: ["button"],
frameId: "main",
},
{
nodeId: "button",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
frameId: "main",
},
] as SemanticAxNode[];
expect(
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
const canvas = domNode(2, {
attrs: {
id: "must-not-be-a-label",
class: "must-not-be-a-label",
"data-testid": "must-not-be-a-label",
},
});
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
expect.not.objectContaining({
label: expect.anything(),
}),
]);
});
it("treats blank explicit Canvas names as unnamed", () => {
expect(
discoverRenderedSurfaces(
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
{ width: 1280, height: 720 },
),
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
});
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces(
[document([owner, canvas])],
{ width: 1280, height: 720 },
new Set([1]),
),
).toEqual([]);
});
it("excludes canvases suppressed by a fully transparent ancestor", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 0, y: 0, w: 900, h: 600 },
visuallySuppressed: true,
});
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
).toEqual([]);
});
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 10, y: 20, w: 100, h: 70 },
overflowX: "hidden",
overflowY: "hidden",
});
const partial = domNode(2, {
parentBackendNodeId: 1,
rect: { x: 55, y: 30, w: 140, h: 60 },
});
const fullyClipped = domNode(3, {
parentBackendNodeId: 1,
rect: { x: 140, y: 30, w: 80, h: 50 },
});
expect(
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
width: 1280,
height: 720,
}),
).toEqual([
expect.objectContaining({
backendNodeId: 2,
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
}),
]);
});
});
describe("rendered surface clustering and projection", () => {
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
const body = domNode(1, { tag: "body", rect: null, localRect: null });
const surfaces = discoverRenderedSurfaces(
[
document([
body,
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
]),
],
{ width: 1280, height: 720 },
);
const groups = clusterRenderedSurfaces(surfaces);
const scene: VomScene = {
viewport: { width: 1280, height: 720 },
nodes: [
{
id: 10,
parentId: null,
backendNodeId: 1,
frameId: "main",
tag: "body",
role: "RootWebArea",
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
};
expect(groups).toHaveLength(1);
expect(groups[0]?.members).toHaveLength(2);
expect(groups[0]?.label).toBeUndefined();
expect(
projectRenderedSurfaces(
scene,
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
groups,
),
).toMatchObject({
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
});
});
it("does not merge a contained canvas with a larger independent canvas", () => {
const groups = clusterRenderedSurfaces([
...discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "main", rect: null, localRect: null }),
domNode(2, { parentBackendNodeId: 1 }),
domNode(3, {
parentBackendNodeId: 1,
rect: { x: 30, y: 40, w: 200, h: 100 },
}),
]),
],
{ width: 1280, height: 720 },
),
]);
expect(groups).toHaveLength(2);
});
it("does not merge coincident canvases owned by different containers", () => {
const groups = clusterRenderedSurfaces(
discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "section", rect: null, localRect: null }),
domNode(2, { tag: "section", rect: null, localRect: null }),
domNode(3, { parentBackendNodeId: 1 }),
domNode(4, { parentBackendNodeId: 2 }),
]),
],
{ width: 1280, height: 720 },
),
);
expect(groups).toHaveLength(2);
});
});
+30 -1
View File
@@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"overflow-x",
"overflow-y",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -39,6 +47,10 @@ export interface CapturedNode {
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
overflowX?: string;
overflowY?: string;
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
visuallySuppressed?: boolean;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
@@ -796,6 +808,8 @@ function parseDocumentNodes(
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const overflowXCol = STYLE_COL["overflow-x"];
const overflowYCol = STYLE_COL["overflow-y"];
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -821,6 +835,7 @@ function parseDocumentNodes(
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
@@ -840,7 +855,11 @@ function parseDocumentNodes(
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let overflowX = "visible";
let overflowY = "visible";
let rendered = false;
let visuallySuppressed =
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
@@ -863,12 +882,19 @@ function parseDocumentNodes(
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
visuallySuppressed ||=
visibility === "hidden" ||
visibility === "collapse" ||
(Number.parseFloat(opacity) || 0) <= 0;
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
@@ -903,6 +929,9 @@ function parseDocumentNodes(
position,
pointerEvents,
cursor,
overflowX,
overflowY,
visuallySuppressed,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
@@ -0,0 +1,65 @@
import type { Rect } from "@browser-skill/vom";
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
const STACK_IOU_RATIO = 0.9;
const STACK_SIZE_RATIO = 0.9;
function area(rect: Rect): number {
return Math.max(0, rect.w) * Math.max(0, rect.h);
}
function intersectionArea(a: Rect, b: Rect): number {
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
return width * height;
}
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
if (a.frameId !== b.frameId) return false;
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
const aArea = area(a.visibleRect);
const bArea = area(b.visibleRect);
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
const union = aArea + bArea - intersection;
const widthRatio =
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
const heightRatio =
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
return (
union > 0 &&
intersection / union >= STACK_IOU_RATIO &&
widthRatio >= STACK_SIZE_RATIO &&
heightRatio >= STACK_SIZE_RATIO
);
}
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
const representative = [...members].sort(
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
)[0];
return {
frameId: representative.frameId,
members,
representative,
label: members.map((surface) => surface.label).find((label) => label !== undefined),
};
}
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
const groups: RenderedSurface[][] = [];
for (const surface of surfaces) {
const matching = groups.find((group) =>
group.some((member) => sameVisualStack(member, surface)),
);
if (matching) matching.push(surface);
else groups.push([surface]);
}
return groups
.map(toGroup)
.sort(
(a, b) =>
a.frameId.localeCompare(b.frameId) ||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
a.representative.visibleRect.x - b.representative.visibleRect.x,
);
}
@@ -0,0 +1,113 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CapturedNode } from "../capture";
import type { CapturedFrameDocument } from "../frame-capture";
import type { SemanticAxNode } from "../semantic-graph";
import type { RenderedSurface } from "./types";
function clean(value: string | undefined): string | undefined {
const normalized = value?.replace(/\s+/g, " ").trim();
return normalized || undefined;
}
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
}
function intersectRect(a: Rect, b: Rect): Rect | null {
const x = Math.max(a.x, b.x);
const y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.w, b.x + b.w);
const bottom = Math.min(a.y + a.h, b.y + b.h);
if (right <= x || bottom <= y) return null;
return { x, y, w: right - x, h: bottom - y };
}
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
function clipToOverflowAncestors(
node: CapturedNode,
rect: Rect,
nodesByBackend: ReadonlyMap<number, CapturedNode>,
): Rect | null {
let clipped: Rect | null = rect;
let parentBackendNodeId = node.parentBackendNodeId;
let guard = 0;
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
const parent = nodesByBackend.get(parentBackendNodeId);
if (!parent) break;
if (parent.rect) {
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
if (clipX || clipY) {
const clipRect = {
x: clipX ? parent.rect.x : clipped.x,
y: clipY ? parent.rect.y : clipped.y,
w: clipX ? parent.rect.w : clipped.w,
h: clipY ? parent.rect.h : clipped.h,
};
clipped = intersectRect(clipped, clipRect);
}
}
parentBackendNodeId = parent.parentBackendNodeId;
guard += 1;
}
return clipped;
}
function excludedByDomPolicy(
node: CapturedNode,
nodesByBackend: Map<number, CapturedNode>,
excludedBackendNodeIds: ReadonlySet<number>,
): boolean {
let current: CapturedNode | undefined = node;
let guard = 0;
while (current && guard <= nodesByBackend.size) {
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
if (current.visuallySuppressed === true) return true;
const attrs = current.attrs;
if (
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
) {
return true;
}
current =
current.parentBackendNodeId === null
? undefined
: nodesByBackend.get(current.parentBackendNodeId);
guard += 1;
}
return false;
}
export function discoverRenderedSurfaces(
documents: CapturedFrameDocument<SemanticAxNode>[],
viewport: Viewport,
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
): RenderedSurface[] {
const surfaces: RenderedSurface[] = [];
for (const document of documents) {
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
for (const node of document.domNodes) {
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
const viewportRect = intersectViewport(node.rect, viewport);
const visibleRect = viewportRect
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
: null;
if (!visibleRect) continue;
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
surfaces.push({
renderingKind: "canvas",
frameId: document.frameId,
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
visibleRect,
paintOrder: node.paintOrder,
...(label ? { label } : {}),
});
}
}
return surfaces;
}
@@ -0,0 +1,4 @@
export { clusterRenderedSurfaces } from "./cluster";
export { discoverRenderedSurfaces } from "./discovery";
export { projectRenderedSurfaces } from "./project";
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
@@ -0,0 +1,100 @@
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
import type { CapturedFrameDocument } from "../frame-capture";
import type { SemanticAxNode } from "../semantic-graph";
import type { RenderedSurfaceGroup } from "./types";
function frameBackendKey(frameId: string, backendNodeId: number): string {
return `${frameId}\u0000${backendNodeId}`;
}
interface ProjectionIndex {
sceneByBackend: ReadonlyMap<string, number>;
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
domByFrame: ReadonlyMap<
string,
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
>;
rootSceneIdByFrame: ReadonlyMap<string, number>;
}
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
const document = index.documentByFrame.get(group.frameId);
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
let parentBackendNodeId = group.representative.parentBackendNodeId;
let guard = 0;
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
if (sceneId !== undefined) return sceneId;
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
guard += 1;
}
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
const owner = index.sceneByBackend.get(
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
);
if (owner !== undefined) return owner;
}
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
}
function buildProjectionIndex(
scene: VomScene,
documents: CapturedFrameDocument<SemanticAxNode>[],
): ProjectionIndex {
const rootSceneIdByFrame = new Map<string, number>();
for (const node of scene.nodes) {
if (
node.frameId &&
!rootSceneIdByFrame.has(node.frameId) &&
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
) {
rootSceneIdByFrame.set(node.frameId, node.id);
}
}
return {
sceneByBackend: new Map(
scene.nodes.flatMap((node) =>
node.backendNodeId === undefined
? []
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
),
),
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
domByFrame: new Map(
documents.map((document) => [
document.frameId,
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
]),
),
rootSceneIdByFrame,
};
}
export function projectRenderedSurfaces(
scene: VomScene,
documents: CapturedFrameDocument<SemanticAxNode>[],
groups: RenderedSurfaceGroup[],
): VomScene {
const index = buildProjectionIndex(scene, documents);
const visualSurfaces: VomVisualSurface[] = [];
for (const group of groups) {
const parentId = nearestSceneParent(index, group);
visualSurfaces.push({
parentId,
backendNodeId: group.representative.backendNodeId,
frameId: group.frameId,
renderingKind: "canvas",
visibleRect: group.representative.visibleRect,
...(group.label ? { label: group.label } : {}),
memberCount: group.members.length,
});
}
return {
...scene,
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
};
}
@@ -0,0 +1,18 @@
import type { Rect } from "@browser-skill/vom";
export interface RenderedSurface {
renderingKind: "canvas";
frameId: string;
backendNodeId: number;
parentBackendNodeId: number | null;
visibleRect: Rect;
paintOrder: number;
label?: string;
}
export interface RenderedSurfaceGroup {
frameId: string;
members: RenderedSurface[];
representative: RenderedSurface;
label?: string;
}
+1
View File
@@ -23,6 +23,7 @@ export type ErrorCode =
export type RpcErrorReason =
| "agent_window_scope"
| "element_not_visible"
| "ref_capability_denied"
| "ref_not_found"
| "selector_not_found"
| "target_not_fillable"
@@ -0,0 +1,105 @@
# Rendered Surface browser fixtures
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
## Serve
From the repository root:
```bash
python3 -m http.server 4173 --bind 127.0.0.1 \
--directory apps/extension/test-fixtures/rendered-surfaces
```
Use an Agent Window large enough to keep the primary matrix visible:
```bash
bsk session start --width 1440 --height 1100
bsk navigate http://127.0.0.1:4173/index.html --session <id>
bsk observe --session <id>
```
Always stop the session after the run.
## Pages
### `index.html`
Expected individual Surface labels:
- `fixture-basic-large`
- `fixture-tiny-8px`
- `fixture-pointer-events-none`
- `fixture-partial-ax`
- `fixture-contained-outer`
- `fixture-contained-inner`
- `fixture-different-parent-a`
- `fixture-different-parent-b`
Expected one grouped Surface:
- `fixture-layered-group`, with `layers=2`
Expected absent:
- `fixture-display-none`
- `fixture-visibility-hidden`
- `fixture-opacity-zero`
- `fixture-parent-opacity-zero`
- `fixture-hidden-attribute`
- `fixture-inert-parent`
- `fixture-aria-hidden-parent`
- `fixture-zero-size`
- `fixture-offscreen-right`
- `fixture-fully-overflow-clipped`
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
### `unlabeled.html`
This page contains eight Canvas elements but must render seven generic Surface markers because
the two coincident layers form one group.
- Every marker uses the stable fallback label `canvas visual surface`.
- Exactly one marker has `layers=2`.
- The two contained canvases remain separate markers.
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
- Blank `aria-label` and `title` values behave as no label.
- Fallback AX content does not suppress the corresponding Surface.
### `frames.html`
Expected Surface labels:
- `fixture-same-origin-frame`
- `fixture-cross-origin-frame`
- `fixture-nested-frame`
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
### `viewport.html`
Run with a `900x700` Agent Window.
- `fixture-partial-viewport` is partly visible and must be represented.
- `fixture-below-fold` is absent initially.
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
### `dynamic.html`
Use the normal DOM buttons and re-observe after each action:
- Initial state: `fixture-dynamic` absent.
- After **Insert canvas**: present.
- After **Resize to zero**: absent.
- After **Restore size**: present with a fresh ref.
- After **Remove canvas**: absent.
## Invariants
- `snapshot` must not emit any of these Surface markers.
- `observe` must emit only currently visible Canvas surfaces.
- Existing DOM/AX controls remain readable alongside Surface markers.
- Missing labels never prevent discovery; naming and discovery are independent.
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
- No fixture URL, id, label, or layout constant may appear in production discovery code.
@@ -0,0 +1,46 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface dynamic matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
button { padding: 8px 12px; }
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
</style>
</head>
<body>
<h1>Dynamic Canvas lifecycle</h1>
<div class="controls">
<button id="insert" type="button">Insert canvas</button>
<button id="zero" type="button">Resize to zero</button>
<button id="restore" type="button">Restore size</button>
<button id="remove" type="button">Remove canvas</button>
</div>
<div id="mount"></div>
<script>
const mount = document.querySelector("#mount");
const getCanvas = () => mount.querySelector("canvas");
document.querySelector("#insert").addEventListener("click", () => {
if (getCanvas()) return;
const canvas = document.createElement("canvas");
canvas.width = 260;
canvas.height = 120;
canvas.setAttribute("aria-label", "fixture-dynamic");
mount.append(canvas);
});
document.querySelector("#zero").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
});
document.querySelector("#restore").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.removeAttribute("style");
});
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
</script>
</body>
</html>
@@ -0,0 +1,28 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 12px; background: #fdfefe; }
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
</style>
</head>
<body>
<canvas id="frame-canvas" width="220" height="90"></canvas>
<script>
const params = new URLSearchParams(location.search);
const canvas = document.querySelector("#frame-canvas");
if (params.get("canvas") === "0") canvas.remove();
else canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
if (params.get("nested") === "1") {
const iframe = document.createElement("iframe");
iframe.title = "nested fixture frame";
iframe.src = "frame.html?label=fixture-nested-frame";
document.body.append(iframe);
}
</script>
</body>
</html>
@@ -0,0 +1,22 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface frame matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
</style>
</head>
<body>
<h1>Frame matrix</h1>
<div class="frames">
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame"></iframe>
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame"></iframe>
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
</div>
</body>
</html>
@@ -0,0 +1,118 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface discovery matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
.same-place { position: relative; width: 180px; height: 90px; }
.same-place > div { position: absolute; inset: 0; }
.same-place canvas { width: 180px; height: 90px; }
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
.tiny { width: 8px; height: 8px; border-width: 1px; }
</style>
</head>
<body>
<h1>Rendered Surface discovery matrix</h1>
<main class="matrix">
<section class="case">
<h2>Visible named canvas</h2>
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Tiny canvas remains addressable</h2>
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
</section>
<section class="case">
<h2>Visible but pointer-events none</h2>
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
</section>
<section class="case">
<h2>Partial AX/fallback content</h2>
<canvas aria-label="fixture-partial-ax" width="180" height="80">
<button type="button">Fallback button must not suppress the surface</button>
</canvas>
</section>
<section class="case">
<h2>Two real rendering layers</h2>
<div class="stack">
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Contained canvases are independent</h2>
<div class="contained">
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
</div>
</section>
<section class="case">
<h2>Same geometry, different parents</h2>
<div class="same-place">
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
</div>
</section>
<section class="case">
<h2>Partially overflow-clipped</h2>
<div class="clip-box">
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
</div>
</section>
<section class="case">
<h2>Fully overflow-clipped</h2>
<div class="clip-box">
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
</div>
</section>
<section class="case">
<h2>CSS-hidden variants</h2>
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
</section>
<section class="case">
<h2>Hidden ancestor variants</h2>
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
</section>
<section class="case">
<h2>Zero-size and offscreen</h2>
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
</section>
</main>
</body>
</html>
@@ -0,0 +1,68 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Unlabeled Rendered Surface matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
.case p { margin: 0 0 8px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
</style>
</head>
<body>
<h1>Unlabeled Rendered Surface matrix</h1>
<main class="matrix">
<section class="case">
<h2>Only engineering attributes</h2>
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Blank explicit names</h2>
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Nearby prose is not a Canvas name</h2>
<p>must-not-be-inferred-as-a-surface-label</p>
<canvas width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Fallback AX content does not name or suppress the Canvas</h2>
<canvas width="180" height="80">
<button type="button">Fallback control remains separate</button>
</canvas>
</section>
<section class="case">
<h2>Unnamed rendering layers still cluster</h2>
<div class="stack">
<canvas width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Unnamed contained canvases stay independent</h2>
<div class="contained">
<canvas width="190" height="100"></canvas>
<canvas width="70" height="40"></canvas>
</div>
</section>
</main>
</body>
</html>
@@ -0,0 +1,22 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface viewport matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { min-height: 1800px; padding: 16px; }
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
</style>
</head>
<body>
<h1>Viewport matrix</h1>
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
</body>
</html>
+2
View File
@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session <id>` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
## Observation priority
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
+20
View File
@@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode;
pub mod reason {
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
@@ -228,6 +229,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
summary: "snapshot ref does not support this operation",
hint: Some(
"visual surface refs are screenshot-only; run `bsk screenshot --ref <ref> --session <id>`",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
summary: "target element is not a <select>",
hint: Some(
@@ -377,6 +385,18 @@ mod tests {
assert!(!info.summary.contains("sandbox"));
}
#[test]
fn ref_capability_denied_explains_visual_surface_refs() {
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
assert_eq!(info.summary, "snapshot ref does not support this operation");
assert!(
info.hint.unwrap().contains("screenshot-only"),
"expected visual-surface-specific hint"
);
assert_eq!(info.exit_code, 1);
}
#[test]
fn session_busy_overrides_timeout_copy() {
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
@@ -1,108 +0,0 @@
/**
* Archive-triggered bsk session cleanup. DSH conversation archival writes
* the workspace registry's durable global state, which the storage domain
* broadcasts as `domain/changed` ({domain: 'workspace', table: ''}) with
* the new `archivedSessionIds`. The watcher diffs that set and stops every
* bsk session owned by a freshly archived conversation — archived sessions
* are hidden from every surface, so their Agent Windows would otherwise
* linger with no way back to them.
*
* Ownership lineage: browser_session_start records the calling agent's
* session id PLUS every ancestor along `header.parentSession` (a subagent's
* browsers are reaped when any ancestor is archived, the root conversation
* included). Lineage is resolved through the host session store; an
* unloaded ancestor simply ends the walk.
*/
import type { Context } from "@deepseek-ai/cordis";
import type { ObservationService } from "./observation";
import type { SessionRegistry } from "./sessions";
/** The wire shape of a `domain/changed` frame (see dsh-storage-domain). */
interface DomainChange {
domain?: string;
table?: string;
value?: unknown;
}
/** The workspace registry's durable global singleton slice we read. */
interface WorkspaceGlobal {
archivedSessionIds?: unknown;
}
/** Structural host session store face: only the lineage read is needed. */
interface SessionStoreLike {
get(id: string): { header: { parentSession?: string } } | undefined;
}
/** Cap on lineage walks — defensive against a malformed parent chain. */
const MAX_LINEAGE_DEPTH = 16;
/**
* The DSH session ids that own a tool call's browser sessions: the calling
* agent's own session plus every ancestor along the seed lineage. Empty
* when the call carried no agent identity (those sessions outlive any
* archive cleanup by design — nothing can name their owner).
*/
export function ownerSessionIds(ctx: Context, agentId: string | undefined): string[] {
if (agentId === undefined) return [];
const store = ctx.get("sessions") as SessionStoreLike | undefined;
const ids: string[] = [];
let current: string | undefined = agentId;
for (let depth = 0; current !== undefined && depth < MAX_LINEAGE_DEPTH; depth += 1) {
if (ids.includes(current)) break; // a cycle in stored headers must not loop
ids.push(current);
current = store?.get(current)?.header.parentSession;
}
return ids;
}
/**
* Watch conversation archival and stop the bsk sessions it opened. Returns
* the disposer (plugin unload). In compositions without the workspace
* domain (headless), the event simply never fires — and a context without
* the events mixin degrades to a no-op like the other optional seams.
*/
export function armArchiveCleanup(
ctx: Context,
registry: SessionRegistry,
observation: ObservationService,
): () => void {
// 'domain/changed' lives outside the vendored Events type map, so the
// listener goes through a structural view of the events mixin.
const on = (
ctx as { on?: (event: string, listener: (change: DomainChange) => void) => () => void }
).on;
if (typeof on !== "function") return () => {};
/** Archived ids already accounted for; lazily seeded from the registry. */
let seen: Set<string> | undefined;
const initialize = (): Set<string> => {
if (seen === undefined) {
const registryService = ctx.get("workspaceRegistry") as
| { archivedSessionIds?: readonly string[] }
| undefined;
seen = new Set(registryService?.archivedSessionIds ?? []);
}
return seen;
};
return on.call(ctx, "domain/changed", (change: DomainChange) => {
if (change?.domain !== "workspace" || change?.table !== "") return;
const archived = (change.value as WorkspaceGlobal | undefined)?.archivedSessionIds;
if (!Array.isArray(archived)) return;
const previous = initialize();
const fresh = archived.filter(
(id): id is string => typeof id === "string" && !previous.has(id),
);
seen = new Set(archived.filter((id): id is string => typeof id === "string"));
for (const dshSessionId of fresh) {
for (const sessionId of registry.ownedByDsh(dshSessionId)) {
// stopSession owns the full teardown (kill in-flight tools, queue
// the daemon stop, drop registry + observation entries); a failure
// just leaves the session for idle timeout or unload cleanup.
void observation.stopSession(sessionId).catch(() => {});
}
}
});
}
@@ -30,51 +30,17 @@
color: var(--card-foreground);
}
/* Sidebar carrier: fill the better-sidebar tab area and let the body flex
(the tab content container owns the outer sizing). */
.sidebar-tab {
display: flex;
flex-direction: column;
height: 100%;
min-height: 0;
}
.sidebar-tab > * {
flex: 1;
min-height: 0;
}
/* The BSK product mark in the sidebar tab strip (the source artwork already
carries rounded corners — just soften the square to match). */
.brand-icon {
display: block;
border-radius: 22%;
}
.header {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 8px;
cursor: move;
user-select: none;
touch-action: none;
border-bottom: 1px solid var(--border);
}
/* Only the floating card drags by its header — the sidebar tab does not. */
.header[data-draggable] {
cursor: move;
}
/* In the sidebar the status row is chrome, not a card edge: drop its bottom
border. A fixed-height bordered row can never track the shell header's
hairline across the panel boundary (its height varies per window), and
the BSK warm border tint clashes with the sidebar's neutral hairlines —
borderless, the stage's own background provides the separation. */
.sidebar-tab .header {
border-bottom: none;
}
.status-text {
flex: 1;
min-width: 0;
@@ -165,12 +131,6 @@
border-top: 1px solid var(--border);
}
.actions-group {
display: inline-flex;
align-items: center;
gap: 4px;
}
.tool-wrap {
position: relative;
display: inline-flex;
@@ -205,14 +165,8 @@
color: var(--destructive);
}
/* Stop session: danger-tinted like interrupt. */
.tool-stop {
color: var(--destructive);
}
/* Hover the wrap (not just :hover on the button) so the tooltip target
lights the icon up too. The corner resize handles sit above this bar, so
the outer sliver of the corner-most buttons starts a resize instead. */
/* Hover the wrap (not just :hover on the button) so the tooltip target and
the left-corner resize overlay still light the icon up. */
.tool-wrap:hover .tool-button:not(:disabled) {
color: var(--foreground);
background: var(--accent);
@@ -227,15 +181,6 @@
background: var(--accent);
}
/* The armed (confirm) state of the stop button: solid destructive fill so
the second click reads as final. Written at the wrap-hover specificity so
the hover rules above never wash it out. */
.tool-button.tool-stop-armed,
.tool-wrap:hover .tool-button.tool-stop-armed {
color: var(--destructive-foreground);
background: var(--destructive);
}
.hint {
position: absolute;
bottom: calc(100% + 4px);
@@ -258,16 +203,13 @@
right: 0;
}
/* Invisible hit targets — no grip glyph; all four corners resize. Must sit
above the bottom .actions bar (z-index 3): it spans the full card width
and would otherwise swallow every pointer event aimed at the sw/se
corners. The top corners already win because .header is not positioned. */
/* Invisible hit targets — no grip glyph; all four corners resize. */
.resize-handle {
position: absolute;
width: 16px;
height: 16px;
touch-action: none;
z-index: 4;
z-index: 2;
}
.resize-handle[data-corner="nw"] {
@@ -3,18 +3,13 @@
// composer's way); collapsed = a status capsule; "pop out" upgrades the same
// content to a native Document PiP window (user gesture required by the
// browser). Multi-session renders a meeting-style strip under the focus view.
// When the dsh-better-sidebar plugin is installed the tracking view moves
// into a sidebar tab instead (see observation-sidebar.tsx) and this floating
// card hides itself via the sidebar-mode flag. Visuals follow the
// BrowserSkill product family: @browser-skill/ui components and oklch tokens
// on a .bsk-obs scope root, so the card reads as BSK's own surface without
// leaking styles into (or inheriting themes from) the shell.
// Visuals follow the BrowserSkill product family: @browser-skill/ui components
// and oklch tokens on a .bsk-obs scope root, so the card reads as BSK's own
// surface without leaking styles into (or inheriting themes from) the shell.
import { cn } from "@browser-skill/ui";
import {
RiArrowDownSLine,
RiCheckLine,
RiCloseCircleLine,
RiErrorWarningLine,
RiPictureInPicture2Line,
RiPushpinFill,
@@ -30,8 +25,6 @@ const IconPip = asIcon(RiPictureInPicture2Line);
const IconDown = asIcon(RiArrowDownSLine);
const IconWarn = asIcon(RiErrorWarningLine);
const IconPin = asIcon(RiPushpinFill);
const IconCloseSession = asIcon(RiCloseCircleLine);
const IconCheck = asIcon(RiCheckLine);
import {
type ReactNode,
@@ -46,12 +39,11 @@ import { createPortal } from "react-dom";
import type { SessionObservation } from "../observation";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { focusOf, statusOf, useObservationView, usePip } from "./observation-view";
import { getSidebarMode, subscribeSidebarMode } from "./sidebar-mode";
// The pure view helpers live in observation-view (shared with the sidebar
// tab); re-export so existing imports of this module keep working.
export { focusOf, statusOf };
/** Minimal Document PiP surface (TS lib.dom lacks it). */
interface DocumentPip {
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
}
interface Point {
x: number;
@@ -69,6 +61,11 @@ const EDGE_MARGIN = 16;
/** Default dock: top-right, just under the shell's top bar (no spacing tokens exist in dsh 0.1). */
const TOP_OFFSET = 64;
function pipApi(): DocumentPip | undefined {
if (typeof window === "undefined") return undefined;
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
}
function clampSize(size: Size, viewport: Size): Size {
const maxW = viewport.w * 0.8;
const maxH = viewport.h * 0.8;
@@ -117,6 +114,23 @@ function formatElapsed(sinceMs: number, nowMs: number): string {
return `${mm}:${ss}`;
}
/**
* Auto-follow focus: the most recently touched session, but never yank focus
* to a session whose latest action failed (errors flag the strip item, they
* do not steal the stage) or one already reported dead.
*/
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
if (sessions.length === 0) return undefined;
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
return healthy ?? byRecency[0];
}
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
if (obs.lastError !== undefined && obs.action === "idle") return "error";
return obs.action === "idle" ? "idle" : "active";
}
/** Compact toolbar icon: no label, hover bubble for the name / semantics. */
function IconAction(props: {
label: string;
@@ -152,83 +166,6 @@ function IconAction(props: {
);
}
/** How long the stop button stays armed before the confirm click expires. */
const STOP_ARM_MS = 3000;
/**
* Stop-session button with a lightweight two-click confirm: the first click
* arms the button (solid red, check icon, short expiry), the second executes
* the stop. No dialog, no layout shift — and a stray single click can never
* close an Agent Window.
*/
function StopSessionAction(props: {
sessionId: string | undefined;
onStop: (sessionId: string) => Promise<boolean>;
}) {
const { sessionId, onStop } = props;
const [hover, setHover] = useState(false);
const [armed, setArmed] = useState(false);
const [stopping, setStopping] = useState(false);
useEffect(() => {
if (!armed) return;
const timer = setTimeout(() => setArmed(false), STOP_ARM_MS);
return () => clearTimeout(timer);
}, [armed]);
// The focused session can vanish mid-arm (stopped elsewhere): disarm.
useEffect(() => {
if (sessionId === undefined) setArmed(false);
}, [sessionId]);
const disabled = sessionId === undefined || stopping;
const label = stopping
? "Stopping session…"
: armed
? `Confirm stop session ${sessionId ?? ""}`
: `Stop session ${sessionId ?? ""}`;
const hint = stopping
? "Closing the Agent Window…"
: armed
? `Click again to stop session ${sessionId ?? ""} and close its Agent Window.`
: `Stop session ${sessionId ?? ""} and close its Agent Window.`;
return (
<span
className={css["tool-wrap"]}
onPointerEnter={() => setHover(true)}
onPointerLeave={() => setHover(false)}
>
<button
type="button"
className={cn(css["tool-button"], css["tool-stop"], armed && css["tool-stop-armed"])}
disabled={disabled}
aria-label={label}
aria-pressed={armed}
data-armed={armed || undefined}
onClick={() => {
if (sessionId === undefined || stopping) return;
if (!armed) {
setArmed(true);
return;
}
setArmed(false);
setStopping(true);
const target = sessionId;
void onStop(target).finally(() => setStopping(false));
}}
>
{armed ? <IconCheck size={16} /> : <IconCloseSession size={16} />}
</button>
{hover ? (
<span className={css.hint} role="tooltip">
{hint}
</span>
) : null}
</span>
);
}
/** Flat status dot, specced after the BSK popup's ConnectionStatusIndicator. */
function StatusDot({ state }: { state: "active" | "idle" | "error" | "dead" }) {
const color =
@@ -309,8 +246,8 @@ function StripItem(props: {
);
}
/** The floating card / PiP / sidebar-tab shared content. */
export function OverlayBody(props: {
/** The floating card / PiP shared content. */
function OverlayBody(props: {
store: ObservationClientStore;
focus: SessionObservation | undefined;
sessions: readonly SessionObservation[];
@@ -373,7 +310,6 @@ export function OverlayBody(props: {
<div
className={css.header}
data-testid="obs-header"
data-draggable={onHeaderPointerDown !== undefined || undefined}
onPointerDown={onHeaderPointerDown}
role="presentation"
>
@@ -427,21 +363,15 @@ export function OverlayBody(props: {
</div>
) : null}
<div className={css.actions}>
<div className={css["actions-group"]}>
<IconAction
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
hint="Stop the current browser action."
disabled={!canInterrupt}
danger
onClick={onInterrupt}
>
<IconStop size={16} />
</IconAction>
<StopSessionAction
sessionId={focus?.sessionId}
onStop={(sessionId) => store.stopSession(sessionId)}
/>
</div>
<IconAction
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
hint="Stop the current browser action."
disabled={!canInterrupt}
danger
onClick={onInterrupt}
>
<IconStop size={16} />
</IconAction>
{onPopOut !== undefined ? (
<IconAction
label="Pop out into a mini window"
@@ -457,17 +387,25 @@ export function OverlayBody(props: {
);
}
/** Clone the host document's style/link nodes into a PiP window. */
function cloneStylesInto(pipWindow: Window): void {
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
pipWindow.document.head.appendChild(node.cloneNode(true));
}
}
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store);
// While the better-sidebar plugin carries the tracking view, this floating
// card (and its capsule) stays out of the way; an already-open PiP window
// keeps its portal until the user closes it.
const sidebarMode = useSyncExternalStore(subscribeSidebarMode, getSidebarMode);
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
useEffect(() => {
store.start();
return () => store.stop();
}, [store]);
const [collapsed, setCollapsed] = useState(false);
const [pos, setPos] = useState<Point | null>(null);
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
const { pipWindow, pipSupported, popOut } = usePip();
const [pipWindow, setPipWindow] = useState<Window | null>(null);
const [pinnedId, setPinnedId] = useState<string | null>(null);
const dragRef = useRef<{
kind: "move" | "resize";
corner?: ResizeCorner;
@@ -476,6 +414,23 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
base: Point & Size;
} | null>(null);
// Elapsed-time ticker: 1s while anything is active.
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!anyActive) return;
const timer = setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, [anyActive]);
// Focus: the pinned session wins while it still exists; otherwise auto-follow.
const pinned =
pinnedId !== null ? snapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
const focus = pinned ?? focusOf(snapshot.sessions);
const onTogglePin = useCallback((sessionId: string) => {
setPinnedId((current) => (current === sessionId ? null : sessionId));
}, []);
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
const onPointerMove = useCallback((event: PointerEvent) => {
@@ -556,21 +511,34 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
document.addEventListener("pointerup", onPointerUp);
};
const popOut = async (): Promise<void> => {
const pip = pipApi();
if (pip === undefined) return;
try {
const win = await pip.requestWindow({ width: size.w, height: size.h });
cloneStylesInto(win);
win.addEventListener("pagehide", () => setPipWindow(null));
setPipWindow(win);
} catch {
// requestWindow rejects without a user gesture (or when the window was
// closed mid-request): stay on the in-page card, no state change.
}
};
// Hidden while no owned session exists (and no PiP is up).
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
const body = (
<OverlayBody
store={store}
focus={focus}
sessions={snapshot.sessions}
available={snapshot.available}
pinnedId={pinnedId}
pinnedId={pinned !== undefined ? pinnedId : null}
onTogglePin={onTogglePin}
now={now}
inPip={pipWindow !== null}
onPopOut={
pipWindow === null && pipSupported
? () => popOut({ width: size.w, height: size.h })
: undefined
}
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
onHeaderPointerDown={pipWindow === null ? beginMove : undefined}
/>
@@ -580,12 +548,6 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
return createPortal(body, pipWindow.document.body);
}
// The sidebar tab is the carrier now — no floating card, no capsule.
if (sidebarMode) return null;
// Hidden while no owned session exists (and no PiP is up).
if (snapshot.sessions.length === 0) return null;
if (collapsed) {
const state = focus !== undefined ? statusOf(focus) : "idle";
return (
@@ -1,8 +0,0 @@
/**
* The BrowserSkill product mark (apps/extension/assets/logo.png, downscaled
* to 32px and inlined): the sidebar tab icon, so the tracking view reads as
* BSK's own surface next to better-sidebar's built-in "browser" tab.
* Regenerate with: resize the source to 32x32 PNG and replace the payload.
*/
export const BSK_LOGO_URL =
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAJGklEQVR42m2X248c2V3HP79zTlV19/TcPOOZWY+vsdmQFdngTQJswgs3hQ15iCJslAey+8xDAIHgH+CBNxIpj0gkEXnAKyKQWITEJlKWXSssEMfJRmuMHa/W9njHMz3Tl+nprsv5/XiomlugpdN16lR1n9/1e75fAbAbeLlO3Pz2ly4vjO7+sRbF58oiP49FEREOPwaIYQjS3IvUy0fP66kAZoYJIM7SpPVY0uy14cyzXzn9u9+4c+MG/vp1otiNa16uvxr7X/+1L3bK3lcTyU9P80ilR/uKCGYHOxxd6s2kmdaGIVYb1qyY1e8HD63UUblstwirf9J95fW/sRvXvAD0v/6Z6/O68Xf5/pS8kigiTsTkuCfIwfzAxQN37chjaAyt594JIXE4gbKMhog6Nd/qthi7lVe6L3/nG7Jz4/fPJ9u3b4U4WSwiKog/EcsTLh+PSuO3GXYs/D44QnBgyniQs7OV0+8bOxPPM6c9l660VEyF0Nmrlp9/IYT+/S/PUJ4a5C46EW8HnjVhBKjXaHJfz9Xq95xzhMThHZR5xXBrwtPNnP6eRzvzhJWzrF1d5UI759//9iYra5lrt1ycDeXsaOf+n4Y4HX8+r9SciasLS/6P12LHoyF47wipYKpM9gqePpyyvVMxjBn+1DILH17l0rkZlhYcLetT3n/A5FGfTtvTCh4ibjpVI81/J0il62V1rNStLp6j4jowxwiJh2iMevtsbU3pjYQim6Nz9hJLH13i8mrGbJgSxjuUvQfEjT2GOw4Zddh0BakPtFPH/jRKqaCSrwetKi9mKIIYhyFuOg6TurCcF7ae5ty5t48/vcTiz3+Ei+szLC06WnGA7TykfNCnmObk5nAtj+53yDcTZteMwZOChbkMVa3/14DCJBBBFBxyopndUUBqwwxu/XjAJ778Bc4ul+R3fgyj9yg/GDMpFZxHgoe0hQ9QDRzFI48zqHzJeKhcvNKiLAyiQ1BwgsOcqNZ9Xo96rgaVgmqdiBACH740y/1/+j4b7w1x80tUkwLM4VptJAQMcN6o+p7iYQIqhBQmsUIrR7cd0NI12ODAwJnWTpvZoRGqdRrS1CECeR7Z2NzHRBjeecgbX/kut39UUlx5ATe3gFUlAM5Tb/4oHCJl0oadUcFsK8E5sFh7ZaqgEFBr3KyhxAySxDMplDv3BuztlmhhZC6QqONDrXkW28Kbf/1d7A9+i6vPr7N/e5vQDZQ7jvxxUqdPwFRwHaP3sGR1boZYGabWwHdd7MHMMBUMh5mRJp6tnZx33tllJbS53J1jZj6BKpKPpkhZ8e5mj3Nf/FV+8ZOnmNz6Pr6TUvUcxeOAc0cN6xwUoWIyUhZXMqqq6bKmrQ0hWBQzdXX+nGOwV3LnJ30+trjE8mxGUUVG/Qn7owkt7/hpb5fwmat84qXz5P95EwmOqh8oNkKTSkFqICTJYDgpCOppZYGyjIfgdoApQRScCtEgDZ5b93dxuWd3OuWD8T5pqbSLyEyWcHdrh+S3r/Kpz15i/z/eQhIh9hOKjQRTyBYgFhAndSpDW+gNC+ZbGR6jjM3BVsNsXQNqhqpiOPIicuXcLFvdKT7xtNXTG+X0Bopt9Wi9dJUXX7rI5O23kNQRB4F8I0CEzgr0/B7t2YB/2iJOQDrK8HHJ+dkZqqI+JfUAYRXMGc6pM1MBBa2Ubttz5ews51c6XFjr8OJHT5N0HPIbz/Ppz11h/+03IXVUg4T8ccAidFaNp7LHzdd2uXdvj866gQq5L8nHMD+TUJUG0aAyJBqidZQcUfFqeK2vlEo1rYh5iUXl6eMB5eoKv37tOUb/9q+4LBD7gXKj3ry7Ak/iHrdeH/ArC7P07ykPekMWzkFvkNMmkHkgagNoDciogYKzaGg0YgSNYLEJj0ndfiGj6o/Z3i5or5+j2BTKJwlawcwKbMQ9fvK9Ib+QtgitQMg8d384wVYLdocls1mCad1+Fg1TDgeV4UxFTB2ow44NjQ5rwcwZ+MjqmLe/dZOBLOHHCeUUZteED+Ie73xvyHNpmyx1TERozXieW1/mO/+yzWhTWVtsU5aCqKvRT4+GmRDEHGJ6yPPMBHNGeqbCdwqs02X9l14g/a8H3PzWm/zyp1ZZXgn8tDfgnbeGPN/qIEDSyhh7w2OcPZWRsESWCq3EoVFPksZDMuUIps7qLjDUQBxkZ0tcu8KSGdIXPo3OLbOSdfj4YMIPbg6ZXxux9wRmWimFKe0k0J7v8H5vQHchpSoiC536bKhKEDlCJzM7JLJqhqtxoEYoUcEFw/Yduh8Jl38OP7eAlBOm29ssziV87JlluqMOL15ZZWVlhu0yMrfYxQQmZUU3TbCmnog1mRE1xOp9xOoaEwVRI9QnUGOECLoPNlcRlucJ6xeIgx3K/3kX23pINWnRdnBxtYNWkbkkMJ5JSLKEwaSgRFlsJWh1mNBDfsEBzZGaYIiBIgRtImCmmArhlBFOR1Q9xZ3b6OOHWFnifEq1U/84TpXgHL4hUh7hv3t91pY6eCeUpTYs2o6RisYEO6B5hpoSRCWaanNAGP5UVZ+r+YB4dxcbZ4QVx/Q9wSYg/uDYNvKyoorwgyfbtDuBC6c65HmFa5Jsh56f3NgO4iGeYC68n8Ty2WkphAUVv2hoLuggoXgUcJkRh0rVF7wHi3Vw1YxuSMh1zMpcmw8tdynzCmmCL1JjzQG3lpNU37xAdNnjYC79xywUfzbJNboEX214iqeeuN/IrkKw6BAvNUiJ1a2K0HKOT144jQBFoTWRayrc9LhWa1TTUTNqJ038wCf/LO//xZfWWw9u/zCt8uWpSiTixZ3UeJzQPkerdkyq1RryZwVN87CJBALR0ExULMtG4/VnPy4AG3/4m19YGG/9fT7NmZpEhzg5lCNHOo//774u7UYp2WEt1XrSftYMTcF32ynbrfmXz3ztjW86u3bNn/nq69/eaZ/6PUlbm/PB+YSapUozMG2uB0M5fI7WSKoKasjBu6qHczEjwWQ2iJc03XmaLb5y5mtvfNOuNeLUruHlVeK7f/7yxbXBu39U5sXnq7I8L2pH6rwRgCZyUpeKnZSOB5K2EbT1l5hPk0c+zV7bXXzmry7/5T/cvXENf/1V4v8CSYIrURx3kHgAAAAASUVORK5CYII=";
@@ -1,9 +1,7 @@
/**
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
* toolview plus the observation overlay (live thumbnails + interrupt). The
* default carrier is a floating card on the `shell.overlay` seat; when the
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
* moves into a sidebar tab instead (see observation-sidebar.tsx).
* toolview plus the observation overlay (live thumbnails + interrupt) floating
* over the shell via the `shell.overlay` seat.
*/
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
@@ -17,7 +15,6 @@ import { createElement } from "react";
import "./bsk-tokens.nomodule.css";
import "./bsk-ui.nomodule.css";
import { ObservationOverlay } from "./ObservationOverlay";
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
import { ScreenshotToolView } from "./ScreenshotToolView";
@@ -84,14 +81,4 @@ export function apply(ctx: ClientContext): void {
createElement(ObservationOverlay, { store }),
),
);
// Optional carrier upgrade: when the dsh-better-sidebar plugin is installed,
// its service moves the tracking view into a sidebar tab (the floating
// overlay hides itself through the sidebar-mode flag). In profiles without
// the sidebar plugin this fiber never runs and nothing changes.
ctx.inject(["betterSidebar"], (injected) =>
registerObservationSidebar(
(injected as unknown as { betterSidebar: BetterSidebarLike }).betterSidebar,
store,
),
);
}
@@ -1,256 +0,0 @@
// better-sidebar carrier for the observation view. When the
// dsh-better-sidebar plugin is installed its client publishes a
// `betterSidebar` cordis service; the client entry then runs the
// registration below and the tracking view moves from the floating card
// into a single-instance sidebar tab (Document PiP pop-out stays available
// from inside the tab). Detection is purely service-based — profiles
// without the sidebar plugin never start this fiber and keep the floating
// overlay.
import { createElement, type ReactNode } from "react";
import { createPortal } from "react-dom";
import { BSK_LOGO_URL } from "./brand-icon";
import { OverlayBody } from "./ObservationOverlay";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { useObservationView, usePip, visibleToScope } from "./observation-view";
import { setSidebarMode } from "./sidebar-mode";
/** The tab title — "Browser Skill", distinct from the sidebar's built-in "browser" tab. */
const TAB_TITLE = "Browser Skill";
/** Tab strip icon: the BrowserSkill product mark at the requested size. */
function TabIcon({ size }: { size: number }) {
return (
<img
src={BSK_LOGO_URL}
width={size}
height={size}
alt=""
aria-hidden
className={css["brand-icon"]}
/>
);
}
/**
* Structural mirrors of dsh-better-sidebar's client service surface (only
* the slices this integration touches — see the upstream
* lib/types/client/service.d.ts). Declared locally so the plugin carries no
* dependency on the sidebar package; the service contract has been stable
* since v0.4.0 and newer capabilities arrive behind its `features` list.
*/
export interface SidebarTabLike {
id: string;
type: string;
title: string;
}
export interface SidebarLeafLike {
kind: "leaf";
id: string;
tabs: SidebarTabLike[];
active: string | null;
}
export interface SidebarSplitLike {
kind: "split";
id: string;
dir: "row" | "col";
sizes: number[];
children: SidebarNodeLike[];
}
export type SidebarNodeLike = SidebarLeafLike | SidebarSplitLike;
export interface SidebarStateLike {
splits: SidebarNodeLike;
bottomSplits: SidebarNodeLike;
/** Whether the right panel is expanded (the merged drawer on narrow screens). */
panelOpen?: boolean;
}
export interface SidebarSnapshotLike {
sessionId?: string;
state?: SidebarStateLike;
}
/** Props every tab component receives (the slices we read). */
export interface TabComponentPropsLike {
/** The conversation this sidebar instance belongs to. */
scope: { sessionId: string };
}
export interface TabDescriptorLike {
id: string;
title: string | (() => string);
icon?: ReactNode | ((size: number) => ReactNode);
order?: number;
/** Single-instance: opening focuses the existing tab instead of duplicating. */
single?: boolean;
/** Small pill on the tab strip; null/undefined hides it. */
badge?: (
ctx: unknown,
scope: { sessionId: string },
state: unknown,
) => string | number | null | undefined;
component: (props: TabComponentPropsLike) => ReactNode;
}
export interface BetterSidebarLike {
registerTab(descriptor: TabDescriptorLike): () => void;
openTab(seed: {
type: string;
title?: string;
/** Content seed (lands on tab.path); content opens expand the panel. */
path?: string;
}): void;
getSnapshot(): SidebarSnapshotLike;
/** Sidebar state feed (v0.12+): session switches, state and prefs changes. */
subscribeState?: (listener: () => void) => () => void;
isTabEnabled(id: string): boolean;
}
/** The registered tab type id (also the SidebarTab.type value). */
export const OBSERVATION_TAB_TYPE = "browserskill:observation";
/**
* Inert content seed carried on auto-opened tabs: its mere presence makes
* the sidebar treat the open as a content open (expanding the hosting panel
* so the tracking view lands in sight). Never read by our component.
*/
export const OBSERVATION_TAB_PATH = "browser-skill:observation";
/**
* The observation tab body: the same OverlayBody the floating card renders,
* minus the card chrome (no drag header, no collapse — the sidebar tab bar
* owns those), plus the PiP pop-out upgrade. The view is scoped to the
* sidebar's conversation: only browser sessions started by it (or its
* descendants) show here — the floating card keeps the global view.
*/
export function ObservationSidebarTab({
store,
scopeId,
}: {
store: ObservationClientStore;
scopeId: string;
}) {
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store, scopeId);
const { pipWindow, pipSupported, popOut } = usePip();
const body = (
<OverlayBody
store={store}
focus={focus}
sessions={snapshot.sessions}
available={snapshot.available}
pinnedId={pinnedId}
onTogglePin={onTogglePin}
now={now}
inPip={pipWindow !== null}
onPopOut={pipWindow === null && pipSupported ? () => popOut() : undefined}
/>
);
if (pipWindow !== null) {
return createPortal(body, pipWindow.document.body);
}
return <div className={css["sidebar-tab"]}>{body}</div>;
}
function* leafNodes(node: SidebarNodeLike): Generator<SidebarLeafLike> {
if (node.kind === "leaf") {
yield node;
return;
}
for (const child of node.children) yield* leafNodes(child);
}
/** Whether a tab of our type is already open in either sidebar workbench. */
export function observationTabOpen(state: SidebarStateLike | undefined): boolean {
if (state === undefined) return false;
for (const root of [state.splits, state.bottomSplits]) {
for (const leaf of leafNodes(root)) {
if (leaf.tabs.some((tab) => tab.type === OBSERVATION_TAB_TYPE)) return true;
}
}
return false;
}
/**
* Register the observation sidebar tab and flip the carrier flag. Returns
* the disposer the cordis fiber invokes when the sidebar service goes away
* (plugin unload/HMR): the floating overlay then resumes as the carrier.
*/
export function registerObservationSidebar(
service: BetterSidebarLike,
store: ObservationClientStore,
): () => void {
setSidebarMode(true);
// Hold the feed for the whole sidebar lifetime so the auto-open watcher
// sees new sessions even while the tab itself is closed.
store.acquire();
const disposeTab = service.registerTab({
id: OBSERVATION_TAB_TYPE,
title: TAB_TITLE,
icon: (size: number) => createElement(TabIcon, { size }),
single: true,
// The badge counts only sessions visible to the tab strip's own
// conversation (global sessions from other conversations stay hidden,
// mirroring the tab body's scoped view).
badge: (_ctx, scope) => {
const count = store
.getSnapshot()
.sessions.filter((s) => visibleToScope(s, scope.sessionId)).length;
return count > 0 ? count : null;
},
component: (props) =>
createElement(ObservationSidebarTab, { store, scopeId: props.scope.sessionId }),
});
// Auto-open the tab when a browser session VISIBLE TO the active
// conversation appears (and right away when one is already live). The
// evaluation runs on observation publishes AND on sidebar state changes:
// the latter covers the sidebar store's async init (its state is
// undefined for the first beats after page load) and conversation
// switches. The open carries a content seed (`path`): only content opens
// land in sight — the sidebar expands the hosting panel for them. While
// the panel is OPEN an existing tab is never re-focused — the user may
// be reading another page on purpose; while it is collapsed a NEW
// session (0→N) nudges the tab back into sight, mirroring how the
// floating card used to reappear.
const activeVisibleCount = (): number => {
const activeId = service.getSnapshot().sessionId;
if (activeId === undefined) return 0;
return store.getSnapshot().sessions.filter((s) => visibleToScope(s, activeId)).length;
};
let previousVisible = activeVisibleCount();
const evaluate = (): void => {
const count = activeVisibleCount();
const { state, sessionId } = service.getSnapshot();
if (
state !== undefined &&
sessionId !== undefined &&
service.isTabEnabled(OBSERVATION_TAB_TYPE)
) {
const open = observationTabOpen(state);
if (count > 0 && !open) {
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
} else if (previousVisible === 0 && count > 0 && open && state.panelOpen === false) {
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
}
}
previousVisible = count;
};
evaluate();
const unsubscribe = store.subscribe(evaluate);
const unsubscribeState = service.subscribeState?.(evaluate);
return () => {
unsubscribe();
unsubscribeState?.();
disposeTab();
store.release();
setSidebarMode(false);
};
}
@@ -53,7 +53,6 @@ export interface OverlaySnapshot {
const STATE_URL = "/bsk-observation/state";
const EVENTS_URL = "/bsk-observation/events";
const INTERRUPT_URL = "/bsk-observation/interrupt";
const STOP_URL = "/bsk-observation/stop";
function revoke(url: string | undefined): void {
if (url !== undefined && typeof URL.revokeObjectURL === "function") {
@@ -79,8 +78,6 @@ export class ObservationClientStore {
};
private available = true;
private started = false;
/** Refcount of mounted consumers (overlay card, sidebar tab, sidebar fiber). */
private consumers = 0;
constructor(private readonly deps: ObservationClientDeps) {}
@@ -192,23 +189,6 @@ export class ObservationClientStore {
this.publish();
}
/**
* Hold the feed for one consumer's lifetime: the stream starts with the
* first holder and stops with the last release. Several carriers can share
* the store (the floating card, the better-sidebar tab, and the sidebar
* integration fiber) without one unmount killing the others' updates.
*/
acquire(): void {
this.consumers += 1;
if (this.consumers === 1) this.start();
}
release(): void {
if (this.consumers === 0) return;
this.consumers -= 1;
if (this.consumers === 0) this.stop();
}
/** Forget one session's tracked + held frames, revoking their blob URLs. */
private dropThumb(sessionId: string): void {
const attachmentId = this.thumbBySession.get(sessionId);
@@ -312,23 +292,4 @@ export class ObservationClientStore {
return false;
}
}
/**
* Stop one session and close its Agent Window. The session's removal
* arrives through the SSE remove event — no local state is touched here.
*/
async stopSession(sessionId: string): Promise<boolean> {
try {
const res = await this.deps.fetchFn(STOP_URL, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ sessionId }),
});
if (!res.ok) return false;
const body = (await res.json()) as { stopped?: boolean };
return body.stopped === true;
} catch {
return false;
}
}
}
@@ -1,148 +0,0 @@
/**
* Shared view logic for the observation carriers (the floating overlay card
* and the better-sidebar tab): the store-backed view model (snapshot, focus
* pinning, elapsed ticker) and the Document PiP pop-out. Extracted from
* ObservationOverlay so both carriers run the same focus/interrupt behavior
* without duplicating hooks.
*/
import { useCallback, useEffect, useState, useSyncExternalStore } from "react";
import type { SessionObservation } from "../observation";
import type { ObservationClientStore, OverlaySnapshot } from "./observation-store";
/**
* Auto-follow focus: the most recently touched session, but never yank focus
* to a session whose latest action failed (errors flag the strip item, they
* do not steal the stage) or one already reported dead.
*/
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
if (sessions.length === 0) return undefined;
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
return healthy ?? byRecency[0];
}
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
if (obs.lastError !== undefined && obs.action === "idle") return "error";
return obs.action === "idle" ? "idle" : "active";
}
/** Minimal Document PiP surface (TS lib.dom lacks it). */
export interface DocumentPip {
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
}
export function pipApi(): DocumentPip | undefined {
if (typeof window === "undefined") return undefined;
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
}
/** Clone the host document's style/link nodes into a PiP window. */
export function cloneStylesInto(pipWindow: Window): void {
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
pipWindow.document.head.appendChild(node.cloneNode(true));
}
}
export interface ObservationView {
readonly snapshot: OverlaySnapshot;
/** Pinned session wins while it still exists; otherwise auto-follow. */
readonly focus: SessionObservation | undefined;
readonly pinnedId: string | null;
readonly onTogglePin: (sessionId: string) => void;
/** 1s ticker while anything is active (drives the elapsed readout). */
readonly now: number;
}
/**
* Whether one session is visible on a surface scoped to one DSH
* conversation: sessions started by that conversation or its descendants
* (the lineage ancestors ride along on the entry). Untracked sessions (no
* owner recorded) are hidden from scoped surfaces but stay in global views.
*/
export function visibleToScope(obs: SessionObservation, scopeId: string): boolean {
return obs.dshSessionIds?.includes(scopeId) === true;
}
/**
* The store-backed observation view model. Holds the feed for the component
* lifetime (refcounted — overlapping carriers never kill each other's
* stream). `scopeId` narrows the view to one DSH conversation's sessions
* (the better-sidebar tab); undefined keeps the global view (floating
* card, PiP).
*/
export function useObservationView(
store: ObservationClientStore,
scopeId?: string,
): ObservationView {
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
useEffect(() => {
store.acquire();
return () => store.release();
}, [store]);
const scopedSnapshot: OverlaySnapshot =
scopeId === undefined
? snapshot
: {
...snapshot,
sessions: snapshot.sessions.filter((s) => visibleToScope(s, scopeId)),
};
const [pinnedId, setPinnedId] = useState<string | null>(null);
const pinned =
pinnedId !== null ? scopedSnapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
const focus = pinned ?? focusOf(scopedSnapshot.sessions);
const onTogglePin = useCallback((sessionId: string) => {
setPinnedId((current) => (current === sessionId ? null : sessionId));
}, []);
const anyActive = scopedSnapshot.sessions.some((s) => s.action !== "idle");
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!anyActive) return;
const timer = setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, [anyActive]);
return {
snapshot: scopedSnapshot,
focus,
pinnedId: pinned !== undefined ? pinnedId : null,
onTogglePin,
now,
};
}
export interface PipHandle {
/** The open PiP window, or null while the content renders in its carrier. */
readonly pipWindow: Window | null;
/** Whether the browser exposes Document PiP (Chrome-only today). */
readonly pipSupported: boolean;
/**
* Upgrade the content into a native PiP window. Rejects silently without a
* user gesture (or when closed mid-request): the carrier keeps the content.
*/
readonly popOut: (size?: { width: number; height: number }) => void;
}
export function usePip(): PipHandle {
const [pipWindow, setPipWindow] = useState<Window | null>(null);
const popOut = useCallback((size?: { width: number; height: number }): void => {
const pip = pipApi();
if (pip === undefined) return;
void pip
.requestWindow(size)
.then((win) => {
cloneStylesInto(win);
win.addEventListener("pagehide", () => setPipWindow(null));
setPipWindow(win);
})
.catch(() => {});
}, []);
// Unmounting the carrier (tab close, conversation switch, plugin HMR) does
// not destroy the PiP browsing context: close it ourselves or it survives
// as a blank window. close() on an already-closed window is a no-op.
useEffect(() => () => pipWindow?.close(), [pipWindow]);
return { pipWindow, pipSupported: pipApi() !== undefined, popOut };
}
@@ -1,26 +0,0 @@
/**
* Carrier switch for the observation view: while the better-sidebar
* integration fiber is alive (the dsh-better-sidebar plugin provides its
* `betterSidebar` service), the tracking view lives in a sidebar tab and the
* floating overlay card/capsule hides itself. The flag is a tiny external
* store so the overlay can read it through useSyncExternalStore and flip
* without a remount when the sidebar plugin (un)loads.
*/
let active = false;
const listeners = new Set<() => void>();
export function getSidebarMode(): boolean {
return active;
}
export function subscribeSidebarMode(listener: () => void): () => void {
listeners.add(listener);
return () => listeners.delete(listener);
}
export function setSidebarMode(next: boolean): void {
if (active === next) return;
active = next;
for (const listener of [...listeners]) listener();
}
@@ -12,7 +12,6 @@
import type { Context } from "@deepseek-ai/cordis";
import Schema from "@deepseek-ai/schemastery";
import { armArchiveCleanup } from "./archive-cleanup";
import { armLazyTools } from "./lazy-tools";
import { ObservationService } from "./observation";
import { registerObservationRoutes } from "./observation-http";
@@ -108,10 +107,6 @@ export function apply(
removeRoutes = registerObservationRoutes(injected, observation);
return () => removeRoutes();
});
// Reap a conversation's browsers when the conversation itself is archived:
// archived sessions are hidden from every surface, so their Agent Windows
// would otherwise linger unreachable until idle timeout or unload.
const disarmArchiveCleanup = armArchiveCleanup(ctx, registry, observation);
// Non-blocking install probe: warn early when bsk is missing instead of
// failing the first tool call with a bare spawn error. Uses --version on
@@ -137,7 +132,6 @@ export function apply(
removeSuite();
unregisterSkill();
removeRoutes();
disarmArchiveCleanup();
observation.dispose();
runner.killAll();
const stops = registry
@@ -150,7 +144,6 @@ export function apply(
});
}
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
export { ObservationService } from "./observation";
export { registerObservationRoutes } from "./observation-http";
@@ -8,7 +8,6 @@
* GET /bsk-observation/state → { sessions, available }
* GET /bsk-observation/events → SSE stream of ObservationEvent
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
* POST /bsk-observation/stop → body {sessionId} → {stopped: boolean}
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
*
* Routes exist only when a webServer service is mounted (web composition);
@@ -167,42 +166,6 @@ export function registerObservationRoutes(
});
},
}),
webServer.register({
kind: "exact",
path: `${ROUTE_BASE}/stop`,
handler: (req, res) => {
if (req.method !== "POST") {
sendJson(res, 405, { error: "method not allowed" });
return;
}
if (fenceRejected(req, res)) return;
let body = "";
req.on("data", (chunk: Buffer | string) => {
body += chunk;
});
req.on("end", () => {
let sessionId: string | undefined;
try {
const parsed = JSON.parse(body || "{}") as { sessionId?: unknown };
if (typeof parsed.sessionId === "string" && parsed.sessionId !== "") {
sessionId = parsed.sessionId;
}
} catch {
sendJson(res, 400, { error: "invalid JSON body" });
return;
}
// A destructive call always names its target (never "current").
if (sessionId === undefined) {
sendJson(res, 400, { error: "sessionId required" });
return;
}
void observation.stopSession(sessionId).then(
(stopped) => sendJson(res, 200, { stopped }),
() => sendJson(res, 500, { error: "stop failed" }),
);
});
},
}),
webServer.register({
kind: "prefix",
path: `${ROUTE_BASE}/thumbnail`,
@@ -39,13 +39,6 @@ export interface SessionObservation {
* greys it out until it is stopped/removed. No more frames are requested.
*/
dead?: boolean;
/**
* The DSH conversations this session belongs to (the starting agent's
* session plus its seed-lineage ancestors), recorded at start. Scoped
* surfaces (the better-sidebar tab) filter by it; absent means untracked
* ownership — visible only in the global (unscoped) view.
*/
dshSessionIds?: string[];
}
/** Incremental event carried to subscribers (SSE on the wire). */
@@ -160,13 +153,11 @@ export class ObservationService {
/** Register a fresh owned session (called from browser_session_start). */
addSession(sessionId: string, url?: string): void {
if (!this.deps.options.enabled || this.disposed) return;
const dshSessionIds = this.deps.registry.dshOwnersOf(sessionId);
this.put({
sessionId,
...(url !== undefined ? { url } : {}),
action: "idle",
since: this.scheduler.now(),
...(dshSessionIds.length > 0 ? { dshSessionIds } : {}),
});
this.lastActivity.set(sessionId, this.scheduler.now());
this.scheduleCapture(sessionId, 0);
@@ -237,36 +228,6 @@ export class ObservationService {
return this.deps.runner.killFor(target) > 0;
}
/**
* Stop one owned session and close its Agent Window (the overlay's stop
* button — same end state as `browser_session_stop`). Never waits behind a
* hung in-flight command: tool children are killed first so the session's
* keyed queue drains immediately, and no further captures queue up. A
* session the daemon already forgot (dead strip entries) stops
* idempotently — the goal state (entry gone) is identical.
* @returns whether the session ended up stopped/removed.
*/
async stopSession(sessionId: string): Promise<boolean> {
if (!this.deps.registry.isOwned(sessionId)) return false;
this.cancelCapture(sessionId);
this.deps.runner.killFor(sessionId);
const result = await this.deps.queue.run(sessionId, () =>
this.deps.runner.run(["session", "stop", sessionId], { timeoutMs: 30_000 }),
);
if (result.code !== 0) {
let code: string | undefined;
try {
code = (JSON.parse(result.stdout) as { code?: string }).code;
} catch {
code = undefined;
}
if (code !== "session_not_found") return false;
}
this.deps.registry.remove(sessionId);
this.removeSession(sessionId);
return true;
}
/**
* Read one captured thumbnail from the in-process ring. Powers the plugin's
* own HTTP thumbnail route — frames are plugin-owned runtime data, never
@@ -24,13 +24,6 @@ export interface TrackedSession {
export class SessionRegistry {
private readonly sessions = new Map<string, TrackedSession>();
private currentId: string | undefined;
/**
* bsk session id → the DSH session ids that may clean it up: the agent
* session that started it plus every ancestor along the seed lineage, so
* archiving a conversation at ANY level of the chain reaps its browsers
* (see archive-cleanup.ts).
*/
private readonly dshOwners = new Map<string, ReadonlySet<string>>();
/**
* Slots reserved by in-flight starts. reserveStart/completeStart/abandonStart
* run synchronously around the async spawn, so concurrent starts can never
@@ -79,37 +72,12 @@ export class SessionRegistry {
/** Forget a session; falls back to the most recent remaining one. */
remove(sessionId: string): void {
this.sessions.delete(sessionId);
this.dshOwners.delete(sessionId);
if (this.currentId === sessionId) {
const rest = [...this.sessions.values()];
this.currentId = rest.length > 0 ? rest[rest.length - 1].sessionId : undefined;
}
}
/**
* Record which DSH conversation(s) a freshly started bsk session belongs
* to (the starting agent's session plus its ancestors). No-op without ids
* — e.g. a start whose caller carried no agent identity.
*/
trackOwner(sessionId: string, dshSessionIds: readonly string[]): void {
if (dshSessionIds.length === 0 || !this.sessions.has(sessionId)) return;
this.dshOwners.set(sessionId, new Set(dshSessionIds));
}
/** bsk session ids owned by the given DSH conversation (or its descendants). */
ownedByDsh(dshSessionId: string): string[] {
const owned: string[] = [];
for (const [sessionId, owners] of this.dshOwners) {
if (owners.has(dshSessionId)) owned.push(sessionId);
}
return owned;
}
/** The DSH conversation ids owning one bsk session (empty when untracked). */
dshOwnersOf(sessionId: string): string[] {
return [...(this.dshOwners.get(sessionId) ?? [])];
}
/** Mark an owned session as most recently used (recency order refresh). */
private touch(sessionId: string): void {
const existing = this.sessions.get(sessionId);
@@ -17,7 +17,6 @@ import {
type ToolResult,
type ToolRunContext,
} from "@deepseek-ai/dsh-tools";
import { ownerSessionIds } from "./archive-cleanup";
import { trySaveScreenshot } from "./image";
import { actionForLabel, type ObservationService } from "./observation";
import type { KeyedExecutor } from "./queue";
@@ -246,9 +245,6 @@ export function registerTools(deps: ToolDeps): () => void {
browserInstanceId: reply.browser_instance_id,
startedAtMs: Date.now(),
});
// Ownership for archive cleanup: the calling conversation and its
// ancestors reap this session when any of them is archived.
registry.trackOwner(reply.session_id, ownerSessionIds(deps.ctx, exec.agent?.id));
deps.observation.addSession(reply.session_id, args.url);
try {
if (args.device !== undefined) {
@@ -489,7 +485,8 @@ export function registerTools(deps: ToolDeps): () => void {
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
"probes) with @eN refs. Visual surface refs are screenshot-only; DOM refs support interaction. " +
"Read-only: never submits input.";
register(
defineTool({
name,
@@ -752,12 +749,14 @@ export function registerTools(deps: ToolDeps): () => void {
name: "browser_screenshot",
description:
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
"Returns the image itself when the deployment supports image input, otherwise a file path.",
"This is the supported operation for visual surface refs emitted by browser_observe. Returns " +
"the image itself when the deployment supports image input, otherwise a file path.",
parameters: {
session: SESSION_PARAM,
ref: {
type: "string",
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
description:
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
},
},
output: {
@@ -1,157 +0,0 @@
// Archive-triggered cleanup: lineage resolution for browser-session
// ownership, the registry's owner index, and the domain/changed watcher
// that reaps a freshly archived conversation's bsk sessions.
import { describe, expect, it, vi } from "vitest";
import { armArchiveCleanup, ownerSessionIds } from "../src/archive-cleanup";
import type { ObservationService } from "../src/observation";
import { SessionRegistry } from "../src/sessions";
/** A ctx stub carrying a session store with the given lineage headers. */
function ctxWithSessions(headers: Record<string, { parentSession?: string }>) {
return {
get: (key: string) =>
key === "sessions"
? {
get: (id: string) => (headers[id] === undefined ? undefined : { header: headers[id] }),
}
: undefined,
} as never;
}
describe("ownerSessionIds", () => {
it("walks the seed lineage to the root; empty without an agent identity", () => {
expect(ownerSessionIds(ctxWithSessions({}), undefined)).toEqual([]);
const ctx = ctxWithSessions({
child: { parentSession: "parent" },
parent: { parentSession: "root" },
root: {},
});
expect(ownerSessionIds(ctx, "child")).toEqual(["child", "parent", "root"]);
});
it("stops the walk at an unloaded ancestor", () => {
const ctx = ctxWithSessions({ child: { parentSession: "gone" } });
expect(ownerSessionIds(ctx, "child")).toEqual(["child", "gone"]);
});
it("never loops on a malformed parent cycle", () => {
const ctx = ctxWithSessions({
a: { parentSession: "b" },
b: { parentSession: "a" },
});
expect(ownerSessionIds(ctx, "a")).toEqual(["a", "b"]);
});
});
describe("SessionRegistry owner tracking", () => {
function start(registry: SessionRegistry, sessionId: string): void {
registry.reserveStart();
registry.completeStart({ sessionId, startedAtMs: 1 });
}
it("indexes owners and forgets them on remove; ignores empty/unknown ownership", () => {
const registry = new SessionRegistry(5);
start(registry, "bsk1");
start(registry, "bsk2");
registry.trackOwner("bsk1", ["conv-a", "root"]);
registry.trackOwner("bsk2", ["conv-b"]);
expect(registry.ownedByDsh("root")).toEqual(["bsk1"]);
expect(registry.ownedByDsh("conv-a")).toEqual(["bsk1"]);
expect(registry.ownedByDsh("conv-b")).toEqual(["bsk2"]);
expect(registry.ownedByDsh("nobody")).toEqual([]);
registry.remove("bsk1");
expect(registry.ownedByDsh("root")).toEqual([]);
// Empty owner lists and unknown session ids record nothing.
registry.trackOwner("bsk2", []);
registry.trackOwner("ghost", ["conv-c"]);
expect(registry.ownedByDsh("conv-c")).toEqual([]);
});
});
describe("armArchiveCleanup", () => {
function harness(opts: { archived?: string[] } = {}) {
const registry = new SessionRegistry(5);
const observation = { stopSession: vi.fn(async () => true) };
const listeners = new Set<(change: unknown) => void>();
const ctx = {
get: (key: string) =>
key === "workspaceRegistry" ? { archivedSessionIds: opts.archived ?? [] } : undefined,
on: (event: string, listener: (change: unknown) => void) => {
if (event === "domain/changed") listeners.add(listener);
return () => listeners.delete(listener);
},
};
const emit = (change: unknown) => {
for (const listener of [...listeners]) listener(change);
};
return {
registry,
observation: observation as unknown as ObservationService,
stopSession: observation.stopSession,
emit,
arm: () =>
armArchiveCleanup(ctx as never, registry, observation as unknown as ObservationService),
};
}
function startOwned(registry: SessionRegistry, bskId: string, owners: string[]): void {
registry.reserveStart();
registry.completeStart({ sessionId: bskId, startedAtMs: 1 });
registry.trackOwner(bskId, owners);
}
it("stops every bsk session owned by a freshly archived conversation, until disarmed", () => {
const h = harness();
startOwned(h.registry, "bsk1", ["conv-a", "root"]);
startOwned(h.registry, "bsk2", ["conv-b"]);
const disarm = h.arm();
h.emit({
domain: "workspace",
table: "",
value: { archivedSessionIds: ["root"] },
});
expect(h.stopSession).toHaveBeenCalledTimes(1);
expect(h.stopSession).toHaveBeenCalledWith("bsk1");
// After the disposer runs the watcher is silent again.
disarm();
h.emit({
domain: "workspace",
table: "",
value: { archivedSessionIds: ["root", "conv-b"] },
});
expect(h.stopSession).toHaveBeenCalledTimes(1);
});
it("ignores pre-archived ids, foreign domains, and malformed frames", () => {
const h = harness({ archived: ["old-conv"] });
startOwned(h.registry, "bsk1", ["old-conv"]);
startOwned(h.registry, "bsk2", ["conv-a"]);
h.arm();
// Seeded from the registry: the pre-archived id must not retro-fire.
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["old-conv"] } });
h.emit({ domain: "settings", table: "", value: { archivedSessionIds: ["conv-a"] } });
h.emit({ domain: "workspace", table: "rows", value: { archivedSessionIds: ["conv-a"] } });
h.emit({ domain: "workspace", table: "", value: {} });
expect(h.stopSession).not.toHaveBeenCalled();
// A genuinely new archive still fires.
h.emit({
domain: "workspace",
table: "",
value: { archivedSessionIds: ["old-conv", "conv-a"] },
});
expect(h.stopSession).toHaveBeenCalledWith("bsk2");
});
it("treats a re-archived session as fresh again after unarchive", () => {
const h = harness();
startOwned(h.registry, "bsk1", ["conv-a"]);
h.arm();
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["conv-a"] } });
expect(h.stopSession).toHaveBeenCalledTimes(1);
// Unarchive, then re-archive: the second archival cleans up again.
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: [] } });
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["conv-a"] } });
expect(h.stopSession).toHaveBeenCalledTimes(2);
});
});
@@ -4,7 +4,7 @@
// upgrade/fallback (mocked documentPictureInPicture). NOTE: use RTL's waitFor
// (act-flushing), not vi.waitFor, when asserting store-driven UI updates.
import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { applyResize, ObservationOverlay } from "../../src/client/ObservationOverlay";
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
@@ -157,45 +157,6 @@ describe("ObservationOverlay", () => {
await screen.findByText(/s1 · clicking/);
});
it("stop session: the first click only arms the confirm, the second posts", async () => {
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
render(<ObservationOverlay store={h.store} />);
await screen.findByText(/s1 · idle/);
const stop = screen.getByRole("button", { name: "Stop session s1" });
fireEvent.click(stop);
// Armed: a plain click must never close an Agent Window by itself.
expect(h.fetches.some((f) => f.url === "/bsk-observation/stop")).toBe(false);
const confirm = await screen.findByRole("button", { name: "Confirm stop session s1" });
expect(confirm.getAttribute("aria-pressed")).toBe("true");
fireEvent.click(confirm);
await waitFor(() =>
expect(
h.fetches.some(
(f) => f.url === "/bsk-observation/stop" && f.init?.body === '{"sessionId":"s1"}',
),
).toBe(true),
);
});
it("disarms the stop confirm after the expiry window", async () => {
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
render(<ObservationOverlay store={h.store} />);
const stop = await screen.findByRole("button", { name: "Stop session s1" });
vi.useFakeTimers();
try {
fireEvent.click(stop);
// Synchronous queries only: waitFor-style finds hang under fake timers.
expect(screen.getByRole("button", { name: "Confirm stop session s1" })).toBeTruthy();
act(() => {
vi.advanceTimersByTime(3100);
});
expect(screen.getByRole("button", { name: "Stop session s1" })).toBeTruthy();
expect(h.fetches.some((f) => f.url === "/bsk-observation/stop")).toBe(false);
} finally {
vi.useRealTimers();
}
});
it("resizes within min/max clamps", async () => {
const h = makeHarness([BUSY]);
render(<ObservationOverlay store={h.store} />);
@@ -261,7 +222,6 @@ describe("ObservationOverlay", () => {
addEventListener: (name: string, fn: () => void) => {
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
},
close: vi.fn(),
} as unknown as Window;
});
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
@@ -276,28 +236,6 @@ describe("ObservationOverlay", () => {
await screen.findByTestId("obs-card");
});
it("closes the PiP window when the carrier unmounts", async () => {
const h = makeHarness([BUSY]);
const pipDoc = document.implementation.createHTMLDocument("pip");
const close = vi.fn();
const requestWindow = vi.fn(async () => {
return {
document: pipDoc,
addEventListener: () => {},
close,
} as unknown as Window;
});
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
const { unmount } = render(<ObservationOverlay store={h.store} />);
const popout = await screen.findByRole("button", { name: /Pop out/ });
fireEvent.click(popout);
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
// Unmounting (tab close, conversation switch, plugin HMR) closes the PiP
// instead of leaving a blank window behind.
unmount();
expect(close).toHaveBeenCalledTimes(1);
});
it("renders the strip for two sessions and pins focus on click", async () => {
const older: SessionObservation = {
sessionId: "s1",
@@ -1,331 +0,0 @@
// @vitest-environment happy-dom
// better-sidebar carrier: tab registration + auto-open, the sidebar-mode
// flag hiding the floating overlay, PiP pop-out from the tab, and the
// store's refcounted acquire/release across overlapping carriers.
import { cleanup, render, screen, waitFor } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ObservationOverlay } from "../../src/client/ObservationOverlay";
import {
type BetterSidebarLike,
OBSERVATION_TAB_PATH,
OBSERVATION_TAB_TYPE,
ObservationSidebarTab,
observationTabOpen,
registerObservationSidebar,
type SidebarNodeLike,
type SidebarStateLike,
type TabDescriptorLike,
} from "../../src/client/observation-sidebar";
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
import { getSidebarMode, setSidebarMode } from "../../src/client/sidebar-mode";
import type { SessionObservation } from "../../src/observation";
// Owned by the sidebar mock's active conversation ("conv-1") so the scoped
// visibility logic lets it through; FOREIGN belongs to another conversation.
const BUSY: SessionObservation = {
sessionId: "s1",
action: "clicking",
since: Date.now() - 7000,
dshSessionIds: ["conv-1"],
};
const FOREIGN: SessionObservation = {
sessionId: "s9",
action: "clicking",
since: Date.now() - 5000,
dshSessionIds: ["other-conv"],
};
interface Harness {
store: ObservationClientStore;
es: () => EventSourceLike | undefined;
push: (sessions: SessionObservation[]) => void;
}
function makeHarness(initial: SessionObservation[]): Harness {
let es: EventSourceLike | undefined;
let current = initial;
const store = new ObservationClientStore({
fetchFn: async (url: string) => {
if (url === "/bsk-observation/state") {
return { ok: true, json: async () => ({ sessions: current }) };
}
return { ok: true, json: async () => ({ interrupted: true }) };
},
eventSourceFactory: () => {
es = { onmessage: null, close: vi.fn() };
return es;
},
loadImage: async (id: string) => `blob:${id}`,
});
return {
store,
es: () => es,
push: (sessions) => {
current = sessions;
for (const s of sessions) {
es?.onmessage?.({ data: JSON.stringify({ type: "upsert", session: s }) });
}
if (sessions.length === 0) es?.onmessage?.({ data: JSON.stringify({ type: "reset" }) });
},
};
}
function leafWith(type: string, panelOpen = true): SidebarStateLike {
return {
panelOpen,
splits: {
kind: "split",
id: "root",
dir: "row",
sizes: [1],
children: [
{
kind: "leaf",
id: "pane-1",
active: type,
tabs: [{ id: type, type, title: type }],
},
],
},
bottomSplits: { kind: "leaf", id: "pane-2", active: null, tabs: [] },
};
}
interface SidebarMock {
service: BetterSidebarLike;
descriptor: () => TabDescriptorLike;
openTab: ReturnType<typeof vi.fn>;
disposeTab: ReturnType<typeof vi.fn>;
setState: (state: SidebarStateLike | undefined) => void;
}
/** Insert a tab into the first leaf of the main tree (mirrors a real open). */
function withTabOpened(state: SidebarStateLike, type: string): SidebarStateLike {
let done = false;
const walk = (node: SidebarNodeLike): SidebarNodeLike => {
if (done) return node;
if (node.kind === "leaf") {
done = true;
return { ...node, active: type, tabs: [...node.tabs, { id: type, type, title: type }] };
}
return { ...node, children: node.children.map(walk) };
};
return { ...state, splits: walk(state.splits) };
}
function makeSidebar(
initialState: SidebarStateLike | undefined = leafWith("explorer"),
): SidebarMock {
let descriptor: TabDescriptorLike | undefined;
let state: SidebarStateLike | undefined = initialState;
const openTab = vi.fn((seed: { type: string }) => {
// Mirror the real feedback loop: a created tab lands in the state, so
// later evaluations see it open (single-instance dedupe re-focuses).
if (state !== undefined && !observationTabOpen(state)) {
state = withTabOpened(state, seed.type);
}
});
const disposeTab = vi.fn();
const service: BetterSidebarLike = {
registerTab: (next) => {
descriptor = next;
return disposeTab;
},
openTab,
getSnapshot: () => ({ sessionId: "conv-1", state }),
isTabEnabled: () => true,
};
return {
service,
descriptor: () => {
if (descriptor === undefined) throw new Error("no tab registered");
return descriptor;
},
openTab,
disposeTab,
setState: (next) => {
state = next;
},
};
}
afterEach(() => {
cleanup();
setSidebarMode(false);
});
describe("registerObservationSidebar", () => {
it("registers a single-instance tab and holds the feed for its lifetime", () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
expect(getSidebarMode()).toBe(false);
const dispose = registerObservationSidebar(sidebar.service, h.store);
expect(getSidebarMode()).toBe(true);
// The fiber holds the observation feed itself, so the auto-open watcher
// sees new sessions even while the tab is closed.
expect(h.es()).toBeDefined();
const descriptor = sidebar.descriptor();
expect(descriptor.id).toBe(OBSERVATION_TAB_TYPE);
expect(descriptor.single).toBe(true);
expect(descriptor.title).toBe("Browser Skill");
dispose();
expect(getSidebarMode()).toBe(false);
expect(sidebar.disposeTab).toHaveBeenCalledTimes(1);
expect(h.es()?.close).toHaveBeenCalled?.();
});
it("opens the tab right away when a session is already live at activation", async () => {
const h = makeHarness([BUSY]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
// The initial state fetch lands asynchronously; the 0→N watcher then fires.
await waitFor(() =>
expect(sidebar.openTab).toHaveBeenCalledWith({
type: OBSERVATION_TAB_TYPE,
path: OBSERVATION_TAB_PATH,
}),
);
});
it("auto-opens on the first session but never re-focuses an open tab", async () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
expect(sidebar.openTab).not.toHaveBeenCalled();
// First session arrives: the tab lands in the sidebar.
h.push([BUSY]);
await waitFor(() => expect(sidebar.openTab).toHaveBeenCalledTimes(1));
// The sidebar now reports the tab open (persisted/restored): further
// session arrivals must not steal focus from whatever the user reads.
sidebar.setState(leafWith(OBSERVATION_TAB_TYPE));
h.push([]);
h.push([{ ...BUSY, sessionId: "s2" }]);
h.push([]);
h.push([{ ...BUSY, sessionId: "s3" }]);
await waitFor(() => expect(h.store.getSnapshot().sessions.length).toBe(1));
expect(sidebar.openTab).toHaveBeenCalledTimes(1);
});
it("does not open while the sidebar has no active session state", () => {
const h = makeHarness([BUSY]);
const sidebar = makeSidebar(undefined);
registerObservationSidebar(sidebar.service, h.store);
expect(sidebar.openTab).not.toHaveBeenCalled();
});
it("nudges an existing tab back into sight when the panel is collapsed", async () => {
const h = makeHarness([]);
// Tab already open (persisted), but the panel is collapsed: a new
// session should surface the tracking view again (focus + expand via a
// content open), like the floating card reappearing.
const sidebar = makeSidebar(leafWith(OBSERVATION_TAB_TYPE, false));
registerObservationSidebar(sidebar.service, h.store);
h.push([BUSY]);
await waitFor(() =>
expect(sidebar.openTab).toHaveBeenCalledWith({
type: OBSERVATION_TAB_TYPE,
path: OBSERVATION_TAB_PATH,
}),
);
});
it("shows the visible session count as the tab badge", () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
const badge = sidebar.descriptor().badge;
const scope = { sessionId: "conv-1" };
expect(badge?.(undefined, scope, undefined)).toBeNull();
h.store.acquire();
h.push([BUSY, FOREIGN]);
expect(badge?.(undefined, scope, undefined)).toBe(1);
expect(badge?.(undefined, { sessionId: "other-conv" }, undefined)).toBe(1);
expect(badge?.(undefined, { sessionId: "nobody" }, undefined)).toBeNull();
h.store.release();
});
it("does not auto-open for sessions owned by other conversations", async () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
h.push([FOREIGN]);
await waitFor(() => expect(h.store.getSnapshot().sessions.length).toBe(1));
expect(sidebar.openTab).not.toHaveBeenCalled();
// A session visible to the active conversation does open the tab.
h.push([BUSY]);
await waitFor(() => expect(sidebar.openTab).toHaveBeenCalledTimes(1));
});
});
describe("observationTabOpen", () => {
it("finds the tab in either workbench tree", () => {
expect(observationTabOpen(undefined)).toBe(false);
expect(observationTabOpen(leafWith("explorer"))).toBe(false);
expect(observationTabOpen(leafWith(OBSERVATION_TAB_TYPE))).toBe(true);
const inBottom: SidebarStateLike = {
splits: { kind: "leaf", id: "p1", active: null, tabs: [] },
bottomSplits: {
kind: "leaf",
id: "p2",
active: OBSERVATION_TAB_TYPE,
tabs: [{ id: OBSERVATION_TAB_TYPE, type: OBSERVATION_TAB_TYPE, title: "Browser" }],
},
};
expect(observationTabOpen(inBottom)).toBe(true);
});
});
describe("ObservationSidebarTab", () => {
it("renders the tracking view without card chrome (no collapse, no drag header)", async () => {
const h = makeHarness([BUSY]);
render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
await screen.findByText(/s1 · clicking/);
expect(screen.queryByRole("button", { name: "Collapse" })).toBeNull();
expect(screen.getByTestId("obs-header").dataset.draggable).toBeUndefined();
// No resize handles — the sidebar owns the geometry.
expect(screen.queryByTestId("obs-resize-se")).toBeNull();
// Session controls ride along: interrupt + stop-with-confirm.
expect(screen.getByRole("button", { name: "Stop session s1" })).toBeTruthy();
});
it("scopes the view to the sidebar's conversation", async () => {
const h = makeHarness([BUSY, FOREIGN]);
render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
// Focus and content belong to the owned session; the foreign one never
// reaches the stage, the status line, or the strip (which needs 2+).
await screen.findByText(/s1 · clicking/);
expect(screen.queryByText(/s9/)).toBeNull();
expect(screen.queryByTestId("obs-strip")).toBeNull();
// Scoping to the other conversation swaps what's visible.
render(<ObservationSidebarTab store={h.store} scopeId="other-conv" />);
await screen.findByText(/s9 · clicking/);
});
});
describe("carrier switching", () => {
it("hides the floating overlay while sidebar mode is active", async () => {
const h = makeHarness([]);
const { container } = render(<ObservationOverlay store={h.store} />);
h.push([BUSY]);
await screen.findByTestId("obs-card");
setSidebarMode(true);
await waitFor(() => expect(container.firstChild).toBeNull());
setSidebarMode(false);
await screen.findByTestId("obs-card");
});
it("keeps the feed alive across overlapping carriers (refcount)", async () => {
const h = makeHarness([]);
const overlay = render(<ObservationOverlay store={h.store} />);
await waitFor(() => expect(h.es()).toBeDefined());
// A second carrier mounts (the sidebar tab), then the overlay unmounts:
// the stream must survive for the remaining consumer.
const tab = render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
overlay.unmount();
h.push([BUSY]);
await screen.findByText(/s1 · clicking/);
expect(h.es()?.close ?? vi.fn()).not.toHaveBeenCalled();
tab.unmount();
});
});
@@ -137,9 +137,6 @@ describe("client plugin registration", () => {
const sessions = { binding: () => undefined };
const ctx = {
get: (key: string) => (key === "sessions" ? sessions : undefined),
// cordis ctx.inject: the betterSidebar carrier upgrade stays dormant in
// this composition (the callback only runs once the service exists).
inject: (_deps: string[], _fn: (injected: unknown) => unknown) => {},
slots: {
inject: (_name: string, fn: () => unknown) => fn(),
register: (slot: { name: string; key?: string; id?: string }, view: unknown) => {
@@ -61,8 +61,6 @@ function fakeRunner(
opts: {
screenshotFails?: boolean;
screenshotNotFound?: boolean;
stopNotFound?: boolean;
stopFails?: boolean;
killCount?: number;
screenshotBytes?: () => Uint8Array;
} = {},
@@ -74,21 +72,6 @@ function fakeRunner(
killed,
async run(args: string[], options: BskRunOptions = {}): Promise<BskRunResult> {
calls.push({ args, options });
if (args[0] === "session" && args[1] === "stop") {
if (opts.stopNotFound) {
return {
code: 4,
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
stderr: "",
timedOut: false,
aborted: false,
};
}
if (opts.stopFails) {
return { code: 1, stdout: "", stderr: "boom", timedOut: false, aborted: false };
}
return { code: 0, stdout: "{}", stderr: "", timedOut: false, aborted: false };
}
if (args[0] === "screenshot") {
if (opts.screenshotNotFound) {
return {
@@ -197,23 +180,6 @@ describe("state machine", () => {
expect(events[events.length - 1].type).toBe("reset");
});
it("stamps the registry-recorded DSH owners onto new entries", () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
own(registry, "s2");
registry.trackOwner("s1", ["conv-a", "root"]);
const { service } = setup({ registry });
service.addSession("s1");
service.addSession("s2");
expect(service.getState()).toEqual([
{ sessionId: "s1", action: "idle", since: 1_000_000, dshSessionIds: ["conv-a", "root"] },
{ sessionId: "s2", action: "idle", since: 1_000_000 },
]);
// The owner stamp survives later upserts (action/url/frame churn).
service.beginAction("s1", "clicking");
expect(service.getState()[0].dshSessionIds).toEqual(["conv-a", "root"]);
});
it("ignores instrumentation for unknown sessions (owned-only by construction)", () => {
const { service } = setup({});
service.beginAction("foreign", "clicking");
@@ -341,57 +307,6 @@ describe("interrupt routing", () => {
});
});
describe("stopSession", () => {
it("stops an owned session: kills in-flight tools, runs session stop, removes the entry", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner();
const { service, events } = setup({ registry, runner });
service.addSession("s1");
expect(registry.isOwned("s1")).toBe(true);
await expect(service.stopSession("s1")).resolves.toBe(true);
expect(runner.killed).toEqual(["s1"]);
const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
expect(stop?.args).toEqual(["session", "stop", "s1"]);
expect(registry.isOwned("s1")).toBe(false);
expect(service.getState()).toEqual([]);
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
});
it("refuses foreign sessions without touching the runner", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner();
const { service } = setup({ registry, runner });
await expect(service.stopSession("foreign")).resolves.toBe(false);
expect(runner.killed).toEqual([]);
expect(runner.calls).toEqual([]);
});
it("stops idempotently when the daemon already forgot the session (dead entries)", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner({ stopNotFound: true });
const { service, events } = setup({ registry, runner });
service.addSession("s1");
await expect(service.stopSession("s1")).resolves.toBe(true);
expect(registry.isOwned("s1")).toBe(false);
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
});
it("returns false and keeps the entry when the stop itself fails", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner({ stopFails: true });
const { service } = setup({ registry, runner });
service.addSession("s1");
await expect(service.stopSession("s1")).resolves.toBe(false);
expect(registry.isOwned("s1")).toBe(true);
expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
});
});
describe("HTTP/SSE interface", () => {
interface RecordedRoute {
path: string;
@@ -490,45 +405,11 @@ describe("HTTP/SSE interface", () => {
await interruptRoute?.handler(postReq, interruptRes.res as never);
expect(JSON.parse(interruptRes.res.body())).toEqual({ interrupted: true });
// stop (POST {sessionId}): owned session stops and leaves the state
const stopRoute = routes.get("/bsk-observation/stop");
const stopRes = fakeRes();
await stopRoute?.handler(postReq, stopRes.res as never);
// The stop is async behind the response-less handler — poll the state.
await waitFor(() => service.getState().length === 0);
expect(JSON.parse(stopRes.res.body())).toEqual({ stopped: true });
dispose();
expect(routes.size).toBe(0);
service.dispose();
});
it("refuses a stop without a sessionId", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const { service } = setup({ registry });
const { routes, webServer } = routeHarness();
const ctx = { get: (key: string) => (key === "webServer" ? webServer : undefined) } as never;
const dispose = registerObservationRoutes(ctx, service);
const stopRoute = routes.get("/bsk-observation/stop");
const res = fakeRes();
const emptyReq = fakeReq({
method: "POST",
headers: { host: "127.0.0.1:3999", "content-type": "application/json" },
on: (event: string, fn: (chunk?: string) => void) => {
if (event === "data") fn("{}");
if (event === "end") fn();
},
});
await stopRoute?.handler(emptyReq, res.res as never);
expect(res.res.status).toBe(400);
expect(registry.isOwned("s1")).toBe(true);
dispose();
service.dispose();
});
it("fences off non-loopback, cross-site, and simple-request traffic", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
+60
View File
@@ -29,6 +29,66 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
}
describe("renderVom single-layer page", () => {
it("renders visual surfaces as screenshot-only refs", () => {
const input = scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
]);
input.visualSurfaces = [
{
parentId: 2,
backendNodeId: 99,
frameId: "child",
renderingKind: "canvas",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
label: "Data grid",
memberCount: 2,
},
];
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
ref: "e1",
backendNodeId: 99,
kind: "surface",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
capabilities: ["screenshot"],
}),
);
});
it("renders an unnamed visual surface with a stable generic label", () => {
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
input.visualSurfaces = [
{
parentId: 1,
backendNodeId: 99,
frameId: "main",
renderingKind: "canvas",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
memberCount: 1,
},
];
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
ref: "e1",
name: "canvas visual surface",
kind: "surface",
capabilities: ["screenshot"],
}),
);
});
it("does not derive handle context across frame scopes", () => {
const out = renderVom(
scene([
+2
View File
@@ -15,6 +15,8 @@ export type {
VomNode,
VomOptions,
VomRef,
VomRefCapability,
VomResult,
VomScene,
VomVisualSurface,
} from "./types";
+65 -1
View File
@@ -9,6 +9,7 @@ import type {
VomOptions,
VomResult,
VomScene,
VomVisualSurface,
} from "./types";
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -737,6 +738,53 @@ interface RenderState {
duplicateRefNames: Set<string>;
surfaceMap: Map<number, CondSurface>;
scopeMap: Map<number, ActiveScopeBlock>;
visualSurfaces: Map<number | null, VomVisualSurface[]>;
}
function groupedByParent<T extends { parentId: number | null }>(
items: T[],
): Map<number | null, T[]> {
const grouped = new Map<number | null, T[]>();
for (const item of items) {
const siblings = grouped.get(item.parentId) ?? [];
siblings.push(item);
grouped.set(item.parentId, siblings);
}
return grouped;
}
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
if (state.stopped) return;
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
if (depth > state.maxDepth) {
state.truncated = true;
continue;
}
const ref = `e${state.nextRef}`;
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; use: bsk screenshot --ref @${ref}]`;
const nextTokens = state.tokens + estimateTokens(line);
if (nextTokens > state.maxTokens) {
state.truncated = true;
state.stopped = true;
return;
}
state.lines.push(line);
state.tokens = nextTokens;
state.refs.push({
ref,
backendNodeId: surface.backendNodeId,
frameId: surface.frameId,
visibleRect: surface.visibleRect,
role: "surface",
name: label,
kind: "surface",
capabilities: ["screenshot"],
line: state.lines.length - 1,
});
state.nextRef += 1;
}
}
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
@@ -812,6 +860,8 @@ function renderTree(
...(node.role ? { role: node.role } : {}),
...(name ? { name } : {}),
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
kind: "dom",
capabilities: ["interact", "screenshot"],
line: state.lines.length - 1,
});
state.nextRef += 1;
@@ -821,10 +871,12 @@ function renderTree(
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
emitVisualEntries(node.id, depth + 1, state);
continue;
}
renderTree(children, node.id, depth + 1, state);
}
emitVisualEntries(parentId, depth, state);
}
function renderNodes(
@@ -833,6 +885,7 @@ function renderNodes(
initialLines: string[],
surfaces: CondSurface[] = [],
activeScopeBlocks: ActiveScopeBlock[] = [],
visualSurfaces: VomVisualSurface[] = [],
): RenderState {
const children = buildChildren(nodes);
const state: RenderState = {
@@ -851,6 +904,7 @@ function renderNodes(
duplicateRefNames: duplicateReferenceNames(nodes),
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
visualSurfaces: groupedByParent(visualSurfaces),
};
renderTree(children, null, 1, state);
@@ -1056,7 +1110,16 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
"@layers 2 focus=L1",
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
];
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
const visibleSurface = (parentId: number | null): boolean =>
parentId === null || included.has(parentId);
const state = renderNodes(
visibleNodes,
options,
header,
scene.surfaces,
scene.activeScopeBlocks,
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
);
state.lines.push(renderPageOcclusionLine(hiddenCount));
return {
@@ -1084,6 +1147,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
],
scene.surfaces,
scene.activeScopeBlocks,
scene.visualSurfaces,
);
return {
+18
View File
@@ -61,6 +61,8 @@ export interface VomScene {
rootFrameId?: string;
surfaces?: CondSurface[];
activeScopeBlocks?: ActiveScopeBlock[];
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
visualSurfaces?: VomVisualSurface[];
}
export interface VomOptions {
@@ -94,6 +96,22 @@ export interface VomRef {
ref: string;
backendNodeId: number;
frameId?: string;
/** Observation-time top-viewport crop for screenshot-only surfaces. */
visibleRect?: Rect;
kind?: "dom" | "surface";
capabilities?: VomRefCapability[];
}
export type VomRefCapability = "interact" | "screenshot";
export interface VomVisualSurface {
parentId: number | null;
backendNodeId: number;
frameId: string;
renderingKind: "canvas";
visibleRect: Rect;
label?: string;
memberCount: number;
}
export interface RenderedRef extends VomRef {
+2
View File
@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session <id>` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
## Observation priority
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: