mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eeba301080 | ||
|
|
5ca158a729 | ||
|
|
ed91f41291 | ||
|
|
b9fd6e46af | ||
|
|
36135c1a2d | ||
|
|
28d8a5f605 | ||
|
|
54ce2f8dc3 | ||
|
|
b7e62d8cf1 |
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.1.11"
|
||||
version = "0.1.10"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bsk-protocol",
|
||||
@@ -168,7 +168,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.1.11"
|
||||
version = "0.1.10"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.11"
|
||||
version = "0.1.10"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.1.7",
|
||||
"version": "0.1.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -13,6 +13,8 @@ describe("RefStore", () => {
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
generation: 0,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
expect(s.size()).toBe(1);
|
||||
expect(s.isEmpty()).toBe(false);
|
||||
@@ -48,4 +50,31 @@ describe("RefStore", () => {
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit capabilities for visual surface refs", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
});
|
||||
|
||||
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, { tabId: 7, kind: "surface" });
|
||||
s.set("e2", 43, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
||||
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
@@ -12,12 +14,17 @@
|
||||
* inside the `SessionContext`.
|
||||
*/
|
||||
export type BackendNodeId = number;
|
||||
export type RefCapability = "interact" | "screenshot";
|
||||
export type RefTargetKind = "dom" | "surface";
|
||||
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
@@ -28,8 +35,19 @@ export type RefInput =
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
};
|
||||
|
||||
function capabilitiesFor(
|
||||
kind: RefTargetKind,
|
||||
capabilities: RefCapability[] | undefined,
|
||||
): RefCapability[] {
|
||||
if (kind === "surface") return ["screenshot"];
|
||||
return capabilities ?? ["interact", "screenshot"];
|
||||
}
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
private generation = 0;
|
||||
@@ -70,13 +88,20 @@ export class RefStore {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
} = {},
|
||||
): void {
|
||||
const kind = opts.kind ?? "dom";
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, opts.capabilities),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -94,14 +119,20 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input,
|
||||
tabId: null,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
const kind = input.kind ?? "dom";
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, input.capabilities),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -123,6 +123,29 @@ describe("handleClick", () => {
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||
});
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -300,6 +300,76 @@ describe("handleScreenshot", () => {
|
||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
||||
});
|
||||
|
||||
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { width: number; height: number; scale: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
|
||||
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
|
||||
});
|
||||
|
||||
it("crops a visual surface screenshot to its observation-time visible region", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -2693,6 +2763,85 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Canvas app" },
|
||||
backendDOMNodeId: 100,
|
||||
};
|
||||
const strings = ["body", "canvas", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("canvas")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 30, 800, 500],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
conditionalSurfaceProbe: false,
|
||||
};
|
||||
|
||||
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
|
||||
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
|
||||
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 200,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
|
||||
expect(snapshot.text).not.toContain("visual surface");
|
||||
expect(snapshot.ref_count).toBe(0);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
|
||||
@@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => {
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,6 +57,8 @@ describe("resolveSnapshotRef", () => {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
const expected = {
|
||||
@@ -60,6 +66,8 @@ describe("resolveSnapshotRef", () => {
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom" as const,
|
||||
capabilities: ["interact", "screenshot"] as const,
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
@@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => {
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an operation outside the ref's declared capabilities", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
|
||||
code: "permission_denied",
|
||||
message: "ref @e3 does not support interact",
|
||||
data: {
|
||||
reason: "ref_capability_denied",
|
||||
required_capability: "interact",
|
||||
kind: "surface",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -159,7 +159,7 @@ async function resolveBackendNode(
|
||||
};
|
||||
}
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
type Rect,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
@@ -59,6 +60,11 @@ import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "./vom/rendered-surfaces";
|
||||
import {
|
||||
buildSemanticGraph,
|
||||
buildSemanticVomScene,
|
||||
@@ -90,6 +96,9 @@ export const normaliseRef = sharedNormaliseRef;
|
||||
// screenshot — `tool.screenshot`
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
|
||||
|
||||
/**
|
||||
* Strip the `data:image/...;base64,` prefix from a Chrome
|
||||
* `captureVisibleTab` dataURL and return the raw base64 payload.
|
||||
@@ -182,6 +191,8 @@ async function captureElementScreenshot(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
visualSurface = false,
|
||||
observedVisibleRect?: Rect,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
@@ -189,11 +200,36 @@ async function captureElementScreenshot(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: true },
|
||||
{ scrollIntoView: !visualSurface },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
const liveRect = geometry.topBounds;
|
||||
const rect =
|
||||
visualSurface && observedVisibleRect
|
||||
? (() => {
|
||||
const x = Math.max(liveRect.x, observedVisibleRect.x);
|
||||
const y = Math.max(liveRect.y, observedVisibleRect.y);
|
||||
const right = Math.min(
|
||||
liveRect.x + liveRect.width,
|
||||
observedVisibleRect.x + observedVisibleRect.w,
|
||||
);
|
||||
const bottom = Math.min(
|
||||
liveRect.y + liveRect.height,
|
||||
observedVisibleRect.y + observedVisibleRect.h,
|
||||
);
|
||||
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
|
||||
})()
|
||||
: liveRect;
|
||||
if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
|
||||
const scale = visualSurface
|
||||
? Math.min(
|
||||
1,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
|
||||
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
|
||||
)
|
||||
: 1;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
@@ -203,7 +239,7 @@ async function captureElementScreenshot(
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale: 1,
|
||||
scale,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
@@ -310,7 +346,7 @@ export async function handleScreenshot(
|
||||
if (!deps.cdp) {
|
||||
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -330,6 +366,8 @@ export async function handleScreenshot(
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
node.kind === "surface",
|
||||
node.visibleRect,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
@@ -1023,6 +1061,7 @@ async function captureForVom(
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
renderedSurfaceDiscovery?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
@@ -1064,7 +1103,20 @@ export async function captureVomObservation(
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
const observedScene = options.renderedSurfaceDiscovery
|
||||
? projectRenderedSurfaces(
|
||||
decoratedScene,
|
||||
normalizedDocuments,
|
||||
clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
normalizedDocuments,
|
||||
captured.viewport,
|
||||
captured.excludedBackendNodeIds,
|
||||
),
|
||||
),
|
||||
)
|
||||
: decoratedScene;
|
||||
const rendered = renderVom(observedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
@@ -1115,6 +1167,7 @@ async function handleVomObservation(
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
renderedSurfaceDiscovery: toolName === "observe",
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
@@ -1132,6 +1185,9 @@ async function handleVomObservation(
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
|
||||
...(ref.kind ? { kind: ref.kind } : {}),
|
||||
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
|
||||
@@ -12,6 +13,9 @@ export interface SnapshotRefLookup {
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
@@ -37,6 +41,9 @@ export function lookupSnapshotRef(
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
|
||||
kind: entry.kind,
|
||||
capabilities: entry.capabilities,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -49,6 +56,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
requiredCapability?: RefCapability,
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
@@ -58,5 +66,13 @@ export function resolveSnapshotRef(
|
||||
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
|
||||
);
|
||||
}
|
||||
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"ref_capability_denied",
|
||||
`ref ${ref} does not support ${requiredCapability}`,
|
||||
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
|
||||
);
|
||||
}
|
||||
return looked;
|
||||
}
|
||||
|
||||
@@ -722,6 +722,44 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||
});
|
||||
|
||||
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
|
||||
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
|
||||
const i = (value: string) => S.indexOf(value);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 100],
|
||||
[10, 10, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1, 2],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const canvas = nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(canvas?.rendered).toBe(true);
|
||||
expect(canvas?.visuallySuppressed).toBe(true);
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import type { VomScene } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "../rendered-surfaces";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
|
||||
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "canvas",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
localRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
rendered: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function document(
|
||||
domNodes: CapturedNode[],
|
||||
axNodes: SemanticAxNode[] = [],
|
||||
): CapturedFrameDocument<SemanticAxNode> {
|
||||
return {
|
||||
frameId: domNodes[0]?.frameId ?? "main",
|
||||
contextScopeId: domNodes[0]?.frameId ?? "main",
|
||||
target: { tabId: 4 },
|
||||
domNodes,
|
||||
axNodes,
|
||||
};
|
||||
}
|
||||
|
||||
describe("rendered surface discovery", () => {
|
||||
it("finds prominent rendered canvases and excludes hidden descendants", () => {
|
||||
const visible = domNode(2, { parentBackendNodeId: 1 });
|
||||
const hidden = domNode(3, { parentBackendNodeId: 4 });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
visible,
|
||||
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
|
||||
hidden,
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
|
||||
expect(surfaces).toHaveLength(1);
|
||||
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
|
||||
});
|
||||
|
||||
it("keeps tiny canvases as individually addressable exact discoveries", () => {
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("keeps a surface when the canvas only has partial native accessibility content", () => {
|
||||
const axNodes = [
|
||||
{
|
||||
nodeId: "canvas",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "presentation" },
|
||||
childIds: ["button"],
|
||||
frameId: "main",
|
||||
},
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
frameId: "main",
|
||||
},
|
||||
] as SemanticAxNode[];
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
|
||||
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
|
||||
const canvas = domNode(2, {
|
||||
attrs: {
|
||||
id: "must-not-be-a-label",
|
||||
class: "must-not-be-a-label",
|
||||
"data-testid": "must-not-be-a-label",
|
||||
},
|
||||
});
|
||||
|
||||
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
|
||||
expect.not.objectContaining({
|
||||
label: expect.anything(),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats blank explicit Canvas names as unnamed", () => {
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
|
||||
});
|
||||
|
||||
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
|
||||
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([owner, canvas])],
|
||||
{ width: 1280, height: 720 },
|
||||
new Set([1]),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("excludes canvases suppressed by a fully transparent ancestor", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 900, h: 600 },
|
||||
visuallySuppressed: true,
|
||||
});
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 10, y: 20, w: 100, h: 70 },
|
||||
overflowX: "hidden",
|
||||
overflowY: "hidden",
|
||||
});
|
||||
const partial = domNode(2, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 55, y: 30, w: 140, h: 60 },
|
||||
});
|
||||
const fullyClipped = domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 140, y: 30, w: 80, h: 50 },
|
||||
});
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
|
||||
width: 1280,
|
||||
height: 720,
|
||||
}),
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 2,
|
||||
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rendered surface clustering and projection", () => {
|
||||
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
|
||||
const body = domNode(1, { tag: "body", rect: null, localRect: null });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
body,
|
||||
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
|
||||
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
const groups = clusterRenderedSurfaces(surfaces);
|
||||
const scene: VomScene = {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [
|
||||
{
|
||||
id: 10,
|
||||
parentId: null,
|
||||
backendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
role: "RootWebArea",
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]?.members).toHaveLength(2);
|
||||
expect(groups[0]?.label).toBeUndefined();
|
||||
expect(
|
||||
projectRenderedSurfaces(
|
||||
scene,
|
||||
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
|
||||
groups,
|
||||
),
|
||||
).toMatchObject({
|
||||
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge a contained canvas with a larger independent canvas", () => {
|
||||
const groups = clusterRenderedSurfaces([
|
||||
...discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
domNode(2, { parentBackendNodeId: 1 }),
|
||||
domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 30, y: 40, w: 200, h: 100 },
|
||||
}),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
]);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not merge coincident canvases owned by different containers", () => {
|
||||
const groups = clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(2, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(3, { parentBackendNodeId: 1 }),
|
||||
domNode(4, { parentBackendNodeId: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
@@ -39,6 +47,10 @@ export interface CapturedNode {
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
overflowX?: string;
|
||||
overflowY?: string;
|
||||
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
|
||||
visuallySuppressed?: boolean;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
@@ -796,6 +808,8 @@ function parseDocumentNodes(
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const overflowXCol = STYLE_COL["overflow-x"];
|
||||
const overflowYCol = STYLE_COL["overflow-y"];
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
@@ -821,6 +835,7 @@ function parseDocumentNodes(
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
@@ -840,7 +855,11 @@ function parseDocumentNodes(
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let overflowX = "visible";
|
||||
let overflowY = "visible";
|
||||
let rendered = false;
|
||||
let visuallySuppressed =
|
||||
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
@@ -863,12 +882,19 @@ function parseDocumentNodes(
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
|
||||
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
|
||||
visuallySuppressed ||=
|
||||
visibility === "hidden" ||
|
||||
visibility === "collapse" ||
|
||||
(Number.parseFloat(opacity) || 0) <= 0;
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
@@ -903,6 +929,9 @@ function parseDocumentNodes(
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
overflowX,
|
||||
overflowY,
|
||||
visuallySuppressed,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
|
||||
const STACK_IOU_RATIO = 0.9;
|
||||
const STACK_SIZE_RATIO = 0.9;
|
||||
|
||||
function area(rect: Rect): number {
|
||||
return Math.max(0, rect.w) * Math.max(0, rect.h);
|
||||
}
|
||||
|
||||
function intersectionArea(a: Rect, b: Rect): number {
|
||||
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
|
||||
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
|
||||
return width * height;
|
||||
}
|
||||
|
||||
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
||||
if (a.frameId !== b.frameId) return false;
|
||||
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
|
||||
const aArea = area(a.visibleRect);
|
||||
const bArea = area(b.visibleRect);
|
||||
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
|
||||
const union = aArea + bArea - intersection;
|
||||
const widthRatio =
|
||||
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
|
||||
const heightRatio =
|
||||
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
|
||||
return (
|
||||
union > 0 &&
|
||||
intersection / union >= STACK_IOU_RATIO &&
|
||||
widthRatio >= STACK_SIZE_RATIO &&
|
||||
heightRatio >= STACK_SIZE_RATIO
|
||||
);
|
||||
}
|
||||
|
||||
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
|
||||
const representative = [...members].sort(
|
||||
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
|
||||
)[0];
|
||||
return {
|
||||
frameId: representative.frameId,
|
||||
members,
|
||||
representative,
|
||||
label: members.map((surface) => surface.label).find((label) => label !== undefined),
|
||||
};
|
||||
}
|
||||
|
||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
|
||||
const groups: RenderedSurface[][] = [];
|
||||
for (const surface of surfaces) {
|
||||
const matching = groups.find((group) =>
|
||||
group.some((member) => sameVisualStack(member, surface)),
|
||||
);
|
||||
if (matching) matching.push(surface);
|
||||
else groups.push([surface]);
|
||||
}
|
||||
return groups
|
||||
.map(toGroup)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurface } from "./types";
|
||||
|
||||
function clean(value: string | undefined): string | undefined {
|
||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
|
||||
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
function intersectRect(a: Rect, b: Rect): Rect | null {
|
||||
const x = Math.max(a.x, b.x);
|
||||
const y = Math.max(a.y, b.y);
|
||||
const right = Math.min(a.x + a.w, b.x + b.w);
|
||||
const bottom = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, w: right - x, h: bottom - y };
|
||||
}
|
||||
|
||||
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
|
||||
|
||||
function clipToOverflowAncestors(
|
||||
node: CapturedNode,
|
||||
rect: Rect,
|
||||
nodesByBackend: ReadonlyMap<number, CapturedNode>,
|
||||
): Rect | null {
|
||||
let clipped: Rect | null = rect;
|
||||
let parentBackendNodeId = node.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
|
||||
const parent = nodesByBackend.get(parentBackendNodeId);
|
||||
if (!parent) break;
|
||||
if (parent.rect) {
|
||||
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
|
||||
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
|
||||
if (clipX || clipY) {
|
||||
const clipRect = {
|
||||
x: clipX ? parent.rect.x : clipped.x,
|
||||
y: clipY ? parent.rect.y : clipped.y,
|
||||
w: clipX ? parent.rect.w : clipped.w,
|
||||
h: clipY ? parent.rect.h : clipped.h,
|
||||
};
|
||||
clipped = intersectRect(clipped, clipRect);
|
||||
}
|
||||
}
|
||||
parentBackendNodeId = parent.parentBackendNodeId;
|
||||
guard += 1;
|
||||
}
|
||||
return clipped;
|
||||
}
|
||||
|
||||
function excludedByDomPolicy(
|
||||
node: CapturedNode,
|
||||
nodesByBackend: Map<number, CapturedNode>,
|
||||
excludedBackendNodeIds: ReadonlySet<number>,
|
||||
): boolean {
|
||||
let current: CapturedNode | undefined = node;
|
||||
let guard = 0;
|
||||
while (current && guard <= nodesByBackend.size) {
|
||||
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
|
||||
if (current.visuallySuppressed === true) return true;
|
||||
const attrs = current.attrs;
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
||||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
||||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
current =
|
||||
current.parentBackendNodeId === null
|
||||
? undefined
|
||||
: nodesByBackend.get(current.parentBackendNodeId);
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function discoverRenderedSurfaces(
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
viewport: Viewport,
|
||||
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
|
||||
): RenderedSurface[] {
|
||||
const surfaces: RenderedSurface[] = [];
|
||||
for (const document of documents) {
|
||||
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
|
||||
for (const node of document.domNodes) {
|
||||
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
|
||||
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
|
||||
const viewportRect = intersectViewport(node.rect, viewport);
|
||||
const visibleRect = viewportRect
|
||||
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
|
||||
: null;
|
||||
if (!visibleRect) continue;
|
||||
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
|
||||
surfaces.push({
|
||||
renderingKind: "canvas",
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
parentBackendNodeId: node.parentBackendNodeId,
|
||||
visibleRect,
|
||||
paintOrder: node.paintOrder,
|
||||
...(label ? { label } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return surfaces;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export { clusterRenderedSurfaces } from "./cluster";
|
||||
export { discoverRenderedSurfaces } from "./discovery";
|
||||
export { projectRenderedSurfaces } from "./project";
|
||||
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurfaceGroup } from "./types";
|
||||
|
||||
function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
interface ProjectionIndex {
|
||||
sceneByBackend: ReadonlyMap<string, number>;
|
||||
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
|
||||
domByFrame: ReadonlyMap<
|
||||
string,
|
||||
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
|
||||
>;
|
||||
rootSceneIdByFrame: ReadonlyMap<string, number>;
|
||||
}
|
||||
|
||||
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
|
||||
const document = index.documentByFrame.get(group.frameId);
|
||||
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
|
||||
let parentBackendNodeId = group.representative.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
|
||||
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
|
||||
if (sceneId !== undefined) return sceneId;
|
||||
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
|
||||
guard += 1;
|
||||
}
|
||||
|
||||
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
|
||||
const owner = index.sceneByBackend.get(
|
||||
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
|
||||
);
|
||||
if (owner !== undefined) return owner;
|
||||
}
|
||||
|
||||
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
|
||||
}
|
||||
|
||||
function buildProjectionIndex(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
): ProjectionIndex {
|
||||
const rootSceneIdByFrame = new Map<string, number>();
|
||||
for (const node of scene.nodes) {
|
||||
if (
|
||||
node.frameId &&
|
||||
!rootSceneIdByFrame.has(node.frameId) &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
|
||||
) {
|
||||
rootSceneIdByFrame.set(node.frameId, node.id);
|
||||
}
|
||||
}
|
||||
return {
|
||||
sceneByBackend: new Map(
|
||||
scene.nodes.flatMap((node) =>
|
||||
node.backendNodeId === undefined
|
||||
? []
|
||||
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
|
||||
),
|
||||
),
|
||||
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
|
||||
domByFrame: new Map(
|
||||
documents.map((document) => [
|
||||
document.frameId,
|
||||
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
|
||||
]),
|
||||
),
|
||||
rootSceneIdByFrame,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectRenderedSurfaces(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
groups: RenderedSurfaceGroup[],
|
||||
): VomScene {
|
||||
const index = buildProjectionIndex(scene, documents);
|
||||
const visualSurfaces: VomVisualSurface[] = [];
|
||||
|
||||
for (const group of groups) {
|
||||
const parentId = nearestSceneParent(index, group);
|
||||
visualSurfaces.push({
|
||||
parentId,
|
||||
backendNodeId: group.representative.backendNodeId,
|
||||
frameId: group.frameId,
|
||||
renderingKind: "canvas",
|
||||
visibleRect: group.representative.visibleRect,
|
||||
...(group.label ? { label: group.label } : {}),
|
||||
memberCount: group.members.length,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...scene,
|
||||
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface RenderedSurface {
|
||||
renderingKind: "canvas";
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
visibleRect: Rect;
|
||||
paintOrder: number;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface RenderedSurfaceGroup {
|
||||
frameId: string;
|
||||
members: RenderedSurface[];
|
||||
representative: RenderedSurface;
|
||||
label?: string;
|
||||
}
|
||||
@@ -23,6 +23,7 @@ export type ErrorCode =
|
||||
export type RpcErrorReason =
|
||||
| "agent_window_scope"
|
||||
| "element_not_visible"
|
||||
| "ref_capability_denied"
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# Rendered Surface browser fixtures
|
||||
|
||||
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
|
||||
|
||||
## Serve
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
python3 -m http.server 4173 --bind 127.0.0.1 \
|
||||
--directory apps/extension/test-fixtures/rendered-surfaces
|
||||
```
|
||||
|
||||
Use an Agent Window large enough to keep the primary matrix visible:
|
||||
|
||||
```bash
|
||||
bsk session start --width 1440 --height 1100
|
||||
bsk navigate http://127.0.0.1:4173/index.html --session <id>
|
||||
bsk observe --session <id>
|
||||
```
|
||||
|
||||
Always stop the session after the run.
|
||||
|
||||
## Pages
|
||||
|
||||
### `index.html`
|
||||
|
||||
Expected individual Surface labels:
|
||||
|
||||
- `fixture-basic-large`
|
||||
- `fixture-tiny-8px`
|
||||
- `fixture-pointer-events-none`
|
||||
- `fixture-partial-ax`
|
||||
- `fixture-contained-outer`
|
||||
- `fixture-contained-inner`
|
||||
- `fixture-different-parent-a`
|
||||
- `fixture-different-parent-b`
|
||||
|
||||
Expected one grouped Surface:
|
||||
|
||||
- `fixture-layered-group`, with `layers=2`
|
||||
|
||||
Expected absent:
|
||||
|
||||
- `fixture-display-none`
|
||||
- `fixture-visibility-hidden`
|
||||
- `fixture-opacity-zero`
|
||||
- `fixture-parent-opacity-zero`
|
||||
- `fixture-hidden-attribute`
|
||||
- `fixture-inert-parent`
|
||||
- `fixture-aria-hidden-parent`
|
||||
- `fixture-zero-size`
|
||||
- `fixture-offscreen-right`
|
||||
- `fixture-fully-overflow-clipped`
|
||||
|
||||
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
|
||||
|
||||
### `unlabeled.html`
|
||||
|
||||
This page contains eight Canvas elements but must render seven generic Surface markers because
|
||||
the two coincident layers form one group.
|
||||
|
||||
- Every marker uses the stable fallback label `canvas visual surface`.
|
||||
- Exactly one marker has `layers=2`.
|
||||
- The two contained canvases remain separate markers.
|
||||
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
|
||||
- Blank `aria-label` and `title` values behave as no label.
|
||||
- Fallback AX content does not suppress the corresponding Surface.
|
||||
|
||||
### `frames.html`
|
||||
|
||||
Expected Surface labels:
|
||||
|
||||
- `fixture-same-origin-frame`
|
||||
- `fixture-cross-origin-frame`
|
||||
- `fixture-nested-frame`
|
||||
|
||||
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
|
||||
|
||||
### `viewport.html`
|
||||
|
||||
Run with a `900x700` Agent Window.
|
||||
|
||||
- `fixture-partial-viewport` is partly visible and must be represented.
|
||||
- `fixture-below-fold` is absent initially.
|
||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
||||
|
||||
### `dynamic.html`
|
||||
|
||||
Use the normal DOM buttons and re-observe after each action:
|
||||
|
||||
- Initial state: `fixture-dynamic` absent.
|
||||
- After **Insert canvas**: present.
|
||||
- After **Resize to zero**: absent.
|
||||
- After **Restore size**: present with a fresh ref.
|
||||
- After **Remove canvas**: absent.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `snapshot` must not emit any of these Surface markers.
|
||||
- `observe` must emit only currently visible Canvas surfaces.
|
||||
- Existing DOM/AX controls remain readable alongside Surface markers.
|
||||
- Missing labels never prevent discovery; naming and discovery are independent.
|
||||
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
|
||||
- No fixture URL, id, label, or layout constant may appear in production discovery code.
|
||||
@@ -0,0 +1,46 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface dynamic matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
|
||||
button { padding: 8px 12px; }
|
||||
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Dynamic Canvas lifecycle</h1>
|
||||
<div class="controls">
|
||||
<button id="insert" type="button">Insert canvas</button>
|
||||
<button id="zero" type="button">Resize to zero</button>
|
||||
<button id="restore" type="button">Restore size</button>
|
||||
<button id="remove" type="button">Remove canvas</button>
|
||||
</div>
|
||||
<div id="mount"></div>
|
||||
<script>
|
||||
const mount = document.querySelector("#mount");
|
||||
const getCanvas = () => mount.querySelector("canvas");
|
||||
document.querySelector("#insert").addEventListener("click", () => {
|
||||
if (getCanvas()) return;
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 260;
|
||||
canvas.height = 120;
|
||||
canvas.setAttribute("aria-label", "fixture-dynamic");
|
||||
mount.append(canvas);
|
||||
});
|
||||
document.querySelector("#zero").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
|
||||
});
|
||||
document.querySelector("#restore").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.removeAttribute("style");
|
||||
});
|
||||
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,28 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 12px; background: #fdfefe; }
|
||||
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
|
||||
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="frame-canvas" width="220" height="90"></canvas>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const canvas = document.querySelector("#frame-canvas");
|
||||
if (params.get("canvas") === "0") canvas.remove();
|
||||
else canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested fixture frame";
|
||||
iframe.src = "frame.html?label=fixture-nested-frame";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,22 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface frame matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame matrix</h1>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame"></iframe>
|
||||
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame"></iframe>
|
||||
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,118 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface discovery matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
.same-place { position: relative; width: 180px; height: 90px; }
|
||||
.same-place > div { position: absolute; inset: 0; }
|
||||
.same-place canvas { width: 180px; height: 90px; }
|
||||
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
|
||||
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
|
||||
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
|
||||
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
|
||||
.tiny { width: 8px; height: 8px; border-width: 1px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Rendered Surface discovery matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Visible named canvas</h2>
|
||||
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Tiny canvas remains addressable</h2>
|
||||
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Visible but pointer-events none</h2>
|
||||
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partial AX/fallback content</h2>
|
||||
<canvas aria-label="fixture-partial-ax" width="180" height="80">
|
||||
<button type="button">Fallback button must not suppress the surface</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Two real rendering layers</h2>
|
||||
<div class="stack">
|
||||
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Contained canvases are independent</h2>
|
||||
<div class="contained">
|
||||
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
|
||||
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Same geometry, different parents</h2>
|
||||
<div class="same-place">
|
||||
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
|
||||
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partially overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fully overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>CSS-hidden variants</h2>
|
||||
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
|
||||
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
|
||||
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Hidden ancestor variants</h2>
|
||||
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
|
||||
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
|
||||
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
|
||||
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Zero-size and offscreen</h2>
|
||||
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
|
||||
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Unlabeled Rendered Surface matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
.case p { margin: 0 0 8px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Unlabeled Rendered Surface matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Only engineering attributes</h2>
|
||||
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Blank explicit names</h2>
|
||||
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Nearby prose is not a Canvas name</h2>
|
||||
<p>must-not-be-inferred-as-a-surface-label</p>
|
||||
<canvas width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fallback AX content does not name or suppress the Canvas</h2>
|
||||
<canvas width="180" height="80">
|
||||
<button type="button">Fallback control remains separate</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed rendering layers still cluster</h2>
|
||||
<div class="stack">
|
||||
<canvas width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed contained canvases stay independent</h2>
|
||||
<div class="contained">
|
||||
<canvas width="190" height="100"></canvas>
|
||||
<canvas width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,22 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface viewport matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { min-height: 1800px; padding: 16px; }
|
||||
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
|
||||
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
|
||||
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Viewport matrix</h1>
|
||||
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
|
||||
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
|
||||
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session <id>` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
@@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode;
|
||||
pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
@@ -228,6 +229,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||
summary: "snapshot ref does not support this operation",
|
||||
hint: Some(
|
||||
"visual surface refs are screenshot-only; run `bsk screenshot --ref <ref> --session <id>`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
|
||||
summary: "target element is not a <select>",
|
||||
hint: Some(
|
||||
@@ -377,6 +385,18 @@ mod tests {
|
||||
assert!(!info.summary.contains("sandbox"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_capability_denied_explains_visual_surface_refs() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(info.summary, "snapshot ref does not support this operation");
|
||||
assert!(
|
||||
info.hint.unwrap().contains("screenshot-only"),
|
||||
"expected visual-surface-specific hint"
|
||||
);
|
||||
assert_eq!(info.exit_code, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_busy_overrides_timeout_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
/**
|
||||
* Archive-triggered bsk session cleanup. DSH conversation archival writes
|
||||
* the workspace registry's durable global state, which the storage domain
|
||||
* broadcasts as `domain/changed` ({domain: 'workspace', table: ''}) with
|
||||
* the new `archivedSessionIds`. The watcher diffs that set and stops every
|
||||
* bsk session owned by a freshly archived conversation — archived sessions
|
||||
* are hidden from every surface, so their Agent Windows would otherwise
|
||||
* linger with no way back to them.
|
||||
*
|
||||
* Ownership lineage: browser_session_start records the calling agent's
|
||||
* session id PLUS every ancestor along `header.parentSession` (a subagent's
|
||||
* browsers are reaped when any ancestor is archived, the root conversation
|
||||
* included). Lineage is resolved through the host session store; an
|
||||
* unloaded ancestor simply ends the walk.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ObservationService } from "./observation";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
/** The wire shape of a `domain/changed` frame (see dsh-storage-domain). */
|
||||
interface DomainChange {
|
||||
domain?: string;
|
||||
table?: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
/** The workspace registry's durable global singleton slice we read. */
|
||||
interface WorkspaceGlobal {
|
||||
archivedSessionIds?: unknown;
|
||||
}
|
||||
|
||||
/** Structural host session store face: only the lineage read is needed. */
|
||||
interface SessionStoreLike {
|
||||
get(id: string): { header: { parentSession?: string } } | undefined;
|
||||
}
|
||||
|
||||
/** Cap on lineage walks — defensive against a malformed parent chain. */
|
||||
const MAX_LINEAGE_DEPTH = 16;
|
||||
|
||||
/**
|
||||
* The DSH session ids that own a tool call's browser sessions: the calling
|
||||
* agent's own session plus every ancestor along the seed lineage. Empty
|
||||
* when the call carried no agent identity (those sessions outlive any
|
||||
* archive cleanup by design — nothing can name their owner).
|
||||
*/
|
||||
export function ownerSessionIds(ctx: Context, agentId: string | undefined): string[] {
|
||||
if (agentId === undefined) return [];
|
||||
const store = ctx.get("sessions") as SessionStoreLike | undefined;
|
||||
const ids: string[] = [];
|
||||
let current: string | undefined = agentId;
|
||||
for (let depth = 0; current !== undefined && depth < MAX_LINEAGE_DEPTH; depth += 1) {
|
||||
if (ids.includes(current)) break; // a cycle in stored headers must not loop
|
||||
ids.push(current);
|
||||
current = store?.get(current)?.header.parentSession;
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch conversation archival and stop the bsk sessions it opened. Returns
|
||||
* the disposer (plugin unload). In compositions without the workspace
|
||||
* domain (headless), the event simply never fires — and a context without
|
||||
* the events mixin degrades to a no-op like the other optional seams.
|
||||
*/
|
||||
export function armArchiveCleanup(
|
||||
ctx: Context,
|
||||
registry: SessionRegistry,
|
||||
observation: ObservationService,
|
||||
): () => void {
|
||||
// 'domain/changed' lives outside the vendored Events type map, so the
|
||||
// listener goes through a structural view of the events mixin.
|
||||
const on = (
|
||||
ctx as { on?: (event: string, listener: (change: DomainChange) => void) => () => void }
|
||||
).on;
|
||||
if (typeof on !== "function") return () => {};
|
||||
|
||||
/** Archived ids already accounted for; lazily seeded from the registry. */
|
||||
let seen: Set<string> | undefined;
|
||||
const initialize = (): Set<string> => {
|
||||
if (seen === undefined) {
|
||||
const registryService = ctx.get("workspaceRegistry") as
|
||||
| { archivedSessionIds?: readonly string[] }
|
||||
| undefined;
|
||||
seen = new Set(registryService?.archivedSessionIds ?? []);
|
||||
}
|
||||
return seen;
|
||||
};
|
||||
|
||||
return on.call(ctx, "domain/changed", (change: DomainChange) => {
|
||||
if (change?.domain !== "workspace" || change?.table !== "") return;
|
||||
const archived = (change.value as WorkspaceGlobal | undefined)?.archivedSessionIds;
|
||||
if (!Array.isArray(archived)) return;
|
||||
const previous = initialize();
|
||||
const fresh = archived.filter(
|
||||
(id): id is string => typeof id === "string" && !previous.has(id),
|
||||
);
|
||||
seen = new Set(archived.filter((id): id is string => typeof id === "string"));
|
||||
for (const dshSessionId of fresh) {
|
||||
for (const sessionId of registry.ownedByDsh(dshSessionId)) {
|
||||
// stopSession owns the full teardown (kill in-flight tools, queue
|
||||
// the daemon stop, drop registry + observation entries); a failure
|
||||
// just leaves the session for idle timeout or unload cleanup.
|
||||
void observation.stopSession(sessionId).catch(() => {});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -30,51 +30,17 @@
|
||||
color: var(--card-foreground);
|
||||
}
|
||||
|
||||
/* Sidebar carrier: fill the better-sidebar tab area and let the body flex
|
||||
(the tab content container owns the outer sizing). */
|
||||
.sidebar-tab {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.sidebar-tab > * {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
/* The BSK product mark in the sidebar tab strip (the source artwork already
|
||||
carries rounded corners — just soften the square to match). */
|
||||
.brand-icon {
|
||||
display: block;
|
||||
border-radius: 22%;
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 8px;
|
||||
cursor: move;
|
||||
user-select: none;
|
||||
touch-action: none;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
/* Only the floating card drags by its header — the sidebar tab does not. */
|
||||
.header[data-draggable] {
|
||||
cursor: move;
|
||||
}
|
||||
|
||||
/* In the sidebar the status row is chrome, not a card edge: drop its bottom
|
||||
border. A fixed-height bordered row can never track the shell header's
|
||||
hairline across the panel boundary (its height varies per window), and
|
||||
the BSK warm border tint clashes with the sidebar's neutral hairlines —
|
||||
borderless, the stage's own background provides the separation. */
|
||||
.sidebar-tab .header {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.status-text {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
@@ -165,12 +131,6 @@
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.actions-group {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.tool-wrap {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
@@ -205,14 +165,8 @@
|
||||
color: var(--destructive);
|
||||
}
|
||||
|
||||
/* Stop session: danger-tinted like interrupt. */
|
||||
.tool-stop {
|
||||
color: var(--destructive);
|
||||
}
|
||||
|
||||
/* Hover the wrap (not just :hover on the button) so the tooltip target
|
||||
lights the icon up too. The corner resize handles sit above this bar, so
|
||||
the outer sliver of the corner-most buttons starts a resize instead. */
|
||||
/* Hover the wrap (not just :hover on the button) so the tooltip target and
|
||||
the left-corner resize overlay still light the icon up. */
|
||||
.tool-wrap:hover .tool-button:not(:disabled) {
|
||||
color: var(--foreground);
|
||||
background: var(--accent);
|
||||
@@ -227,15 +181,6 @@
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
/* The armed (confirm) state of the stop button: solid destructive fill so
|
||||
the second click reads as final. Written at the wrap-hover specificity so
|
||||
the hover rules above never wash it out. */
|
||||
.tool-button.tool-stop-armed,
|
||||
.tool-wrap:hover .tool-button.tool-stop-armed {
|
||||
color: var(--destructive-foreground);
|
||||
background: var(--destructive);
|
||||
}
|
||||
|
||||
.hint {
|
||||
position: absolute;
|
||||
bottom: calc(100% + 4px);
|
||||
@@ -258,16 +203,13 @@
|
||||
right: 0;
|
||||
}
|
||||
|
||||
/* Invisible hit targets — no grip glyph; all four corners resize. Must sit
|
||||
above the bottom .actions bar (z-index 3): it spans the full card width
|
||||
and would otherwise swallow every pointer event aimed at the sw/se
|
||||
corners. The top corners already win because .header is not positioned. */
|
||||
/* Invisible hit targets — no grip glyph; all four corners resize. */
|
||||
.resize-handle {
|
||||
position: absolute;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
touch-action: none;
|
||||
z-index: 4;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.resize-handle[data-corner="nw"] {
|
||||
|
||||
@@ -3,18 +3,13 @@
|
||||
// composer's way); collapsed = a status capsule; "pop out" upgrades the same
|
||||
// content to a native Document PiP window (user gesture required by the
|
||||
// browser). Multi-session renders a meeting-style strip under the focus view.
|
||||
// When the dsh-better-sidebar plugin is installed the tracking view moves
|
||||
// into a sidebar tab instead (see observation-sidebar.tsx) and this floating
|
||||
// card hides itself via the sidebar-mode flag. Visuals follow the
|
||||
// BrowserSkill product family: @browser-skill/ui components and oklch tokens
|
||||
// on a .bsk-obs scope root, so the card reads as BSK's own surface without
|
||||
// leaking styles into (or inheriting themes from) the shell.
|
||||
// Visuals follow the BrowserSkill product family: @browser-skill/ui components
|
||||
// and oklch tokens on a .bsk-obs scope root, so the card reads as BSK's own
|
||||
// surface without leaking styles into (or inheriting themes from) the shell.
|
||||
|
||||
import { cn } from "@browser-skill/ui";
|
||||
import {
|
||||
RiArrowDownSLine,
|
||||
RiCheckLine,
|
||||
RiCloseCircleLine,
|
||||
RiErrorWarningLine,
|
||||
RiPictureInPicture2Line,
|
||||
RiPushpinFill,
|
||||
@@ -30,8 +25,6 @@ const IconPip = asIcon(RiPictureInPicture2Line);
|
||||
const IconDown = asIcon(RiArrowDownSLine);
|
||||
const IconWarn = asIcon(RiErrorWarningLine);
|
||||
const IconPin = asIcon(RiPushpinFill);
|
||||
const IconCloseSession = asIcon(RiCloseCircleLine);
|
||||
const IconCheck = asIcon(RiCheckLine);
|
||||
|
||||
import {
|
||||
type ReactNode,
|
||||
@@ -46,12 +39,11 @@ import { createPortal } from "react-dom";
|
||||
import type { SessionObservation } from "../observation";
|
||||
import css from "./ObservationOverlay.module.css";
|
||||
import type { ObservationClientStore } from "./observation-store";
|
||||
import { focusOf, statusOf, useObservationView, usePip } from "./observation-view";
|
||||
import { getSidebarMode, subscribeSidebarMode } from "./sidebar-mode";
|
||||
|
||||
// The pure view helpers live in observation-view (shared with the sidebar
|
||||
// tab); re-export so existing imports of this module keep working.
|
||||
export { focusOf, statusOf };
|
||||
/** Minimal Document PiP surface (TS lib.dom lacks it). */
|
||||
interface DocumentPip {
|
||||
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
|
||||
}
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
@@ -69,6 +61,11 @@ const EDGE_MARGIN = 16;
|
||||
/** Default dock: top-right, just under the shell's top bar (no spacing tokens exist in dsh 0.1). */
|
||||
const TOP_OFFSET = 64;
|
||||
|
||||
function pipApi(): DocumentPip | undefined {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
|
||||
}
|
||||
|
||||
function clampSize(size: Size, viewport: Size): Size {
|
||||
const maxW = viewport.w * 0.8;
|
||||
const maxH = viewport.h * 0.8;
|
||||
@@ -117,6 +114,23 @@ function formatElapsed(sinceMs: number, nowMs: number): string {
|
||||
return `${mm}:${ss}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-follow focus: the most recently touched session, but never yank focus
|
||||
* to a session whose latest action failed (errors flag the strip item, they
|
||||
* do not steal the stage) or one already reported dead.
|
||||
*/
|
||||
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
|
||||
if (sessions.length === 0) return undefined;
|
||||
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
|
||||
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
|
||||
return healthy ?? byRecency[0];
|
||||
}
|
||||
|
||||
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
|
||||
if (obs.lastError !== undefined && obs.action === "idle") return "error";
|
||||
return obs.action === "idle" ? "idle" : "active";
|
||||
}
|
||||
|
||||
/** Compact toolbar icon: no label, hover bubble for the name / semantics. */
|
||||
function IconAction(props: {
|
||||
label: string;
|
||||
@@ -152,83 +166,6 @@ function IconAction(props: {
|
||||
);
|
||||
}
|
||||
|
||||
/** How long the stop button stays armed before the confirm click expires. */
|
||||
const STOP_ARM_MS = 3000;
|
||||
|
||||
/**
|
||||
* Stop-session button with a lightweight two-click confirm: the first click
|
||||
* arms the button (solid red, check icon, short expiry), the second executes
|
||||
* the stop. No dialog, no layout shift — and a stray single click can never
|
||||
* close an Agent Window.
|
||||
*/
|
||||
function StopSessionAction(props: {
|
||||
sessionId: string | undefined;
|
||||
onStop: (sessionId: string) => Promise<boolean>;
|
||||
}) {
|
||||
const { sessionId, onStop } = props;
|
||||
const [hover, setHover] = useState(false);
|
||||
const [armed, setArmed] = useState(false);
|
||||
const [stopping, setStopping] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!armed) return;
|
||||
const timer = setTimeout(() => setArmed(false), STOP_ARM_MS);
|
||||
return () => clearTimeout(timer);
|
||||
}, [armed]);
|
||||
|
||||
// The focused session can vanish mid-arm (stopped elsewhere): disarm.
|
||||
useEffect(() => {
|
||||
if (sessionId === undefined) setArmed(false);
|
||||
}, [sessionId]);
|
||||
|
||||
const disabled = sessionId === undefined || stopping;
|
||||
const label = stopping
|
||||
? "Stopping session…"
|
||||
: armed
|
||||
? `Confirm stop session ${sessionId ?? ""}`
|
||||
: `Stop session ${sessionId ?? ""}`;
|
||||
const hint = stopping
|
||||
? "Closing the Agent Window…"
|
||||
: armed
|
||||
? `Click again to stop session ${sessionId ?? ""} and close its Agent Window.`
|
||||
: `Stop session ${sessionId ?? ""} and close its Agent Window.`;
|
||||
|
||||
return (
|
||||
<span
|
||||
className={css["tool-wrap"]}
|
||||
onPointerEnter={() => setHover(true)}
|
||||
onPointerLeave={() => setHover(false)}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(css["tool-button"], css["tool-stop"], armed && css["tool-stop-armed"])}
|
||||
disabled={disabled}
|
||||
aria-label={label}
|
||||
aria-pressed={armed}
|
||||
data-armed={armed || undefined}
|
||||
onClick={() => {
|
||||
if (sessionId === undefined || stopping) return;
|
||||
if (!armed) {
|
||||
setArmed(true);
|
||||
return;
|
||||
}
|
||||
setArmed(false);
|
||||
setStopping(true);
|
||||
const target = sessionId;
|
||||
void onStop(target).finally(() => setStopping(false));
|
||||
}}
|
||||
>
|
||||
{armed ? <IconCheck size={16} /> : <IconCloseSession size={16} />}
|
||||
</button>
|
||||
{hover ? (
|
||||
<span className={css.hint} role="tooltip">
|
||||
{hint}
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
/** Flat status dot, specced after the BSK popup's ConnectionStatusIndicator. */
|
||||
function StatusDot({ state }: { state: "active" | "idle" | "error" | "dead" }) {
|
||||
const color =
|
||||
@@ -309,8 +246,8 @@ function StripItem(props: {
|
||||
);
|
||||
}
|
||||
|
||||
/** The floating card / PiP / sidebar-tab shared content. */
|
||||
export function OverlayBody(props: {
|
||||
/** The floating card / PiP shared content. */
|
||||
function OverlayBody(props: {
|
||||
store: ObservationClientStore;
|
||||
focus: SessionObservation | undefined;
|
||||
sessions: readonly SessionObservation[];
|
||||
@@ -373,7 +310,6 @@ export function OverlayBody(props: {
|
||||
<div
|
||||
className={css.header}
|
||||
data-testid="obs-header"
|
||||
data-draggable={onHeaderPointerDown !== undefined || undefined}
|
||||
onPointerDown={onHeaderPointerDown}
|
||||
role="presentation"
|
||||
>
|
||||
@@ -427,21 +363,15 @@ export function OverlayBody(props: {
|
||||
</div>
|
||||
) : null}
|
||||
<div className={css.actions}>
|
||||
<div className={css["actions-group"]}>
|
||||
<IconAction
|
||||
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
|
||||
hint="Stop the current browser action."
|
||||
disabled={!canInterrupt}
|
||||
danger
|
||||
onClick={onInterrupt}
|
||||
>
|
||||
<IconStop size={16} />
|
||||
</IconAction>
|
||||
<StopSessionAction
|
||||
sessionId={focus?.sessionId}
|
||||
onStop={(sessionId) => store.stopSession(sessionId)}
|
||||
/>
|
||||
</div>
|
||||
<IconAction
|
||||
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
|
||||
hint="Stop the current browser action."
|
||||
disabled={!canInterrupt}
|
||||
danger
|
||||
onClick={onInterrupt}
|
||||
>
|
||||
<IconStop size={16} />
|
||||
</IconAction>
|
||||
{onPopOut !== undefined ? (
|
||||
<IconAction
|
||||
label="Pop out into a mini window"
|
||||
@@ -457,17 +387,25 @@ export function OverlayBody(props: {
|
||||
);
|
||||
}
|
||||
|
||||
/** Clone the host document's style/link nodes into a PiP window. */
|
||||
function cloneStylesInto(pipWindow: Window): void {
|
||||
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
|
||||
pipWindow.document.head.appendChild(node.cloneNode(true));
|
||||
}
|
||||
}
|
||||
|
||||
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
|
||||
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store);
|
||||
// While the better-sidebar plugin carries the tracking view, this floating
|
||||
// card (and its capsule) stays out of the way; an already-open PiP window
|
||||
// keeps its portal until the user closes it.
|
||||
const sidebarMode = useSyncExternalStore(subscribeSidebarMode, getSidebarMode);
|
||||
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
|
||||
useEffect(() => {
|
||||
store.start();
|
||||
return () => store.stop();
|
||||
}, [store]);
|
||||
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [pos, setPos] = useState<Point | null>(null);
|
||||
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
|
||||
const { pipWindow, pipSupported, popOut } = usePip();
|
||||
const [pipWindow, setPipWindow] = useState<Window | null>(null);
|
||||
const [pinnedId, setPinnedId] = useState<string | null>(null);
|
||||
const dragRef = useRef<{
|
||||
kind: "move" | "resize";
|
||||
corner?: ResizeCorner;
|
||||
@@ -476,6 +414,23 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
|
||||
base: Point & Size;
|
||||
} | null>(null);
|
||||
|
||||
// Elapsed-time ticker: 1s while anything is active.
|
||||
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
useEffect(() => {
|
||||
if (!anyActive) return;
|
||||
const timer = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => clearInterval(timer);
|
||||
}, [anyActive]);
|
||||
|
||||
// Focus: the pinned session wins while it still exists; otherwise auto-follow.
|
||||
const pinned =
|
||||
pinnedId !== null ? snapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
|
||||
const focus = pinned ?? focusOf(snapshot.sessions);
|
||||
const onTogglePin = useCallback((sessionId: string) => {
|
||||
setPinnedId((current) => (current === sessionId ? null : sessionId));
|
||||
}, []);
|
||||
|
||||
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
|
||||
|
||||
const onPointerMove = useCallback((event: PointerEvent) => {
|
||||
@@ -556,21 +511,34 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
|
||||
document.addEventListener("pointerup", onPointerUp);
|
||||
};
|
||||
|
||||
const popOut = async (): Promise<void> => {
|
||||
const pip = pipApi();
|
||||
if (pip === undefined) return;
|
||||
try {
|
||||
const win = await pip.requestWindow({ width: size.w, height: size.h });
|
||||
cloneStylesInto(win);
|
||||
win.addEventListener("pagehide", () => setPipWindow(null));
|
||||
setPipWindow(win);
|
||||
} catch {
|
||||
// requestWindow rejects without a user gesture (or when the window was
|
||||
// closed mid-request): stay on the in-page card, no state change.
|
||||
}
|
||||
};
|
||||
|
||||
// Hidden while no owned session exists (and no PiP is up).
|
||||
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
|
||||
|
||||
const body = (
|
||||
<OverlayBody
|
||||
store={store}
|
||||
focus={focus}
|
||||
sessions={snapshot.sessions}
|
||||
available={snapshot.available}
|
||||
pinnedId={pinnedId}
|
||||
pinnedId={pinned !== undefined ? pinnedId : null}
|
||||
onTogglePin={onTogglePin}
|
||||
now={now}
|
||||
inPip={pipWindow !== null}
|
||||
onPopOut={
|
||||
pipWindow === null && pipSupported
|
||||
? () => popOut({ width: size.w, height: size.h })
|
||||
: undefined
|
||||
}
|
||||
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
|
||||
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
|
||||
onHeaderPointerDown={pipWindow === null ? beginMove : undefined}
|
||||
/>
|
||||
@@ -580,12 +548,6 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
|
||||
return createPortal(body, pipWindow.document.body);
|
||||
}
|
||||
|
||||
// The sidebar tab is the carrier now — no floating card, no capsule.
|
||||
if (sidebarMode) return null;
|
||||
|
||||
// Hidden while no owned session exists (and no PiP is up).
|
||||
if (snapshot.sessions.length === 0) return null;
|
||||
|
||||
if (collapsed) {
|
||||
const state = focus !== undefined ? statusOf(focus) : "idle";
|
||||
return (
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
/**
|
||||
* The BrowserSkill product mark (apps/extension/assets/logo.png, downscaled
|
||||
* to 32px and inlined): the sidebar tab icon, so the tracking view reads as
|
||||
* BSK's own surface next to better-sidebar's built-in "browser" tab.
|
||||
* Regenerate with: resize the source to 32x32 PNG and replace the payload.
|
||||
*/
|
||||
export const BSK_LOGO_URL =
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAJGklEQVR42m2X248c2V3HP79zTlV19/TcPOOZWY+vsdmQFdngTQJswgs3hQ15iCJslAey+8xDAIHgH+CBNxIpj0gkEXnAKyKQWITEJlKWXSssEMfJRmuMHa/W9njHMz3Tl+nprsv5/XiomlugpdN16lR1n9/1e75fAbAbeLlO3Pz2ly4vjO7+sRbF58oiP49FEREOPwaIYQjS3IvUy0fP66kAZoYJIM7SpPVY0uy14cyzXzn9u9+4c+MG/vp1otiNa16uvxr7X/+1L3bK3lcTyU9P80ilR/uKCGYHOxxd6s2kmdaGIVYb1qyY1e8HD63UUblstwirf9J95fW/sRvXvAD0v/6Z6/O68Xf5/pS8kigiTsTkuCfIwfzAxQN37chjaAyt594JIXE4gbKMhog6Nd/qthi7lVe6L3/nG7Jz4/fPJ9u3b4U4WSwiKog/EcsTLh+PSuO3GXYs/D44QnBgyniQs7OV0+8bOxPPM6c9l660VEyF0Nmrlp9/IYT+/S/PUJ4a5C46EW8HnjVhBKjXaHJfz9Xq95xzhMThHZR5xXBrwtPNnP6eRzvzhJWzrF1d5UI759//9iYra5lrt1ycDeXsaOf+n4Y4HX8+r9SciasLS/6P12LHoyF47wipYKpM9gqePpyyvVMxjBn+1DILH17l0rkZlhYcLetT3n/A5FGfTtvTCh4ibjpVI81/J0il62V1rNStLp6j4jowxwiJh2iMevtsbU3pjYQim6Nz9hJLH13i8mrGbJgSxjuUvQfEjT2GOw4Zddh0BakPtFPH/jRKqaCSrwetKi9mKIIYhyFuOg6TurCcF7ae5ty5t48/vcTiz3+Ei+szLC06WnGA7TykfNCnmObk5nAtj+53yDcTZteMwZOChbkMVa3/14DCJBBBFBxyopndUUBqwwxu/XjAJ778Bc4ul+R3fgyj9yg/GDMpFZxHgoe0hQ9QDRzFI48zqHzJeKhcvNKiLAyiQ1BwgsOcqNZ9Xo96rgaVgmqdiBACH740y/1/+j4b7w1x80tUkwLM4VptJAQMcN6o+p7iYQIqhBQmsUIrR7cd0NI12ODAwJnWTpvZoRGqdRrS1CECeR7Z2NzHRBjeecgbX/kut39UUlx5ATe3gFUlAM5Tb/4oHCJl0oadUcFsK8E5sFh7ZaqgEFBr3KyhxAySxDMplDv3BuztlmhhZC6QqONDrXkW28Kbf/1d7A9+i6vPr7N/e5vQDZQ7jvxxUqdPwFRwHaP3sGR1boZYGabWwHdd7MHMMBUMh5mRJp6tnZx33tllJbS53J1jZj6BKpKPpkhZ8e5mj3Nf/FV+8ZOnmNz6Pr6TUvUcxeOAc0cN6xwUoWIyUhZXMqqq6bKmrQ0hWBQzdXX+nGOwV3LnJ30+trjE8mxGUUVG/Qn7owkt7/hpb5fwmat84qXz5P95EwmOqh8oNkKTSkFqICTJYDgpCOppZYGyjIfgdoApQRScCtEgDZ5b93dxuWd3OuWD8T5pqbSLyEyWcHdrh+S3r/Kpz15i/z/eQhIh9hOKjQRTyBYgFhAndSpDW+gNC+ZbGR6jjM3BVsNsXQNqhqpiOPIicuXcLFvdKT7xtNXTG+X0Bopt9Wi9dJUXX7rI5O23kNQRB4F8I0CEzgr0/B7t2YB/2iJOQDrK8HHJ+dkZqqI+JfUAYRXMGc6pM1MBBa2Ubttz5ews51c6XFjr8OJHT5N0HPIbz/Ppz11h/+03IXVUg4T8ccAidFaNp7LHzdd2uXdvj866gQq5L8nHMD+TUJUG0aAyJBqidZQcUfFqeK2vlEo1rYh5iUXl6eMB5eoKv37tOUb/9q+4LBD7gXKj3ry7Ak/iHrdeH/ArC7P07ykPekMWzkFvkNMmkHkgagNoDciogYKzaGg0YgSNYLEJj0ndfiGj6o/Z3i5or5+j2BTKJwlawcwKbMQ9fvK9Ib+QtgitQMg8d384wVYLdocls1mCad1+Fg1TDgeV4UxFTB2ow44NjQ5rwcwZ+MjqmLe/dZOBLOHHCeUUZteED+Ie73xvyHNpmyx1TERozXieW1/mO/+yzWhTWVtsU5aCqKvRT4+GmRDEHGJ6yPPMBHNGeqbCdwqs02X9l14g/a8H3PzWm/zyp1ZZXgn8tDfgnbeGPN/qIEDSyhh7w2OcPZWRsESWCq3EoVFPksZDMuUIps7qLjDUQBxkZ0tcu8KSGdIXPo3OLbOSdfj4YMIPbg6ZXxux9wRmWimFKe0k0J7v8H5vQHchpSoiC536bKhKEDlCJzM7JLJqhqtxoEYoUcEFw/Yduh8Jl38OP7eAlBOm29ssziV87JlluqMOL15ZZWVlhu0yMrfYxQQmZUU3TbCmnog1mRE1xOp9xOoaEwVRI9QnUGOECLoPNlcRlucJ6xeIgx3K/3kX23pINWnRdnBxtYNWkbkkMJ5JSLKEwaSgRFlsJWh1mNBDfsEBzZGaYIiBIgRtImCmmArhlBFOR1Q9xZ3b6OOHWFnifEq1U/84TpXgHL4hUh7hv3t91pY6eCeUpTYs2o6RisYEO6B5hpoSRCWaanNAGP5UVZ+r+YB4dxcbZ4QVx/Q9wSYg/uDYNvKyoorwgyfbtDuBC6c65HmFa5Jsh56f3NgO4iGeYC68n8Ty2WkphAUVv2hoLuggoXgUcJkRh0rVF7wHi3Vw1YxuSMh1zMpcmw8tdynzCmmCL1JjzQG3lpNU37xAdNnjYC79xywUfzbJNboEX214iqeeuN/IrkKw6BAvNUiJ1a2K0HKOT144jQBFoTWRayrc9LhWa1TTUTNqJ038wCf/LO//xZfWWw9u/zCt8uWpSiTixZ3UeJzQPkerdkyq1RryZwVN87CJBALR0ExULMtG4/VnPy4AG3/4m19YGG/9fT7NmZpEhzg5lCNHOo//774u7UYp2WEt1XrSftYMTcF32ynbrfmXz3ztjW86u3bNn/nq69/eaZ/6PUlbm/PB+YSapUozMG2uB0M5fI7WSKoKasjBu6qHczEjwWQ2iJc03XmaLb5y5mtvfNOuNeLUruHlVeK7f/7yxbXBu39U5sXnq7I8L2pH6rwRgCZyUpeKnZSOB5K2EbT1l5hPk0c+zV7bXXzmry7/5T/cvXENf/1V4v8CSYIrURx3kHgAAAAASUVORK5CYII=";
|
||||
@@ -1,9 +1,7 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt). The
|
||||
* default carrier is a floating card on the `shell.overlay` seat; when the
|
||||
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
|
||||
* moves into a sidebar tab instead (see observation-sidebar.tsx).
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt) floating
|
||||
* over the shell via the `shell.overlay` seat.
|
||||
*/
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
@@ -17,7 +15,6 @@ import { createElement } from "react";
|
||||
import "./bsk-tokens.nomodule.css";
|
||||
import "./bsk-ui.nomodule.css";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
@@ -84,14 +81,4 @@ export function apply(ctx: ClientContext): void {
|
||||
createElement(ObservationOverlay, { store }),
|
||||
),
|
||||
);
|
||||
// Optional carrier upgrade: when the dsh-better-sidebar plugin is installed,
|
||||
// its service moves the tracking view into a sidebar tab (the floating
|
||||
// overlay hides itself through the sidebar-mode flag). In profiles without
|
||||
// the sidebar plugin this fiber never runs and nothing changes.
|
||||
ctx.inject(["betterSidebar"], (injected) =>
|
||||
registerObservationSidebar(
|
||||
(injected as unknown as { betterSidebar: BetterSidebarLike }).betterSidebar,
|
||||
store,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
// better-sidebar carrier for the observation view. When the
|
||||
// dsh-better-sidebar plugin is installed its client publishes a
|
||||
// `betterSidebar` cordis service; the client entry then runs the
|
||||
// registration below and the tracking view moves from the floating card
|
||||
// into a single-instance sidebar tab (Document PiP pop-out stays available
|
||||
// from inside the tab). Detection is purely service-based — profiles
|
||||
// without the sidebar plugin never start this fiber and keep the floating
|
||||
// overlay.
|
||||
|
||||
import { createElement, type ReactNode } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { BSK_LOGO_URL } from "./brand-icon";
|
||||
import { OverlayBody } from "./ObservationOverlay";
|
||||
import css from "./ObservationOverlay.module.css";
|
||||
import type { ObservationClientStore } from "./observation-store";
|
||||
import { useObservationView, usePip, visibleToScope } from "./observation-view";
|
||||
import { setSidebarMode } from "./sidebar-mode";
|
||||
|
||||
/** The tab title — "Browser Skill", distinct from the sidebar's built-in "browser" tab. */
|
||||
const TAB_TITLE = "Browser Skill";
|
||||
|
||||
/** Tab strip icon: the BrowserSkill product mark at the requested size. */
|
||||
function TabIcon({ size }: { size: number }) {
|
||||
return (
|
||||
<img
|
||||
src={BSK_LOGO_URL}
|
||||
width={size}
|
||||
height={size}
|
||||
alt=""
|
||||
aria-hidden
|
||||
className={css["brand-icon"]}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural mirrors of dsh-better-sidebar's client service surface (only
|
||||
* the slices this integration touches — see the upstream
|
||||
* lib/types/client/service.d.ts). Declared locally so the plugin carries no
|
||||
* dependency on the sidebar package; the service contract has been stable
|
||||
* since v0.4.0 and newer capabilities arrive behind its `features` list.
|
||||
*/
|
||||
export interface SidebarTabLike {
|
||||
id: string;
|
||||
type: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
export interface SidebarLeafLike {
|
||||
kind: "leaf";
|
||||
id: string;
|
||||
tabs: SidebarTabLike[];
|
||||
active: string | null;
|
||||
}
|
||||
|
||||
export interface SidebarSplitLike {
|
||||
kind: "split";
|
||||
id: string;
|
||||
dir: "row" | "col";
|
||||
sizes: number[];
|
||||
children: SidebarNodeLike[];
|
||||
}
|
||||
|
||||
export type SidebarNodeLike = SidebarLeafLike | SidebarSplitLike;
|
||||
|
||||
export interface SidebarStateLike {
|
||||
splits: SidebarNodeLike;
|
||||
bottomSplits: SidebarNodeLike;
|
||||
/** Whether the right panel is expanded (the merged drawer on narrow screens). */
|
||||
panelOpen?: boolean;
|
||||
}
|
||||
|
||||
export interface SidebarSnapshotLike {
|
||||
sessionId?: string;
|
||||
state?: SidebarStateLike;
|
||||
}
|
||||
|
||||
/** Props every tab component receives (the slices we read). */
|
||||
export interface TabComponentPropsLike {
|
||||
/** The conversation this sidebar instance belongs to. */
|
||||
scope: { sessionId: string };
|
||||
}
|
||||
|
||||
export interface TabDescriptorLike {
|
||||
id: string;
|
||||
title: string | (() => string);
|
||||
icon?: ReactNode | ((size: number) => ReactNode);
|
||||
order?: number;
|
||||
/** Single-instance: opening focuses the existing tab instead of duplicating. */
|
||||
single?: boolean;
|
||||
/** Small pill on the tab strip; null/undefined hides it. */
|
||||
badge?: (
|
||||
ctx: unknown,
|
||||
scope: { sessionId: string },
|
||||
state: unknown,
|
||||
) => string | number | null | undefined;
|
||||
component: (props: TabComponentPropsLike) => ReactNode;
|
||||
}
|
||||
|
||||
export interface BetterSidebarLike {
|
||||
registerTab(descriptor: TabDescriptorLike): () => void;
|
||||
openTab(seed: {
|
||||
type: string;
|
||||
title?: string;
|
||||
/** Content seed (lands on tab.path); content opens expand the panel. */
|
||||
path?: string;
|
||||
}): void;
|
||||
getSnapshot(): SidebarSnapshotLike;
|
||||
/** Sidebar state feed (v0.12+): session switches, state and prefs changes. */
|
||||
subscribeState?: (listener: () => void) => () => void;
|
||||
isTabEnabled(id: string): boolean;
|
||||
}
|
||||
|
||||
/** The registered tab type id (also the SidebarTab.type value). */
|
||||
export const OBSERVATION_TAB_TYPE = "browserskill:observation";
|
||||
|
||||
/**
|
||||
* Inert content seed carried on auto-opened tabs: its mere presence makes
|
||||
* the sidebar treat the open as a content open (expanding the hosting panel
|
||||
* so the tracking view lands in sight). Never read by our component.
|
||||
*/
|
||||
export const OBSERVATION_TAB_PATH = "browser-skill:observation";
|
||||
|
||||
/**
|
||||
* The observation tab body: the same OverlayBody the floating card renders,
|
||||
* minus the card chrome (no drag header, no collapse — the sidebar tab bar
|
||||
* owns those), plus the PiP pop-out upgrade. The view is scoped to the
|
||||
* sidebar's conversation: only browser sessions started by it (or its
|
||||
* descendants) show here — the floating card keeps the global view.
|
||||
*/
|
||||
export function ObservationSidebarTab({
|
||||
store,
|
||||
scopeId,
|
||||
}: {
|
||||
store: ObservationClientStore;
|
||||
scopeId: string;
|
||||
}) {
|
||||
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store, scopeId);
|
||||
const { pipWindow, pipSupported, popOut } = usePip();
|
||||
|
||||
const body = (
|
||||
<OverlayBody
|
||||
store={store}
|
||||
focus={focus}
|
||||
sessions={snapshot.sessions}
|
||||
available={snapshot.available}
|
||||
pinnedId={pinnedId}
|
||||
onTogglePin={onTogglePin}
|
||||
now={now}
|
||||
inPip={pipWindow !== null}
|
||||
onPopOut={pipWindow === null && pipSupported ? () => popOut() : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
if (pipWindow !== null) {
|
||||
return createPortal(body, pipWindow.document.body);
|
||||
}
|
||||
return <div className={css["sidebar-tab"]}>{body}</div>;
|
||||
}
|
||||
|
||||
function* leafNodes(node: SidebarNodeLike): Generator<SidebarLeafLike> {
|
||||
if (node.kind === "leaf") {
|
||||
yield node;
|
||||
return;
|
||||
}
|
||||
for (const child of node.children) yield* leafNodes(child);
|
||||
}
|
||||
|
||||
/** Whether a tab of our type is already open in either sidebar workbench. */
|
||||
export function observationTabOpen(state: SidebarStateLike | undefined): boolean {
|
||||
if (state === undefined) return false;
|
||||
for (const root of [state.splits, state.bottomSplits]) {
|
||||
for (const leaf of leafNodes(root)) {
|
||||
if (leaf.tabs.some((tab) => tab.type === OBSERVATION_TAB_TYPE)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the observation sidebar tab and flip the carrier flag. Returns
|
||||
* the disposer the cordis fiber invokes when the sidebar service goes away
|
||||
* (plugin unload/HMR): the floating overlay then resumes as the carrier.
|
||||
*/
|
||||
export function registerObservationSidebar(
|
||||
service: BetterSidebarLike,
|
||||
store: ObservationClientStore,
|
||||
): () => void {
|
||||
setSidebarMode(true);
|
||||
// Hold the feed for the whole sidebar lifetime so the auto-open watcher
|
||||
// sees new sessions even while the tab itself is closed.
|
||||
store.acquire();
|
||||
const disposeTab = service.registerTab({
|
||||
id: OBSERVATION_TAB_TYPE,
|
||||
title: TAB_TITLE,
|
||||
icon: (size: number) => createElement(TabIcon, { size }),
|
||||
single: true,
|
||||
// The badge counts only sessions visible to the tab strip's own
|
||||
// conversation (global sessions from other conversations stay hidden,
|
||||
// mirroring the tab body's scoped view).
|
||||
badge: (_ctx, scope) => {
|
||||
const count = store
|
||||
.getSnapshot()
|
||||
.sessions.filter((s) => visibleToScope(s, scope.sessionId)).length;
|
||||
return count > 0 ? count : null;
|
||||
},
|
||||
component: (props) =>
|
||||
createElement(ObservationSidebarTab, { store, scopeId: props.scope.sessionId }),
|
||||
});
|
||||
|
||||
// Auto-open the tab when a browser session VISIBLE TO the active
|
||||
// conversation appears (and right away when one is already live). The
|
||||
// evaluation runs on observation publishes AND on sidebar state changes:
|
||||
// the latter covers the sidebar store's async init (its state is
|
||||
// undefined for the first beats after page load) and conversation
|
||||
// switches. The open carries a content seed (`path`): only content opens
|
||||
// land in sight — the sidebar expands the hosting panel for them. While
|
||||
// the panel is OPEN an existing tab is never re-focused — the user may
|
||||
// be reading another page on purpose; while it is collapsed a NEW
|
||||
// session (0→N) nudges the tab back into sight, mirroring how the
|
||||
// floating card used to reappear.
|
||||
const activeVisibleCount = (): number => {
|
||||
const activeId = service.getSnapshot().sessionId;
|
||||
if (activeId === undefined) return 0;
|
||||
return store.getSnapshot().sessions.filter((s) => visibleToScope(s, activeId)).length;
|
||||
};
|
||||
let previousVisible = activeVisibleCount();
|
||||
const evaluate = (): void => {
|
||||
const count = activeVisibleCount();
|
||||
const { state, sessionId } = service.getSnapshot();
|
||||
if (
|
||||
state !== undefined &&
|
||||
sessionId !== undefined &&
|
||||
service.isTabEnabled(OBSERVATION_TAB_TYPE)
|
||||
) {
|
||||
const open = observationTabOpen(state);
|
||||
if (count > 0 && !open) {
|
||||
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
|
||||
} else if (previousVisible === 0 && count > 0 && open && state.panelOpen === false) {
|
||||
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
|
||||
}
|
||||
}
|
||||
previousVisible = count;
|
||||
};
|
||||
evaluate();
|
||||
const unsubscribe = store.subscribe(evaluate);
|
||||
const unsubscribeState = service.subscribeState?.(evaluate);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
unsubscribeState?.();
|
||||
disposeTab();
|
||||
store.release();
|
||||
setSidebarMode(false);
|
||||
};
|
||||
}
|
||||
@@ -53,7 +53,6 @@ export interface OverlaySnapshot {
|
||||
const STATE_URL = "/bsk-observation/state";
|
||||
const EVENTS_URL = "/bsk-observation/events";
|
||||
const INTERRUPT_URL = "/bsk-observation/interrupt";
|
||||
const STOP_URL = "/bsk-observation/stop";
|
||||
|
||||
function revoke(url: string | undefined): void {
|
||||
if (url !== undefined && typeof URL.revokeObjectURL === "function") {
|
||||
@@ -79,8 +78,6 @@ export class ObservationClientStore {
|
||||
};
|
||||
private available = true;
|
||||
private started = false;
|
||||
/** Refcount of mounted consumers (overlay card, sidebar tab, sidebar fiber). */
|
||||
private consumers = 0;
|
||||
|
||||
constructor(private readonly deps: ObservationClientDeps) {}
|
||||
|
||||
@@ -192,23 +189,6 @@ export class ObservationClientStore {
|
||||
this.publish();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hold the feed for one consumer's lifetime: the stream starts with the
|
||||
* first holder and stops with the last release. Several carriers can share
|
||||
* the store (the floating card, the better-sidebar tab, and the sidebar
|
||||
* integration fiber) without one unmount killing the others' updates.
|
||||
*/
|
||||
acquire(): void {
|
||||
this.consumers += 1;
|
||||
if (this.consumers === 1) this.start();
|
||||
}
|
||||
|
||||
release(): void {
|
||||
if (this.consumers === 0) return;
|
||||
this.consumers -= 1;
|
||||
if (this.consumers === 0) this.stop();
|
||||
}
|
||||
|
||||
/** Forget one session's tracked + held frames, revoking their blob URLs. */
|
||||
private dropThumb(sessionId: string): void {
|
||||
const attachmentId = this.thumbBySession.get(sessionId);
|
||||
@@ -312,23 +292,4 @@ export class ObservationClientStore {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop one session and close its Agent Window. The session's removal
|
||||
* arrives through the SSE remove event — no local state is touched here.
|
||||
*/
|
||||
async stopSession(sessionId: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await this.deps.fetchFn(STOP_URL, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ sessionId }),
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
const body = (await res.json()) as { stopped?: boolean };
|
||||
return body.stopped === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
/**
|
||||
* Shared view logic for the observation carriers (the floating overlay card
|
||||
* and the better-sidebar tab): the store-backed view model (snapshot, focus
|
||||
* pinning, elapsed ticker) and the Document PiP pop-out. Extracted from
|
||||
* ObservationOverlay so both carriers run the same focus/interrupt behavior
|
||||
* without duplicating hooks.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useState, useSyncExternalStore } from "react";
|
||||
import type { SessionObservation } from "../observation";
|
||||
import type { ObservationClientStore, OverlaySnapshot } from "./observation-store";
|
||||
|
||||
/**
|
||||
* Auto-follow focus: the most recently touched session, but never yank focus
|
||||
* to a session whose latest action failed (errors flag the strip item, they
|
||||
* do not steal the stage) or one already reported dead.
|
||||
*/
|
||||
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
|
||||
if (sessions.length === 0) return undefined;
|
||||
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
|
||||
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
|
||||
return healthy ?? byRecency[0];
|
||||
}
|
||||
|
||||
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
|
||||
if (obs.lastError !== undefined && obs.action === "idle") return "error";
|
||||
return obs.action === "idle" ? "idle" : "active";
|
||||
}
|
||||
|
||||
/** Minimal Document PiP surface (TS lib.dom lacks it). */
|
||||
export interface DocumentPip {
|
||||
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
|
||||
}
|
||||
|
||||
export function pipApi(): DocumentPip | undefined {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
|
||||
}
|
||||
|
||||
/** Clone the host document's style/link nodes into a PiP window. */
|
||||
export function cloneStylesInto(pipWindow: Window): void {
|
||||
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
|
||||
pipWindow.document.head.appendChild(node.cloneNode(true));
|
||||
}
|
||||
}
|
||||
|
||||
export interface ObservationView {
|
||||
readonly snapshot: OverlaySnapshot;
|
||||
/** Pinned session wins while it still exists; otherwise auto-follow. */
|
||||
readonly focus: SessionObservation | undefined;
|
||||
readonly pinnedId: string | null;
|
||||
readonly onTogglePin: (sessionId: string) => void;
|
||||
/** 1s ticker while anything is active (drives the elapsed readout). */
|
||||
readonly now: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether one session is visible on a surface scoped to one DSH
|
||||
* conversation: sessions started by that conversation or its descendants
|
||||
* (the lineage ancestors ride along on the entry). Untracked sessions (no
|
||||
* owner recorded) are hidden from scoped surfaces but stay in global views.
|
||||
*/
|
||||
export function visibleToScope(obs: SessionObservation, scopeId: string): boolean {
|
||||
return obs.dshSessionIds?.includes(scopeId) === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* The store-backed observation view model. Holds the feed for the component
|
||||
* lifetime (refcounted — overlapping carriers never kill each other's
|
||||
* stream). `scopeId` narrows the view to one DSH conversation's sessions
|
||||
* (the better-sidebar tab); undefined keeps the global view (floating
|
||||
* card, PiP).
|
||||
*/
|
||||
export function useObservationView(
|
||||
store: ObservationClientStore,
|
||||
scopeId?: string,
|
||||
): ObservationView {
|
||||
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
|
||||
useEffect(() => {
|
||||
store.acquire();
|
||||
return () => store.release();
|
||||
}, [store]);
|
||||
|
||||
const scopedSnapshot: OverlaySnapshot =
|
||||
scopeId === undefined
|
||||
? snapshot
|
||||
: {
|
||||
...snapshot,
|
||||
sessions: snapshot.sessions.filter((s) => visibleToScope(s, scopeId)),
|
||||
};
|
||||
|
||||
const [pinnedId, setPinnedId] = useState<string | null>(null);
|
||||
const pinned =
|
||||
pinnedId !== null ? scopedSnapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
|
||||
const focus = pinned ?? focusOf(scopedSnapshot.sessions);
|
||||
const onTogglePin = useCallback((sessionId: string) => {
|
||||
setPinnedId((current) => (current === sessionId ? null : sessionId));
|
||||
}, []);
|
||||
|
||||
const anyActive = scopedSnapshot.sessions.some((s) => s.action !== "idle");
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
useEffect(() => {
|
||||
if (!anyActive) return;
|
||||
const timer = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => clearInterval(timer);
|
||||
}, [anyActive]);
|
||||
|
||||
return {
|
||||
snapshot: scopedSnapshot,
|
||||
focus,
|
||||
pinnedId: pinned !== undefined ? pinnedId : null,
|
||||
onTogglePin,
|
||||
now,
|
||||
};
|
||||
}
|
||||
|
||||
export interface PipHandle {
|
||||
/** The open PiP window, or null while the content renders in its carrier. */
|
||||
readonly pipWindow: Window | null;
|
||||
/** Whether the browser exposes Document PiP (Chrome-only today). */
|
||||
readonly pipSupported: boolean;
|
||||
/**
|
||||
* Upgrade the content into a native PiP window. Rejects silently without a
|
||||
* user gesture (or when closed mid-request): the carrier keeps the content.
|
||||
*/
|
||||
readonly popOut: (size?: { width: number; height: number }) => void;
|
||||
}
|
||||
|
||||
export function usePip(): PipHandle {
|
||||
const [pipWindow, setPipWindow] = useState<Window | null>(null);
|
||||
const popOut = useCallback((size?: { width: number; height: number }): void => {
|
||||
const pip = pipApi();
|
||||
if (pip === undefined) return;
|
||||
void pip
|
||||
.requestWindow(size)
|
||||
.then((win) => {
|
||||
cloneStylesInto(win);
|
||||
win.addEventListener("pagehide", () => setPipWindow(null));
|
||||
setPipWindow(win);
|
||||
})
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
// Unmounting the carrier (tab close, conversation switch, plugin HMR) does
|
||||
// not destroy the PiP browsing context: close it ourselves or it survives
|
||||
// as a blank window. close() on an already-closed window is a no-op.
|
||||
useEffect(() => () => pipWindow?.close(), [pipWindow]);
|
||||
return { pipWindow, pipSupported: pipApi() !== undefined, popOut };
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
/**
|
||||
* Carrier switch for the observation view: while the better-sidebar
|
||||
* integration fiber is alive (the dsh-better-sidebar plugin provides its
|
||||
* `betterSidebar` service), the tracking view lives in a sidebar tab and the
|
||||
* floating overlay card/capsule hides itself. The flag is a tiny external
|
||||
* store so the overlay can read it through useSyncExternalStore and flip
|
||||
* without a remount when the sidebar plugin (un)loads.
|
||||
*/
|
||||
|
||||
let active = false;
|
||||
const listeners = new Set<() => void>();
|
||||
|
||||
export function getSidebarMode(): boolean {
|
||||
return active;
|
||||
}
|
||||
|
||||
export function subscribeSidebarMode(listener: () => void): () => void {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
}
|
||||
|
||||
export function setSidebarMode(next: boolean): void {
|
||||
if (active === next) return;
|
||||
active = next;
|
||||
for (const listener of [...listeners]) listener();
|
||||
}
|
||||
@@ -12,7 +12,6 @@
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import Schema from "@deepseek-ai/schemastery";
|
||||
import { armArchiveCleanup } from "./archive-cleanup";
|
||||
import { armLazyTools } from "./lazy-tools";
|
||||
import { ObservationService } from "./observation";
|
||||
import { registerObservationRoutes } from "./observation-http";
|
||||
@@ -108,10 +107,6 @@ export function apply(
|
||||
removeRoutes = registerObservationRoutes(injected, observation);
|
||||
return () => removeRoutes();
|
||||
});
|
||||
// Reap a conversation's browsers when the conversation itself is archived:
|
||||
// archived sessions are hidden from every surface, so their Agent Windows
|
||||
// would otherwise linger unreachable until idle timeout or unload.
|
||||
const disarmArchiveCleanup = armArchiveCleanup(ctx, registry, observation);
|
||||
|
||||
// Non-blocking install probe: warn early when bsk is missing instead of
|
||||
// failing the first tool call with a bare spawn error. Uses --version on
|
||||
@@ -137,7 +132,6 @@ export function apply(
|
||||
removeSuite();
|
||||
unregisterSkill();
|
||||
removeRoutes();
|
||||
disarmArchiveCleanup();
|
||||
observation.dispose();
|
||||
runner.killAll();
|
||||
const stops = registry
|
||||
@@ -150,7 +144,6 @@ export function apply(
|
||||
});
|
||||
}
|
||||
|
||||
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
|
||||
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
|
||||
export { ObservationService } from "./observation";
|
||||
export { registerObservationRoutes } from "./observation-http";
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
* GET /bsk-observation/state → { sessions, available }
|
||||
* GET /bsk-observation/events → SSE stream of ObservationEvent
|
||||
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
|
||||
* POST /bsk-observation/stop → body {sessionId} → {stopped: boolean}
|
||||
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
|
||||
*
|
||||
* Routes exist only when a webServer service is mounted (web composition);
|
||||
@@ -167,42 +166,6 @@ export function registerObservationRoutes(
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/stop`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "POST") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
if (fenceRejected(req, res)) return;
|
||||
let body = "";
|
||||
req.on("data", (chunk: Buffer | string) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
let sessionId: string | undefined;
|
||||
try {
|
||||
const parsed = JSON.parse(body || "{}") as { sessionId?: unknown };
|
||||
if (typeof parsed.sessionId === "string" && parsed.sessionId !== "") {
|
||||
sessionId = parsed.sessionId;
|
||||
}
|
||||
} catch {
|
||||
sendJson(res, 400, { error: "invalid JSON body" });
|
||||
return;
|
||||
}
|
||||
// A destructive call always names its target (never "current").
|
||||
if (sessionId === undefined) {
|
||||
sendJson(res, 400, { error: "sessionId required" });
|
||||
return;
|
||||
}
|
||||
void observation.stopSession(sessionId).then(
|
||||
(stopped) => sendJson(res, 200, { stopped }),
|
||||
() => sendJson(res, 500, { error: "stop failed" }),
|
||||
);
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "prefix",
|
||||
path: `${ROUTE_BASE}/thumbnail`,
|
||||
|
||||
@@ -39,13 +39,6 @@ export interface SessionObservation {
|
||||
* greys it out until it is stopped/removed. No more frames are requested.
|
||||
*/
|
||||
dead?: boolean;
|
||||
/**
|
||||
* The DSH conversations this session belongs to (the starting agent's
|
||||
* session plus its seed-lineage ancestors), recorded at start. Scoped
|
||||
* surfaces (the better-sidebar tab) filter by it; absent means untracked
|
||||
* ownership — visible only in the global (unscoped) view.
|
||||
*/
|
||||
dshSessionIds?: string[];
|
||||
}
|
||||
|
||||
/** Incremental event carried to subscribers (SSE on the wire). */
|
||||
@@ -160,13 +153,11 @@ export class ObservationService {
|
||||
/** Register a fresh owned session (called from browser_session_start). */
|
||||
addSession(sessionId: string, url?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const dshSessionIds = this.deps.registry.dshOwnersOf(sessionId);
|
||||
this.put({
|
||||
sessionId,
|
||||
...(url !== undefined ? { url } : {}),
|
||||
action: "idle",
|
||||
since: this.scheduler.now(),
|
||||
...(dshSessionIds.length > 0 ? { dshSessionIds } : {}),
|
||||
});
|
||||
this.lastActivity.set(sessionId, this.scheduler.now());
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
@@ -237,36 +228,6 @@ export class ObservationService {
|
||||
return this.deps.runner.killFor(target) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop one owned session and close its Agent Window (the overlay's stop
|
||||
* button — same end state as `browser_session_stop`). Never waits behind a
|
||||
* hung in-flight command: tool children are killed first so the session's
|
||||
* keyed queue drains immediately, and no further captures queue up. A
|
||||
* session the daemon already forgot (dead strip entries) stops
|
||||
* idempotently — the goal state (entry gone) is identical.
|
||||
* @returns whether the session ended up stopped/removed.
|
||||
*/
|
||||
async stopSession(sessionId: string): Promise<boolean> {
|
||||
if (!this.deps.registry.isOwned(sessionId)) return false;
|
||||
this.cancelCapture(sessionId);
|
||||
this.deps.runner.killFor(sessionId);
|
||||
const result = await this.deps.queue.run(sessionId, () =>
|
||||
this.deps.runner.run(["session", "stop", sessionId], { timeoutMs: 30_000 }),
|
||||
);
|
||||
if (result.code !== 0) {
|
||||
let code: string | undefined;
|
||||
try {
|
||||
code = (JSON.parse(result.stdout) as { code?: string }).code;
|
||||
} catch {
|
||||
code = undefined;
|
||||
}
|
||||
if (code !== "session_not_found") return false;
|
||||
}
|
||||
this.deps.registry.remove(sessionId);
|
||||
this.removeSession(sessionId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one captured thumbnail from the in-process ring. Powers the plugin's
|
||||
* own HTTP thumbnail route — frames are plugin-owned runtime data, never
|
||||
|
||||
@@ -24,13 +24,6 @@ export interface TrackedSession {
|
||||
export class SessionRegistry {
|
||||
private readonly sessions = new Map<string, TrackedSession>();
|
||||
private currentId: string | undefined;
|
||||
/**
|
||||
* bsk session id → the DSH session ids that may clean it up: the agent
|
||||
* session that started it plus every ancestor along the seed lineage, so
|
||||
* archiving a conversation at ANY level of the chain reaps its browsers
|
||||
* (see archive-cleanup.ts).
|
||||
*/
|
||||
private readonly dshOwners = new Map<string, ReadonlySet<string>>();
|
||||
/**
|
||||
* Slots reserved by in-flight starts. reserveStart/completeStart/abandonStart
|
||||
* run synchronously around the async spawn, so concurrent starts can never
|
||||
@@ -79,37 +72,12 @@ export class SessionRegistry {
|
||||
/** Forget a session; falls back to the most recent remaining one. */
|
||||
remove(sessionId: string): void {
|
||||
this.sessions.delete(sessionId);
|
||||
this.dshOwners.delete(sessionId);
|
||||
if (this.currentId === sessionId) {
|
||||
const rest = [...this.sessions.values()];
|
||||
this.currentId = rest.length > 0 ? rest[rest.length - 1].sessionId : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record which DSH conversation(s) a freshly started bsk session belongs
|
||||
* to (the starting agent's session plus its ancestors). No-op without ids
|
||||
* — e.g. a start whose caller carried no agent identity.
|
||||
*/
|
||||
trackOwner(sessionId: string, dshSessionIds: readonly string[]): void {
|
||||
if (dshSessionIds.length === 0 || !this.sessions.has(sessionId)) return;
|
||||
this.dshOwners.set(sessionId, new Set(dshSessionIds));
|
||||
}
|
||||
|
||||
/** bsk session ids owned by the given DSH conversation (or its descendants). */
|
||||
ownedByDsh(dshSessionId: string): string[] {
|
||||
const owned: string[] = [];
|
||||
for (const [sessionId, owners] of this.dshOwners) {
|
||||
if (owners.has(dshSessionId)) owned.push(sessionId);
|
||||
}
|
||||
return owned;
|
||||
}
|
||||
|
||||
/** The DSH conversation ids owning one bsk session (empty when untracked). */
|
||||
dshOwnersOf(sessionId: string): string[] {
|
||||
return [...(this.dshOwners.get(sessionId) ?? [])];
|
||||
}
|
||||
|
||||
/** Mark an owned session as most recently used (recency order refresh). */
|
||||
private touch(sessionId: string): void {
|
||||
const existing = this.sessions.get(sessionId);
|
||||
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
type ToolResult,
|
||||
type ToolRunContext,
|
||||
} from "@deepseek-ai/dsh-tools";
|
||||
import { ownerSessionIds } from "./archive-cleanup";
|
||||
import { trySaveScreenshot } from "./image";
|
||||
import { actionForLabel, type ObservationService } from "./observation";
|
||||
import type { KeyedExecutor } from "./queue";
|
||||
@@ -246,9 +245,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
browserInstanceId: reply.browser_instance_id,
|
||||
startedAtMs: Date.now(),
|
||||
});
|
||||
// Ownership for archive cleanup: the calling conversation and its
|
||||
// ancestors reap this session when any of them is archived.
|
||||
registry.trackOwner(reply.session_id, ownerSessionIds(deps.ctx, exec.agent?.id));
|
||||
deps.observation.addSession(reply.session_id, args.url);
|
||||
try {
|
||||
if (args.device !== undefined) {
|
||||
@@ -489,7 +485,8 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
"probes) with @eN refs. Visual surface refs are screenshot-only; DOM refs support interaction. " +
|
||||
"Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
@@ -752,12 +749,14 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
name: "browser_screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
"This is the supported operation for visual surface refs emitted by browser_observe. Returns " +
|
||||
"the image itself when the deployment supports image input, otherwise a file path.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
ref: {
|
||||
type: "string",
|
||||
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||
description:
|
||||
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
// Archive-triggered cleanup: lineage resolution for browser-session
|
||||
// ownership, the registry's owner index, and the domain/changed watcher
|
||||
// that reaps a freshly archived conversation's bsk sessions.
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { armArchiveCleanup, ownerSessionIds } from "../src/archive-cleanup";
|
||||
import type { ObservationService } from "../src/observation";
|
||||
import { SessionRegistry } from "../src/sessions";
|
||||
|
||||
/** A ctx stub carrying a session store with the given lineage headers. */
|
||||
function ctxWithSessions(headers: Record<string, { parentSession?: string }>) {
|
||||
return {
|
||||
get: (key: string) =>
|
||||
key === "sessions"
|
||||
? {
|
||||
get: (id: string) => (headers[id] === undefined ? undefined : { header: headers[id] }),
|
||||
}
|
||||
: undefined,
|
||||
} as never;
|
||||
}
|
||||
|
||||
describe("ownerSessionIds", () => {
|
||||
it("walks the seed lineage to the root; empty without an agent identity", () => {
|
||||
expect(ownerSessionIds(ctxWithSessions({}), undefined)).toEqual([]);
|
||||
const ctx = ctxWithSessions({
|
||||
child: { parentSession: "parent" },
|
||||
parent: { parentSession: "root" },
|
||||
root: {},
|
||||
});
|
||||
expect(ownerSessionIds(ctx, "child")).toEqual(["child", "parent", "root"]);
|
||||
});
|
||||
|
||||
it("stops the walk at an unloaded ancestor", () => {
|
||||
const ctx = ctxWithSessions({ child: { parentSession: "gone" } });
|
||||
expect(ownerSessionIds(ctx, "child")).toEqual(["child", "gone"]);
|
||||
});
|
||||
|
||||
it("never loops on a malformed parent cycle", () => {
|
||||
const ctx = ctxWithSessions({
|
||||
a: { parentSession: "b" },
|
||||
b: { parentSession: "a" },
|
||||
});
|
||||
expect(ownerSessionIds(ctx, "a")).toEqual(["a", "b"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SessionRegistry owner tracking", () => {
|
||||
function start(registry: SessionRegistry, sessionId: string): void {
|
||||
registry.reserveStart();
|
||||
registry.completeStart({ sessionId, startedAtMs: 1 });
|
||||
}
|
||||
|
||||
it("indexes owners and forgets them on remove; ignores empty/unknown ownership", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "bsk1");
|
||||
start(registry, "bsk2");
|
||||
registry.trackOwner("bsk1", ["conv-a", "root"]);
|
||||
registry.trackOwner("bsk2", ["conv-b"]);
|
||||
expect(registry.ownedByDsh("root")).toEqual(["bsk1"]);
|
||||
expect(registry.ownedByDsh("conv-a")).toEqual(["bsk1"]);
|
||||
expect(registry.ownedByDsh("conv-b")).toEqual(["bsk2"]);
|
||||
expect(registry.ownedByDsh("nobody")).toEqual([]);
|
||||
registry.remove("bsk1");
|
||||
expect(registry.ownedByDsh("root")).toEqual([]);
|
||||
// Empty owner lists and unknown session ids record nothing.
|
||||
registry.trackOwner("bsk2", []);
|
||||
registry.trackOwner("ghost", ["conv-c"]);
|
||||
expect(registry.ownedByDsh("conv-c")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("armArchiveCleanup", () => {
|
||||
function harness(opts: { archived?: string[] } = {}) {
|
||||
const registry = new SessionRegistry(5);
|
||||
const observation = { stopSession: vi.fn(async () => true) };
|
||||
const listeners = new Set<(change: unknown) => void>();
|
||||
const ctx = {
|
||||
get: (key: string) =>
|
||||
key === "workspaceRegistry" ? { archivedSessionIds: opts.archived ?? [] } : undefined,
|
||||
on: (event: string, listener: (change: unknown) => void) => {
|
||||
if (event === "domain/changed") listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
};
|
||||
const emit = (change: unknown) => {
|
||||
for (const listener of [...listeners]) listener(change);
|
||||
};
|
||||
return {
|
||||
registry,
|
||||
observation: observation as unknown as ObservationService,
|
||||
stopSession: observation.stopSession,
|
||||
emit,
|
||||
arm: () =>
|
||||
armArchiveCleanup(ctx as never, registry, observation as unknown as ObservationService),
|
||||
};
|
||||
}
|
||||
|
||||
function startOwned(registry: SessionRegistry, bskId: string, owners: string[]): void {
|
||||
registry.reserveStart();
|
||||
registry.completeStart({ sessionId: bskId, startedAtMs: 1 });
|
||||
registry.trackOwner(bskId, owners);
|
||||
}
|
||||
|
||||
it("stops every bsk session owned by a freshly archived conversation, until disarmed", () => {
|
||||
const h = harness();
|
||||
startOwned(h.registry, "bsk1", ["conv-a", "root"]);
|
||||
startOwned(h.registry, "bsk2", ["conv-b"]);
|
||||
const disarm = h.arm();
|
||||
h.emit({
|
||||
domain: "workspace",
|
||||
table: "",
|
||||
value: { archivedSessionIds: ["root"] },
|
||||
});
|
||||
expect(h.stopSession).toHaveBeenCalledTimes(1);
|
||||
expect(h.stopSession).toHaveBeenCalledWith("bsk1");
|
||||
// After the disposer runs the watcher is silent again.
|
||||
disarm();
|
||||
h.emit({
|
||||
domain: "workspace",
|
||||
table: "",
|
||||
value: { archivedSessionIds: ["root", "conv-b"] },
|
||||
});
|
||||
expect(h.stopSession).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("ignores pre-archived ids, foreign domains, and malformed frames", () => {
|
||||
const h = harness({ archived: ["old-conv"] });
|
||||
startOwned(h.registry, "bsk1", ["old-conv"]);
|
||||
startOwned(h.registry, "bsk2", ["conv-a"]);
|
||||
h.arm();
|
||||
// Seeded from the registry: the pre-archived id must not retro-fire.
|
||||
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["old-conv"] } });
|
||||
h.emit({ domain: "settings", table: "", value: { archivedSessionIds: ["conv-a"] } });
|
||||
h.emit({ domain: "workspace", table: "rows", value: { archivedSessionIds: ["conv-a"] } });
|
||||
h.emit({ domain: "workspace", table: "", value: {} });
|
||||
expect(h.stopSession).not.toHaveBeenCalled();
|
||||
// A genuinely new archive still fires.
|
||||
h.emit({
|
||||
domain: "workspace",
|
||||
table: "",
|
||||
value: { archivedSessionIds: ["old-conv", "conv-a"] },
|
||||
});
|
||||
expect(h.stopSession).toHaveBeenCalledWith("bsk2");
|
||||
});
|
||||
|
||||
it("treats a re-archived session as fresh again after unarchive", () => {
|
||||
const h = harness();
|
||||
startOwned(h.registry, "bsk1", ["conv-a"]);
|
||||
h.arm();
|
||||
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["conv-a"] } });
|
||||
expect(h.stopSession).toHaveBeenCalledTimes(1);
|
||||
// Unarchive, then re-archive: the second archival cleans up again.
|
||||
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: [] } });
|
||||
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["conv-a"] } });
|
||||
expect(h.stopSession).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,7 @@
|
||||
// upgrade/fallback (mocked documentPictureInPicture). NOTE: use RTL's waitFor
|
||||
// (act-flushing), not vi.waitFor, when asserting store-driven UI updates.
|
||||
|
||||
import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyResize, ObservationOverlay } from "../../src/client/ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
@@ -157,45 +157,6 @@ describe("ObservationOverlay", () => {
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
});
|
||||
|
||||
it("stop session: the first click only arms the confirm, the second posts", async () => {
|
||||
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · idle/);
|
||||
const stop = screen.getByRole("button", { name: "Stop session s1" });
|
||||
fireEvent.click(stop);
|
||||
// Armed: a plain click must never close an Agent Window by itself.
|
||||
expect(h.fetches.some((f) => f.url === "/bsk-observation/stop")).toBe(false);
|
||||
const confirm = await screen.findByRole("button", { name: "Confirm stop session s1" });
|
||||
expect(confirm.getAttribute("aria-pressed")).toBe("true");
|
||||
fireEvent.click(confirm);
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
h.fetches.some(
|
||||
(f) => f.url === "/bsk-observation/stop" && f.init?.body === '{"sessionId":"s1"}',
|
||||
),
|
||||
).toBe(true),
|
||||
);
|
||||
});
|
||||
|
||||
it("disarms the stop confirm after the expiry window", async () => {
|
||||
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const stop = await screen.findByRole("button", { name: "Stop session s1" });
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
fireEvent.click(stop);
|
||||
// Synchronous queries only: waitFor-style finds hang under fake timers.
|
||||
expect(screen.getByRole("button", { name: "Confirm stop session s1" })).toBeTruthy();
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(3100);
|
||||
});
|
||||
expect(screen.getByRole("button", { name: "Stop session s1" })).toBeTruthy();
|
||||
expect(h.fetches.some((f) => f.url === "/bsk-observation/stop")).toBe(false);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("resizes within min/max clamps", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
@@ -261,7 +222,6 @@ describe("ObservationOverlay", () => {
|
||||
addEventListener: (name: string, fn: () => void) => {
|
||||
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
|
||||
},
|
||||
close: vi.fn(),
|
||||
} as unknown as Window;
|
||||
});
|
||||
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
|
||||
@@ -276,28 +236,6 @@ describe("ObservationOverlay", () => {
|
||||
await screen.findByTestId("obs-card");
|
||||
});
|
||||
|
||||
it("closes the PiP window when the carrier unmounts", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const pipDoc = document.implementation.createHTMLDocument("pip");
|
||||
const close = vi.fn();
|
||||
const requestWindow = vi.fn(async () => {
|
||||
return {
|
||||
document: pipDoc,
|
||||
addEventListener: () => {},
|
||||
close,
|
||||
} as unknown as Window;
|
||||
});
|
||||
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
|
||||
const { unmount } = render(<ObservationOverlay store={h.store} />);
|
||||
const popout = await screen.findByRole("button", { name: /Pop out/ });
|
||||
fireEvent.click(popout);
|
||||
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
|
||||
// Unmounting (tab close, conversation switch, plugin HMR) closes the PiP
|
||||
// instead of leaving a blank window behind.
|
||||
unmount();
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("renders the strip for two sessions and pins focus on click", async () => {
|
||||
const older: SessionObservation = {
|
||||
sessionId: "s1",
|
||||
|
||||
@@ -1,331 +0,0 @@
|
||||
// @vitest-environment happy-dom
|
||||
// better-sidebar carrier: tab registration + auto-open, the sidebar-mode
|
||||
// flag hiding the floating overlay, PiP pop-out from the tab, and the
|
||||
// store's refcounted acquire/release across overlapping carriers.
|
||||
|
||||
import { cleanup, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ObservationOverlay } from "../../src/client/ObservationOverlay";
|
||||
import {
|
||||
type BetterSidebarLike,
|
||||
OBSERVATION_TAB_PATH,
|
||||
OBSERVATION_TAB_TYPE,
|
||||
ObservationSidebarTab,
|
||||
observationTabOpen,
|
||||
registerObservationSidebar,
|
||||
type SidebarNodeLike,
|
||||
type SidebarStateLike,
|
||||
type TabDescriptorLike,
|
||||
} from "../../src/client/observation-sidebar";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import { getSidebarMode, setSidebarMode } from "../../src/client/sidebar-mode";
|
||||
import type { SessionObservation } from "../../src/observation";
|
||||
|
||||
// Owned by the sidebar mock's active conversation ("conv-1") so the scoped
|
||||
// visibility logic lets it through; FOREIGN belongs to another conversation.
|
||||
const BUSY: SessionObservation = {
|
||||
sessionId: "s1",
|
||||
action: "clicking",
|
||||
since: Date.now() - 7000,
|
||||
dshSessionIds: ["conv-1"],
|
||||
};
|
||||
const FOREIGN: SessionObservation = {
|
||||
sessionId: "s9",
|
||||
action: "clicking",
|
||||
since: Date.now() - 5000,
|
||||
dshSessionIds: ["other-conv"],
|
||||
};
|
||||
|
||||
interface Harness {
|
||||
store: ObservationClientStore;
|
||||
es: () => EventSourceLike | undefined;
|
||||
push: (sessions: SessionObservation[]) => void;
|
||||
}
|
||||
|
||||
function makeHarness(initial: SessionObservation[]): Harness {
|
||||
let es: EventSourceLike | undefined;
|
||||
let current = initial;
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: async (url: string) => {
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: current }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
},
|
||||
eventSourceFactory: () => {
|
||||
es = { onmessage: null, close: vi.fn() };
|
||||
return es;
|
||||
},
|
||||
loadImage: async (id: string) => `blob:${id}`,
|
||||
});
|
||||
return {
|
||||
store,
|
||||
es: () => es,
|
||||
push: (sessions) => {
|
||||
current = sessions;
|
||||
for (const s of sessions) {
|
||||
es?.onmessage?.({ data: JSON.stringify({ type: "upsert", session: s }) });
|
||||
}
|
||||
if (sessions.length === 0) es?.onmessage?.({ data: JSON.stringify({ type: "reset" }) });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function leafWith(type: string, panelOpen = true): SidebarStateLike {
|
||||
return {
|
||||
panelOpen,
|
||||
splits: {
|
||||
kind: "split",
|
||||
id: "root",
|
||||
dir: "row",
|
||||
sizes: [1],
|
||||
children: [
|
||||
{
|
||||
kind: "leaf",
|
||||
id: "pane-1",
|
||||
active: type,
|
||||
tabs: [{ id: type, type, title: type }],
|
||||
},
|
||||
],
|
||||
},
|
||||
bottomSplits: { kind: "leaf", id: "pane-2", active: null, tabs: [] },
|
||||
};
|
||||
}
|
||||
|
||||
interface SidebarMock {
|
||||
service: BetterSidebarLike;
|
||||
descriptor: () => TabDescriptorLike;
|
||||
openTab: ReturnType<typeof vi.fn>;
|
||||
disposeTab: ReturnType<typeof vi.fn>;
|
||||
setState: (state: SidebarStateLike | undefined) => void;
|
||||
}
|
||||
|
||||
/** Insert a tab into the first leaf of the main tree (mirrors a real open). */
|
||||
function withTabOpened(state: SidebarStateLike, type: string): SidebarStateLike {
|
||||
let done = false;
|
||||
const walk = (node: SidebarNodeLike): SidebarNodeLike => {
|
||||
if (done) return node;
|
||||
if (node.kind === "leaf") {
|
||||
done = true;
|
||||
return { ...node, active: type, tabs: [...node.tabs, { id: type, type, title: type }] };
|
||||
}
|
||||
return { ...node, children: node.children.map(walk) };
|
||||
};
|
||||
return { ...state, splits: walk(state.splits) };
|
||||
}
|
||||
|
||||
function makeSidebar(
|
||||
initialState: SidebarStateLike | undefined = leafWith("explorer"),
|
||||
): SidebarMock {
|
||||
let descriptor: TabDescriptorLike | undefined;
|
||||
let state: SidebarStateLike | undefined = initialState;
|
||||
const openTab = vi.fn((seed: { type: string }) => {
|
||||
// Mirror the real feedback loop: a created tab lands in the state, so
|
||||
// later evaluations see it open (single-instance dedupe re-focuses).
|
||||
if (state !== undefined && !observationTabOpen(state)) {
|
||||
state = withTabOpened(state, seed.type);
|
||||
}
|
||||
});
|
||||
const disposeTab = vi.fn();
|
||||
const service: BetterSidebarLike = {
|
||||
registerTab: (next) => {
|
||||
descriptor = next;
|
||||
return disposeTab;
|
||||
},
|
||||
openTab,
|
||||
getSnapshot: () => ({ sessionId: "conv-1", state }),
|
||||
isTabEnabled: () => true,
|
||||
};
|
||||
return {
|
||||
service,
|
||||
descriptor: () => {
|
||||
if (descriptor === undefined) throw new Error("no tab registered");
|
||||
return descriptor;
|
||||
},
|
||||
openTab,
|
||||
disposeTab,
|
||||
setState: (next) => {
|
||||
state = next;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
setSidebarMode(false);
|
||||
});
|
||||
|
||||
describe("registerObservationSidebar", () => {
|
||||
it("registers a single-instance tab and holds the feed for its lifetime", () => {
|
||||
const h = makeHarness([]);
|
||||
const sidebar = makeSidebar();
|
||||
expect(getSidebarMode()).toBe(false);
|
||||
const dispose = registerObservationSidebar(sidebar.service, h.store);
|
||||
expect(getSidebarMode()).toBe(true);
|
||||
// The fiber holds the observation feed itself, so the auto-open watcher
|
||||
// sees new sessions even while the tab is closed.
|
||||
expect(h.es()).toBeDefined();
|
||||
const descriptor = sidebar.descriptor();
|
||||
expect(descriptor.id).toBe(OBSERVATION_TAB_TYPE);
|
||||
expect(descriptor.single).toBe(true);
|
||||
expect(descriptor.title).toBe("Browser Skill");
|
||||
dispose();
|
||||
expect(getSidebarMode()).toBe(false);
|
||||
expect(sidebar.disposeTab).toHaveBeenCalledTimes(1);
|
||||
expect(h.es()?.close).toHaveBeenCalled?.();
|
||||
});
|
||||
|
||||
it("opens the tab right away when a session is already live at activation", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const sidebar = makeSidebar();
|
||||
registerObservationSidebar(sidebar.service, h.store);
|
||||
// The initial state fetch lands asynchronously; the 0→N watcher then fires.
|
||||
await waitFor(() =>
|
||||
expect(sidebar.openTab).toHaveBeenCalledWith({
|
||||
type: OBSERVATION_TAB_TYPE,
|
||||
path: OBSERVATION_TAB_PATH,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("auto-opens on the first session but never re-focuses an open tab", async () => {
|
||||
const h = makeHarness([]);
|
||||
const sidebar = makeSidebar();
|
||||
registerObservationSidebar(sidebar.service, h.store);
|
||||
expect(sidebar.openTab).not.toHaveBeenCalled();
|
||||
// First session arrives: the tab lands in the sidebar.
|
||||
h.push([BUSY]);
|
||||
await waitFor(() => expect(sidebar.openTab).toHaveBeenCalledTimes(1));
|
||||
// The sidebar now reports the tab open (persisted/restored): further
|
||||
// session arrivals must not steal focus from whatever the user reads.
|
||||
sidebar.setState(leafWith(OBSERVATION_TAB_TYPE));
|
||||
h.push([]);
|
||||
h.push([{ ...BUSY, sessionId: "s2" }]);
|
||||
h.push([]);
|
||||
h.push([{ ...BUSY, sessionId: "s3" }]);
|
||||
await waitFor(() => expect(h.store.getSnapshot().sessions.length).toBe(1));
|
||||
expect(sidebar.openTab).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not open while the sidebar has no active session state", () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const sidebar = makeSidebar(undefined);
|
||||
registerObservationSidebar(sidebar.service, h.store);
|
||||
expect(sidebar.openTab).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("nudges an existing tab back into sight when the panel is collapsed", async () => {
|
||||
const h = makeHarness([]);
|
||||
// Tab already open (persisted), but the panel is collapsed: a new
|
||||
// session should surface the tracking view again (focus + expand via a
|
||||
// content open), like the floating card reappearing.
|
||||
const sidebar = makeSidebar(leafWith(OBSERVATION_TAB_TYPE, false));
|
||||
registerObservationSidebar(sidebar.service, h.store);
|
||||
h.push([BUSY]);
|
||||
await waitFor(() =>
|
||||
expect(sidebar.openTab).toHaveBeenCalledWith({
|
||||
type: OBSERVATION_TAB_TYPE,
|
||||
path: OBSERVATION_TAB_PATH,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the visible session count as the tab badge", () => {
|
||||
const h = makeHarness([]);
|
||||
const sidebar = makeSidebar();
|
||||
registerObservationSidebar(sidebar.service, h.store);
|
||||
const badge = sidebar.descriptor().badge;
|
||||
const scope = { sessionId: "conv-1" };
|
||||
expect(badge?.(undefined, scope, undefined)).toBeNull();
|
||||
h.store.acquire();
|
||||
h.push([BUSY, FOREIGN]);
|
||||
expect(badge?.(undefined, scope, undefined)).toBe(1);
|
||||
expect(badge?.(undefined, { sessionId: "other-conv" }, undefined)).toBe(1);
|
||||
expect(badge?.(undefined, { sessionId: "nobody" }, undefined)).toBeNull();
|
||||
h.store.release();
|
||||
});
|
||||
|
||||
it("does not auto-open for sessions owned by other conversations", async () => {
|
||||
const h = makeHarness([]);
|
||||
const sidebar = makeSidebar();
|
||||
registerObservationSidebar(sidebar.service, h.store);
|
||||
h.push([FOREIGN]);
|
||||
await waitFor(() => expect(h.store.getSnapshot().sessions.length).toBe(1));
|
||||
expect(sidebar.openTab).not.toHaveBeenCalled();
|
||||
// A session visible to the active conversation does open the tab.
|
||||
h.push([BUSY]);
|
||||
await waitFor(() => expect(sidebar.openTab).toHaveBeenCalledTimes(1));
|
||||
});
|
||||
});
|
||||
|
||||
describe("observationTabOpen", () => {
|
||||
it("finds the tab in either workbench tree", () => {
|
||||
expect(observationTabOpen(undefined)).toBe(false);
|
||||
expect(observationTabOpen(leafWith("explorer"))).toBe(false);
|
||||
expect(observationTabOpen(leafWith(OBSERVATION_TAB_TYPE))).toBe(true);
|
||||
const inBottom: SidebarStateLike = {
|
||||
splits: { kind: "leaf", id: "p1", active: null, tabs: [] },
|
||||
bottomSplits: {
|
||||
kind: "leaf",
|
||||
id: "p2",
|
||||
active: OBSERVATION_TAB_TYPE,
|
||||
tabs: [{ id: OBSERVATION_TAB_TYPE, type: OBSERVATION_TAB_TYPE, title: "Browser" }],
|
||||
},
|
||||
};
|
||||
expect(observationTabOpen(inBottom)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ObservationSidebarTab", () => {
|
||||
it("renders the tracking view without card chrome (no collapse, no drag header)", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
expect(screen.queryByRole("button", { name: "Collapse" })).toBeNull();
|
||||
expect(screen.getByTestId("obs-header").dataset.draggable).toBeUndefined();
|
||||
// No resize handles — the sidebar owns the geometry.
|
||||
expect(screen.queryByTestId("obs-resize-se")).toBeNull();
|
||||
// Session controls ride along: interrupt + stop-with-confirm.
|
||||
expect(screen.getByRole("button", { name: "Stop session s1" })).toBeTruthy();
|
||||
});
|
||||
|
||||
it("scopes the view to the sidebar's conversation", async () => {
|
||||
const h = makeHarness([BUSY, FOREIGN]);
|
||||
render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
|
||||
// Focus and content belong to the owned session; the foreign one never
|
||||
// reaches the stage, the status line, or the strip (which needs 2+).
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
expect(screen.queryByText(/s9/)).toBeNull();
|
||||
expect(screen.queryByTestId("obs-strip")).toBeNull();
|
||||
// Scoping to the other conversation swaps what's visible.
|
||||
render(<ObservationSidebarTab store={h.store} scopeId="other-conv" />);
|
||||
await screen.findByText(/s9 · clicking/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("carrier switching", () => {
|
||||
it("hides the floating overlay while sidebar mode is active", async () => {
|
||||
const h = makeHarness([]);
|
||||
const { container } = render(<ObservationOverlay store={h.store} />);
|
||||
h.push([BUSY]);
|
||||
await screen.findByTestId("obs-card");
|
||||
setSidebarMode(true);
|
||||
await waitFor(() => expect(container.firstChild).toBeNull());
|
||||
setSidebarMode(false);
|
||||
await screen.findByTestId("obs-card");
|
||||
});
|
||||
|
||||
it("keeps the feed alive across overlapping carriers (refcount)", async () => {
|
||||
const h = makeHarness([]);
|
||||
const overlay = render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.es()).toBeDefined());
|
||||
// A second carrier mounts (the sidebar tab), then the overlay unmounts:
|
||||
// the stream must survive for the remaining consumer.
|
||||
const tab = render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
|
||||
overlay.unmount();
|
||||
h.push([BUSY]);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
expect(h.es()?.close ?? vi.fn()).not.toHaveBeenCalled();
|
||||
tab.unmount();
|
||||
});
|
||||
});
|
||||
@@ -137,9 +137,6 @@ describe("client plugin registration", () => {
|
||||
const sessions = { binding: () => undefined };
|
||||
const ctx = {
|
||||
get: (key: string) => (key === "sessions" ? sessions : undefined),
|
||||
// cordis ctx.inject: the betterSidebar carrier upgrade stays dormant in
|
||||
// this composition (the callback only runs once the service exists).
|
||||
inject: (_deps: string[], _fn: (injected: unknown) => unknown) => {},
|
||||
slots: {
|
||||
inject: (_name: string, fn: () => unknown) => fn(),
|
||||
register: (slot: { name: string; key?: string; id?: string }, view: unknown) => {
|
||||
|
||||
@@ -61,8 +61,6 @@ function fakeRunner(
|
||||
opts: {
|
||||
screenshotFails?: boolean;
|
||||
screenshotNotFound?: boolean;
|
||||
stopNotFound?: boolean;
|
||||
stopFails?: boolean;
|
||||
killCount?: number;
|
||||
screenshotBytes?: () => Uint8Array;
|
||||
} = {},
|
||||
@@ -74,21 +72,6 @@ function fakeRunner(
|
||||
killed,
|
||||
async run(args: string[], options: BskRunOptions = {}): Promise<BskRunResult> {
|
||||
calls.push({ args, options });
|
||||
if (args[0] === "session" && args[1] === "stop") {
|
||||
if (opts.stopNotFound) {
|
||||
return {
|
||||
code: 4,
|
||||
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
if (opts.stopFails) {
|
||||
return { code: 1, stdout: "", stderr: "boom", timedOut: false, aborted: false };
|
||||
}
|
||||
return { code: 0, stdout: "{}", stderr: "", timedOut: false, aborted: false };
|
||||
}
|
||||
if (args[0] === "screenshot") {
|
||||
if (opts.screenshotNotFound) {
|
||||
return {
|
||||
@@ -197,23 +180,6 @@ describe("state machine", () => {
|
||||
expect(events[events.length - 1].type).toBe("reset");
|
||||
});
|
||||
|
||||
it("stamps the registry-recorded DSH owners onto new entries", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
own(registry, "s2");
|
||||
registry.trackOwner("s1", ["conv-a", "root"]);
|
||||
const { service } = setup({ registry });
|
||||
service.addSession("s1");
|
||||
service.addSession("s2");
|
||||
expect(service.getState()).toEqual([
|
||||
{ sessionId: "s1", action: "idle", since: 1_000_000, dshSessionIds: ["conv-a", "root"] },
|
||||
{ sessionId: "s2", action: "idle", since: 1_000_000 },
|
||||
]);
|
||||
// The owner stamp survives later upserts (action/url/frame churn).
|
||||
service.beginAction("s1", "clicking");
|
||||
expect(service.getState()[0].dshSessionIds).toEqual(["conv-a", "root"]);
|
||||
});
|
||||
|
||||
it("ignores instrumentation for unknown sessions (owned-only by construction)", () => {
|
||||
const { service } = setup({});
|
||||
service.beginAction("foreign", "clicking");
|
||||
@@ -341,57 +307,6 @@ describe("interrupt routing", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("stopSession", () => {
|
||||
it("stops an owned session: kills in-flight tools, runs session stop, removes the entry", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner();
|
||||
const { service, events } = setup({ registry, runner });
|
||||
service.addSession("s1");
|
||||
expect(registry.isOwned("s1")).toBe(true);
|
||||
|
||||
await expect(service.stopSession("s1")).resolves.toBe(true);
|
||||
expect(runner.killed).toEqual(["s1"]);
|
||||
const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
|
||||
expect(stop?.args).toEqual(["session", "stop", "s1"]);
|
||||
expect(registry.isOwned("s1")).toBe(false);
|
||||
expect(service.getState()).toEqual([]);
|
||||
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
|
||||
});
|
||||
|
||||
it("refuses foreign sessions without touching the runner", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ registry, runner });
|
||||
await expect(service.stopSession("foreign")).resolves.toBe(false);
|
||||
expect(runner.killed).toEqual([]);
|
||||
expect(runner.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("stops idempotently when the daemon already forgot the session (dead entries)", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner({ stopNotFound: true });
|
||||
const { service, events } = setup({ registry, runner });
|
||||
service.addSession("s1");
|
||||
await expect(service.stopSession("s1")).resolves.toBe(true);
|
||||
expect(registry.isOwned("s1")).toBe(false);
|
||||
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
|
||||
});
|
||||
|
||||
it("returns false and keeps the entry when the stop itself fails", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner({ stopFails: true });
|
||||
const { service } = setup({ registry, runner });
|
||||
service.addSession("s1");
|
||||
await expect(service.stopSession("s1")).resolves.toBe(false);
|
||||
expect(registry.isOwned("s1")).toBe(true);
|
||||
expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HTTP/SSE interface", () => {
|
||||
interface RecordedRoute {
|
||||
path: string;
|
||||
@@ -490,45 +405,11 @@ describe("HTTP/SSE interface", () => {
|
||||
await interruptRoute?.handler(postReq, interruptRes.res as never);
|
||||
expect(JSON.parse(interruptRes.res.body())).toEqual({ interrupted: true });
|
||||
|
||||
// stop (POST {sessionId}): owned session stops and leaves the state
|
||||
const stopRoute = routes.get("/bsk-observation/stop");
|
||||
const stopRes = fakeRes();
|
||||
await stopRoute?.handler(postReq, stopRes.res as never);
|
||||
// The stop is async behind the response-less handler — poll the state.
|
||||
await waitFor(() => service.getState().length === 0);
|
||||
expect(JSON.parse(stopRes.res.body())).toEqual({ stopped: true });
|
||||
|
||||
dispose();
|
||||
expect(routes.size).toBe(0);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("refuses a stop without a sessionId", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const { service } = setup({ registry });
|
||||
const { routes, webServer } = routeHarness();
|
||||
const ctx = { get: (key: string) => (key === "webServer" ? webServer : undefined) } as never;
|
||||
const dispose = registerObservationRoutes(ctx, service);
|
||||
|
||||
const stopRoute = routes.get("/bsk-observation/stop");
|
||||
const res = fakeRes();
|
||||
const emptyReq = fakeReq({
|
||||
method: "POST",
|
||||
headers: { host: "127.0.0.1:3999", "content-type": "application/json" },
|
||||
on: (event: string, fn: (chunk?: string) => void) => {
|
||||
if (event === "data") fn("{}");
|
||||
if (event === "end") fn();
|
||||
},
|
||||
});
|
||||
await stopRoute?.handler(emptyReq, res.res as never);
|
||||
expect(res.res.status).toBe(400);
|
||||
expect(registry.isOwned("s1")).toBe(true);
|
||||
|
||||
dispose();
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("fences off non-loopback, cross-site, and simple-request traffic", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
|
||||
@@ -29,6 +29,66 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("renders visual surfaces as screenshot-only refs", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 2,
|
||||
backendNodeId: 99,
|
||||
frameId: "child",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
label: "Data grid",
|
||||
memberCount: 2,
|
||||
},
|
||||
];
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
backendNodeId: 99,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("renders an unnamed visual surface with a stable generic label", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
name: "canvas visual surface",
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
@@ -15,6 +15,8 @@ export type {
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomRef,
|
||||
VomRefCapability,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
VomOptions,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
@@ -737,6 +738,53 @@ interface RenderState {
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
visualSurfaces: Map<number | null, VomVisualSurface[]>;
|
||||
}
|
||||
|
||||
function groupedByParent<T extends { parentId: number | null }>(
|
||||
items: T[],
|
||||
): Map<number | null, T[]> {
|
||||
const grouped = new Map<number | null, T[]>();
|
||||
for (const item of items) {
|
||||
const siblings = grouped.get(item.parentId) ?? [];
|
||||
siblings.push(item);
|
||||
grouped.set(item.parentId, siblings);
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
||||
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
|
||||
if (state.stopped) return;
|
||||
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
|
||||
if (depth > state.maxDepth) {
|
||||
state.truncated = true;
|
||||
continue;
|
||||
}
|
||||
const ref = `e${state.nextRef}`;
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; use: bsk screenshot --ref @${ref}]`;
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
state.refs.push({
|
||||
ref,
|
||||
backendNodeId: surface.backendNodeId,
|
||||
frameId: surface.frameId,
|
||||
visibleRect: surface.visibleRect,
|
||||
role: "surface",
|
||||
name: label,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
||||
@@ -812,6 +860,8 @@ function renderTree(
|
||||
...(node.role ? { role: node.role } : {}),
|
||||
...(name ? { name } : {}),
|
||||
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
@@ -821,10 +871,12 @@ function renderTree(
|
||||
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
||||
|
||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||
emitVisualEntries(node.id, depth + 1, state);
|
||||
continue;
|
||||
}
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
}
|
||||
emitVisualEntries(parentId, depth, state);
|
||||
}
|
||||
|
||||
function renderNodes(
|
||||
@@ -833,6 +885,7 @@ function renderNodes(
|
||||
initialLines: string[],
|
||||
surfaces: CondSurface[] = [],
|
||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||
visualSurfaces: VomVisualSurface[] = [],
|
||||
): RenderState {
|
||||
const children = buildChildren(nodes);
|
||||
const state: RenderState = {
|
||||
@@ -851,6 +904,7 @@ function renderNodes(
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
visualSurfaces: groupedByParent(visualSurfaces),
|
||||
};
|
||||
|
||||
renderTree(children, null, 1, state);
|
||||
@@ -1056,7 +1110,16 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
"@layers 2 focus=L1",
|
||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||
];
|
||||
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
|
||||
const visibleSurface = (parentId: number | null): boolean =>
|
||||
parentId === null || included.has(parentId);
|
||||
const state = renderNodes(
|
||||
visibleNodes,
|
||||
options,
|
||||
header,
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||
);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
return {
|
||||
@@ -1084,6 +1147,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
],
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces,
|
||||
);
|
||||
|
||||
return {
|
||||
|
||||
@@ -61,6 +61,8 @@ export interface VomScene {
|
||||
rootFrameId?: string;
|
||||
surfaces?: CondSurface[];
|
||||
activeScopeBlocks?: ActiveScopeBlock[];
|
||||
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
|
||||
visualSurfaces?: VomVisualSurface[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
@@ -94,6 +96,22 @@ export interface VomRef {
|
||||
ref: string;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
/** Observation-time top-viewport crop for screenshot-only surfaces. */
|
||||
visibleRect?: Rect;
|
||||
kind?: "dom" | "surface";
|
||||
capabilities?: VomRefCapability[];
|
||||
}
|
||||
|
||||
export type VomRefCapability = "interact" | "screenshot";
|
||||
|
||||
export interface VomVisualSurface {
|
||||
parentId: number | null;
|
||||
backendNodeId: number;
|
||||
frameId: string;
|
||||
renderingKind: "canvas";
|
||||
visibleRect: Rect;
|
||||
label?: string;
|
||||
memberCount: number;
|
||||
}
|
||||
|
||||
export interface RenderedRef extends VomRef {
|
||||
|
||||
@@ -80,6 +80,8 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session <id>` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
Reference in New Issue
Block a user