mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
918cd4670c | ||
|
|
88da6ca8b8 | ||
|
|
4dfa7aef63 | ||
|
|
0ec0b1d227 | ||
|
|
f33e1a7876 | ||
|
|
6072db8c11 | ||
|
|
ea05ad1f3d | ||
|
|
7701282897 | ||
|
|
7aaa5a2d01 | ||
|
|
210b21b637 | ||
|
|
48f64ec0df | ||
|
|
1a70d2200e | ||
|
|
554861d6b1 | ||
|
|
a4b6a61d29 | ||
|
|
cad0b5cbc9 |
No files matched your search
@@ -13,6 +13,8 @@ describe("RefStore", () => {
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
generation: 0,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
expect(s.size()).toBe(1);
|
||||
expect(s.isEmpty()).toBe(false);
|
||||
@@ -48,4 +50,31 @@ describe("RefStore", () => {
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit capabilities for visual surface refs", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
});
|
||||
|
||||
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, { tabId: 7, kind: "surface" });
|
||||
s.set("e2", 43, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
||||
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
|
||||
|
||||
function input(): SurfaceCaptureInput {
|
||||
return {
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "main",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: 2,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 100,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
};
|
||||
}
|
||||
|
||||
describe("SurfaceCaptureStore", () => {
|
||||
it("creates metadata-only, short-lived captures and consumes them once", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 500,
|
||||
createId: () => "sc_test",
|
||||
});
|
||||
const capture = store.create(input());
|
||||
|
||||
expect(capture).toMatchObject({
|
||||
id: "sc_test",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 1_500,
|
||||
consumed: false,
|
||||
});
|
||||
expect(capture).not.toHaveProperty("imageBase64");
|
||||
expect(store.consume("sc_test")).toMatchObject({ ok: true });
|
||||
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
|
||||
|
||||
now = 1_600;
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects expired and unknown captures", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 10,
|
||||
createId: () => "sc_expired",
|
||||
});
|
||||
store.create(input());
|
||||
now = 1_010;
|
||||
|
||||
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
|
||||
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
|
||||
});
|
||||
|
||||
it("clears every capture on session cleanup", () => {
|
||||
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
|
||||
store.create(input());
|
||||
store.clear();
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,12 @@
|
||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||
import { RefStore } from "./ref-store";
|
||||
import { SurfaceCaptureStore } from "./surface-capture-store";
|
||||
|
||||
export interface SessionContext {
|
||||
sessionId: string;
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
surfaceCaptures: SurfaceCaptureStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
@@ -186,6 +188,7 @@ export class SessionManager {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
|
||||
borrowedTabs: new Map(),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
@@ -12,12 +14,17 @@
|
||||
* inside the `SessionContext`.
|
||||
*/
|
||||
export type BackendNodeId = number;
|
||||
export type RefCapability = "interact" | "screenshot";
|
||||
export type RefTargetKind = "dom" | "surface";
|
||||
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
@@ -28,8 +35,19 @@ export type RefInput =
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
};
|
||||
|
||||
function capabilitiesFor(
|
||||
kind: RefTargetKind,
|
||||
capabilities: RefCapability[] | undefined,
|
||||
): RefCapability[] {
|
||||
if (kind === "surface") return ["screenshot"];
|
||||
return capabilities ?? ["interact", "screenshot"];
|
||||
}
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
private generation = 0;
|
||||
@@ -70,13 +88,20 @@ export class RefStore {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
} = {},
|
||||
): void {
|
||||
const kind = opts.kind ?? "dom";
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, opts.capabilities),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -94,14 +119,20 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input,
|
||||
tabId: null,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
const kind = input.kind ?? "dom";
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, input.capabilities),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export const SURFACE_CAPTURE_TTL_MS = 30_000;
|
||||
|
||||
export interface SurfaceCapture {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
tabId: number;
|
||||
navigationIdentity: string;
|
||||
surface: {
|
||||
ref: string;
|
||||
frameId?: string;
|
||||
backendNodeId: number;
|
||||
observationGeneration: number;
|
||||
};
|
||||
topViewportRect: Rect;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
createdAt: number;
|
||||
expiresAt: number;
|
||||
consumed: boolean;
|
||||
}
|
||||
|
||||
export type SurfaceCaptureInput = Omit<
|
||||
SurfaceCapture,
|
||||
"id" | "createdAt" | "expiresAt" | "consumed"
|
||||
>;
|
||||
|
||||
export type SurfaceCaptureConsumeResult =
|
||||
| { ok: true; capture: SurfaceCapture }
|
||||
| { ok: false; reason: "not_found" | "expired" | "consumed" };
|
||||
|
||||
export interface SurfaceCaptureStoreOptions {
|
||||
now?: () => number;
|
||||
ttlMs?: number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
function randomCaptureId(): string {
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(12));
|
||||
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
||||
}
|
||||
|
||||
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
|
||||
export class SurfaceCaptureStore {
|
||||
private readonly captures = new Map<string, SurfaceCapture>();
|
||||
private readonly now: () => number;
|
||||
private readonly ttlMs: number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(options: SurfaceCaptureStoreOptions = {}) {
|
||||
this.now = options.now ?? Date.now;
|
||||
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
|
||||
this.createId = options.createId ?? randomCaptureId;
|
||||
}
|
||||
|
||||
create(input: SurfaceCaptureInput): SurfaceCapture {
|
||||
this.purgeExpired();
|
||||
const createdAt = this.now();
|
||||
const capture: SurfaceCapture = {
|
||||
...input,
|
||||
id: this.createId(),
|
||||
createdAt,
|
||||
expiresAt: createdAt + this.ttlMs,
|
||||
consumed: false,
|
||||
};
|
||||
this.captures.set(capture.id, capture);
|
||||
return capture;
|
||||
}
|
||||
|
||||
consume(id: string): SurfaceCaptureConsumeResult {
|
||||
const capture = this.captures.get(id);
|
||||
if (!capture) return { ok: false, reason: "not_found" };
|
||||
if (capture.expiresAt <= this.now()) {
|
||||
this.captures.delete(id);
|
||||
return { ok: false, reason: "expired" };
|
||||
}
|
||||
if (capture.consumed) return { ok: false, reason: "consumed" };
|
||||
capture.consumed = true;
|
||||
return { ok: true, capture };
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.captures.clear();
|
||||
}
|
||||
|
||||
size(): number {
|
||||
this.purgeExpired();
|
||||
return this.captures.size;
|
||||
}
|
||||
|
||||
private purgeExpired(): void {
|
||||
const now = this.now();
|
||||
for (const [id, capture] of this.captures) {
|
||||
if (capture.expiresAt <= now) this.captures.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
parseKeySpec,
|
||||
resolveKeyDescriptor,
|
||||
} from "../interaction";
|
||||
import { captureSurfaceEnvironment } from "../surface-target";
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
@@ -33,6 +34,15 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (
|
||||
!h &&
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression ?? "").includes(
|
||||
"bsk-focus-target-probe",
|
||||
)
|
||||
) {
|
||||
return { result: { value: { hasFocus: true, activeTag: "input" } } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -44,6 +54,10 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
getFrameGraph: vi.fn(async (tabId) => ({
|
||||
rootFrameId: "main",
|
||||
frames: [{ frameId: "main", target: { tabId } }],
|
||||
})),
|
||||
};
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
@@ -54,6 +68,71 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
return { cdp, tabsApi, sent };
|
||||
}
|
||||
|
||||
function fillRuntimeReadback(value: string) {
|
||||
return (params: unknown) => {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-input-readback")) {
|
||||
return { result: { value: { supported: true, value } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
};
|
||||
}
|
||||
|
||||
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Page.getFrameTree": () => ({
|
||||
frameTree: {
|
||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
||||
},
|
||||
}),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
scale: 1,
|
||||
},
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { overlayHostPresent: false, overlayHostConnected: false } },
|
||||
}),
|
||||
"Input.dispatchMouseEvent": (params) => {
|
||||
onMouse?.(params as Record<string, unknown>);
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const environment = await captureSurfaceEnvironment(fake.cdp, 4, undefined);
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing Surface ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
return { manager, context, fake, capture };
|
||||
}
|
||||
|
||||
describe("modifiersBitfield", () => {
|
||||
it("matches CDP's expected bit layout", () => {
|
||||
expect(modifiersBitfield([])).toBe(0);
|
||||
@@ -123,6 +202,29 @@ describe("handleClick", () => {
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||
});
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -167,6 +269,74 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
"left",
|
||||
"middle",
|
||||
"right",
|
||||
] as const)("uses the common click executor for a %s-button Surface click", async (button) => {
|
||||
const { manager, fake, capture } = await makeSurfaceClickFixture();
|
||||
const result = await handleClick(
|
||||
manager,
|
||||
{
|
||||
session_id: "aa11",
|
||||
ref: "@e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
button,
|
||||
click_count: 2,
|
||||
modifiers: ["ctrl", "shift"],
|
||||
},
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
used_ref: "e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
x: 35,
|
||||
y: 30,
|
||||
});
|
||||
const mouse = fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent");
|
||||
expect(mouse).toHaveLength(3);
|
||||
expect(mouse[1].params).toMatchObject({
|
||||
type: "mousePressed",
|
||||
button,
|
||||
clickCount: 2,
|
||||
modifiers: 2 | 8,
|
||||
});
|
||||
expect(mouse[2].params).toMatchObject({
|
||||
type: "mouseReleased",
|
||||
button,
|
||||
clickCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("consumes a Surface capture when the common click executor fails", async () => {
|
||||
let mouseCalls = 0;
|
||||
const { manager, fake, capture } = await makeSurfaceClickFixture((params) => {
|
||||
mouseCalls += 1;
|
||||
if (params.type === "mousePressed") throw new Error("input failed");
|
||||
});
|
||||
const params = {
|
||||
session_id: "aa11",
|
||||
ref: "@e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
};
|
||||
|
||||
expect(
|
||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
||||
).toMatchObject({ code: "cdp_failed" });
|
||||
expect(mouseCalls).toBe(2);
|
||||
expect(
|
||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -586,7 +756,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback("hello"),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -619,6 +789,9 @@ describe("handleFill", () => {
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (fn.includes("bsk-input-readback")) {
|
||||
return { result: { value: { supported: true, value: "existingx" } } };
|
||||
}
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
@@ -632,7 +805,7 @@ describe("handleFill", () => {
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.value_length).toBe(1);
|
||||
expect(res.value_length).toBe(9);
|
||||
});
|
||||
|
||||
it("treats contenteditable=true as fillable", async () => {
|
||||
@@ -650,7 +823,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback("rich"),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -661,6 +834,88 @@ describe("handleFill", () => {
|
||||
expect("code" in res).toBe(false);
|
||||
});
|
||||
|
||||
it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
childCalls.push(method);
|
||||
if (method === "DOM.describeNode") {
|
||||
return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
|
||||
}
|
||||
if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
|
||||
if (method === "Input.insertText") {
|
||||
expect(params).toEqual({ text: "中文输入" });
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
return fn.includes("bsk-input-readback")
|
||||
? { result: { value: { supported: true, value: "中文输入" } } }
|
||||
: { result: { type: "undefined" } };
|
||||
}
|
||||
throw new Error(`unexpected child call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleFill(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", value: "中文输入" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(JSON.stringify(res));
|
||||
expect(res.value_length).toBe(4);
|
||||
expect(childCalls).toContain("DOM.focus");
|
||||
expect(childCalls).toContain("Input.insertText");
|
||||
expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
|
||||
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns input_not_applied when a controlled input resets the value", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 8, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback(""),
|
||||
});
|
||||
|
||||
const res = await handleFill(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", value: "rejected" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -855,6 +1110,125 @@ describe("handlePress", () => {
|
||||
expect(res.key).toBe("Enter");
|
||||
});
|
||||
|
||||
it("routes ref-targeted key events through the OOPIF session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 555, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "Enter", ref: "@e1" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expectPressOk(res);
|
||||
expect(childCalls).toEqual([
|
||||
"DOM.scrollIntoViewIfNeeded",
|
||||
"DOM.focus",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "iframe" } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "a" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expectPressOk(res);
|
||||
expect(childCalls).toEqual([
|
||||
"Runtime.evaluate",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
});
|
||||
|
||||
it("refuses an untargeted press when focus ownership is ambiguous", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "input" } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
|
||||
],
|
||||
}));
|
||||
fake.cdp.sendToTarget = vi.fn(async () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "textarea" } },
|
||||
})) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "Enter" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "focus_target_unresolved", candidate_count: 2 },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before focus when abort fires after press scroll", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { dispatchMouseClick } from "../mouse-input";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
function fakeCdp(onSend?: (method: string, params: Record<string, unknown>) => void): {
|
||||
cdp: CdpRunner;
|
||||
events: Record<string, unknown>[];
|
||||
} {
|
||||
const events: Record<string, unknown>[] = [];
|
||||
return {
|
||||
events,
|
||||
cdp: {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
const event = params as Record<string, unknown>;
|
||||
events.push(event);
|
||||
onSend?.(method, event);
|
||||
return {} as never;
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("dispatchMouseClick", () => {
|
||||
it("emits one move, press, and release sequence", async () => {
|
||||
const { cdp, events } = fakeCdp();
|
||||
|
||||
expect(
|
||||
await dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 12, y: 34 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
),
|
||||
).toBe("completed");
|
||||
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "mouseMoved",
|
||||
x: 12,
|
||||
y: 34,
|
||||
modifiers: 0,
|
||||
},
|
||||
{
|
||||
type: "mousePressed",
|
||||
x: 12,
|
||||
y: 34,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
},
|
||||
{
|
||||
type: "mouseReleased",
|
||||
x: 12,
|
||||
y: 34,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("releases a pressed button when cancellation arrives", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, events } = fakeCdp((_method, event) => {
|
||||
if (event.type === "mousePressed") controller.abort();
|
||||
});
|
||||
|
||||
expect(
|
||||
await dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 1, y: 2 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
signal: controller.signal,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
),
|
||||
).toBe("cancelled");
|
||||
expect(events.at(-1)).toMatchObject({ type: "mouseReleased" });
|
||||
});
|
||||
|
||||
it("settles pointer-move hit testing before pressing", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const { cdp, events } = fakeCdp();
|
||||
const click = dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 1, y: 2 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
moveSettleMs: 32,
|
||||
},
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(31);
|
||||
expect(events.map((event) => event.type)).toEqual(["mouseMoved"]);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await click;
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"mouseMoved",
|
||||
"mousePressed",
|
||||
"mouseReleased",
|
||||
]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -67,6 +67,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler && method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -300,6 +303,82 @@ describe("handleScreenshot", () => {
|
||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
||||
});
|
||||
|
||||
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { width: number; height: number; scale: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
|
||||
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
|
||||
expect(res.capture).toMatchObject({
|
||||
surface_ref: "@e5",
|
||||
coordinate_space: "capture-image-pixel",
|
||||
});
|
||||
expect(ctx.surfaceCaptures.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("crops a visual surface screenshot to its observation-time visible region", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
||||
expect(res.capture?.id).toMatch(/^sc_/);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -2693,6 +2772,85 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Canvas app" },
|
||||
backendDOMNodeId: 100,
|
||||
};
|
||||
const strings = ["body", "canvas", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("canvas")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 30, 800, 500],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
conditionalSurfaceProbe: false,
|
||||
};
|
||||
|
||||
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
|
||||
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
|
||||
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 200,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
|
||||
expect(snapshot.text).not.toContain("visual surface");
|
||||
expect(snapshot.ref_count).toBe(0);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
|
||||
@@ -252,12 +252,23 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clears the RefStore before window teardown", async () => {
|
||||
it("clears refs and Surface captures before window teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Insert a fake ref so we can verify clear() ran.
|
||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||
ctx.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 },
|
||||
topViewportRect: { x: 0, y: 0, w: 100, h: 50 },
|
||||
imageWidth: 100,
|
||||
imageHeight: 50,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
});
|
||||
const state: FakeState = {
|
||||
tabs: new Map(),
|
||||
windowsClosed: new Set(),
|
||||
@@ -266,5 +277,6 @@ describe("handleSessionStop with auto-return", () => {
|
||||
const { tabs, windows } = makeApis(state);
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
expect(ctx.surfaceCaptures.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -24,10 +24,16 @@ describe("lookupSnapshotRef", () => {
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,6 +59,8 @@ describe("resolveSnapshotRef", () => {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
const expected = {
|
||||
@@ -60,6 +68,9 @@ describe("resolveSnapshotRef", () => {
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom" as const,
|
||||
capabilities: ["interact", "screenshot"] as const,
|
||||
generation: 0,
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
@@ -105,6 +116,29 @@ describe("resolveSnapshotRef", () => {
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an operation outside the ref's declared capabilities", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
|
||||
code: "permission_denied",
|
||||
message: "ref @e3 does not support interact",
|
||||
data: {
|
||||
reason: "ref_capability_denied",
|
||||
required_capability: "interact",
|
||||
kind: "surface",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
mapImagePointToViewport,
|
||||
pointInRegion,
|
||||
sameRect,
|
||||
surfaceVisibleRect,
|
||||
} from "../surface-coordinate";
|
||||
|
||||
describe("surface coordinate mapping", () => {
|
||||
it("maps capture image pixels through the actual screenshot dimensions", () => {
|
||||
expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({
|
||||
x: 200,
|
||||
y: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects invalid image coordinates and dimensions", () => {
|
||||
const rect = { x: 0, y: 0, w: 100, h: 50 };
|
||||
expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull();
|
||||
});
|
||||
|
||||
it("intersects live and observed visible regions and compares them strictly", () => {
|
||||
const clipped = surfaceVisibleRect(
|
||||
{ x: 0, y: 0, width: 200, height: 100 },
|
||||
{ x: 25, y: 30, w: 50, h: 40 },
|
||||
);
|
||||
expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 });
|
||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 25, y: 30, w: 50, h: 40 })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 26, y: 30, w: 50, h: 40 })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("checks the projected visible region rather than only its bounding box", () => {
|
||||
const triangle = [
|
||||
[
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 100, y: 0 },
|
||||
{ x: 0, y: 100 },
|
||||
],
|
||||
];
|
||||
expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true);
|
||||
expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,306 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import {
|
||||
captureSurfaceEnvironment,
|
||||
resolveSurfacePointerTarget,
|
||||
SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
} from "../surface-target";
|
||||
|
||||
function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeBrowser() {
|
||||
let loaderId = "loader-1";
|
||||
let pageY = 0;
|
||||
let quad = [10, 20, 110, 20, 110, 60, 10, 60];
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Page.getFrameTree") {
|
||||
return {
|
||||
frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } },
|
||||
} as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY,
|
||||
scale: 1,
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "DOM.getContentQuads") return { quads: [quad] } as never;
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
return {
|
||||
cdp,
|
||||
sent,
|
||||
setLoaderId: (value: string) => {
|
||||
loaderId = value;
|
||||
},
|
||||
setPageY: (value: number) => {
|
||||
pageY = value;
|
||||
},
|
||||
setQuad: (value: number[]) => {
|
||||
quad = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function setupCapture() {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
const browser = fakeBrowser();
|
||||
const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined);
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
return { manager, context, browser, capture };
|
||||
}
|
||||
|
||||
function pointParams(captureId: string) {
|
||||
return {
|
||||
ref: "@e3",
|
||||
captureId,
|
||||
imageX: 50,
|
||||
imageY: 20,
|
||||
};
|
||||
}
|
||||
|
||||
describe("Surface screenshot-bound pointer target", () => {
|
||||
it("maps one fresh capture coordinate without executing an action", async () => {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
usedRef: "e3",
|
||||
point: { x: 35, y: 30 },
|
||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
capture: { id: capture.id, imageX: 50, imageY: 20 },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
|
||||
const repeated = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(repeated).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_consumed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a new observation generation before sending input", async () => {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
context.refStore.replace([
|
||||
[
|
||||
"e3",
|
||||
{
|
||||
backendNodeId: 99,
|
||||
tabId: 4,
|
||||
kind: "surface" as const,
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_stale" },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects navigation, scroll, and visible geometry changes", async () => {
|
||||
for (const mutate of [
|
||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setLoaderId("loader-2"),
|
||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setPageY(20),
|
||||
(browser: ReturnType<typeof fakeBrowser>) =>
|
||||
browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]),
|
||||
]) {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
mutate(browser);
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_stale" },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("consumes the capture when coordinate validation fails", async () => {
|
||||
const invalid = await setupCapture();
|
||||
const outside = await resolveSurfacePointerTarget(
|
||||
invalid.context,
|
||||
{ tabId: 4 },
|
||||
{ ...pointParams(invalid.capture.id), imageX: 200 },
|
||||
{ cdp: invalid.browser.cdp },
|
||||
);
|
||||
expect(outside).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "surface_coordinate_invalid" },
|
||||
});
|
||||
expect(
|
||||
await resolveSurfacePointerTarget(
|
||||
invalid.context,
|
||||
{ tabId: 4 },
|
||||
pointParams(invalid.capture.id),
|
||||
{ cdp: invalid.browser.cdp },
|
||||
),
|
||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
||||
});
|
||||
|
||||
it("maps image coordinates through an OOPIF frame projection", async () => {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "surface",
|
||||
visibleRect: { x: 120, y: 110, w: 100, h: 40 },
|
||||
});
|
||||
const graph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 77,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Page.getFrameTree") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
scale: 1,
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never;
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {} as never;
|
||||
throw new Error(`unexpected root CDP call ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never;
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => graph),
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
const environment = await captureSurfaceEnvironment(cdp, 4, "child");
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "child",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 120, y: 110, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
{
|
||||
ref: "@e3",
|
||||
captureId: capture.id,
|
||||
imageX: 100,
|
||||
imageY: 40,
|
||||
},
|
||||
{ cdp },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({ point: { x: 170, y: 130 } });
|
||||
expect(sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,7 @@
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -32,6 +32,7 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import { dispatchMouseClick } from "./mouse-input";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -43,6 +44,7 @@ import {
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||
|
||||
export interface InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
@@ -159,7 +161,7 @@ async function resolveBackendNode(
|
||||
};
|
||||
}
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
@@ -215,15 +217,131 @@ async function resolveBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
interface InputDispatchContext {
|
||||
target: CdpTarget;
|
||||
cdp: CdpRunner;
|
||||
backendNodeId?: number;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
|
||||
function inputContextForNode(
|
||||
cdp: CdpRunner,
|
||||
node: Awaited<ReturnType<typeof resolveBackendNode>>,
|
||||
): (InputDispatchContext & { backendNodeId: number }) | RpcError {
|
||||
if (isRpcError(node)) return node;
|
||||
return {
|
||||
target: node.cdpTarget,
|
||||
cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
|
||||
backendNodeId: node.backendNodeId,
|
||||
frameId: node.frameId,
|
||||
usedRef: node.usedRef,
|
||||
usedSelector: node.usedSelector,
|
||||
};
|
||||
}
|
||||
|
||||
interface FocusProbe {
|
||||
hasFocus: boolean;
|
||||
activeTag: string;
|
||||
}
|
||||
|
||||
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
||||
async function resolveFocusedInputContext(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<InputDispatchContext | RpcError> {
|
||||
if (!cdp.getFrameGraph) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
"press cannot determine the focused frame target",
|
||||
);
|
||||
}
|
||||
|
||||
let targets: CdpTarget[];
|
||||
try {
|
||||
const graph = await cdp.getFrameGraph(tabId);
|
||||
const unique = new Map<string, CdpTarget>();
|
||||
unique.set(cdpTargetKey({ tabId }), { tabId });
|
||||
for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
|
||||
targets = [...unique.values()];
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
|
||||
await Promise.all(
|
||||
targets.map(async (candidate) => {
|
||||
try {
|
||||
const reply = await cdpRunnerForTarget(cdp, candidate).send<{
|
||||
result?: { value?: FocusProbe };
|
||||
}>(tabId, "Runtime.evaluate", {
|
||||
expression: `(() => {
|
||||
/* bsk-focus-target-probe */
|
||||
const active = document.activeElement;
|
||||
return {
|
||||
hasFocus: document.hasFocus(),
|
||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
|
||||
};
|
||||
})()`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const probe = reply.result?.value;
|
||||
if (probe?.hasFocus) focused.push({ target: candidate, probe });
|
||||
} catch {
|
||||
// A target may navigate or detach while the graph is being probed.
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// A top document whose active element is an iframe delegates focus to
|
||||
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
||||
// target while same-process frames continue to use the shared root target.
|
||||
const owners = focused.filter(
|
||||
({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
|
||||
);
|
||||
if (owners.length !== 1) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
owners.length === 0
|
||||
? "press found no unique focused frame target"
|
||||
: "press found multiple focused frame targets",
|
||||
{ candidate_count: owners.length },
|
||||
);
|
||||
}
|
||||
const target = owners[0].target;
|
||||
return { target, cdp: cdpRunnerForTarget(cdp, target) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface ResolvedClickPointerTarget {
|
||||
point: { x: number; y: number };
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
moveSettleMs: number;
|
||||
capture?: {
|
||||
id: string;
|
||||
imageX: number;
|
||||
imageY: number;
|
||||
};
|
||||
}
|
||||
|
||||
export async function handleClick(
|
||||
manager: SessionManager,
|
||||
params: ClickParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const hasSurfacePointParams =
|
||||
params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined;
|
||||
const ctxOrErr = lookupSession(manager, params, "click");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -235,39 +353,67 @@ export async function handleClick(
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
const button: MouseButton = params.button ?? "left";
|
||||
const clickCount = params.click_count ?? 1;
|
||||
if (!Number.isSafeInteger(clickCount) || clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be a positive integer" };
|
||||
}
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
const pointerTarget: ResolvedClickPointerTarget | RpcError = hasSurfacePointParams
|
||||
? await resolveSurfacePointerTarget(
|
||||
ctx,
|
||||
target,
|
||||
{
|
||||
ref: params.ref,
|
||||
selector: params.selector,
|
||||
captureId: params.capture_id,
|
||||
imageX: params.image_x,
|
||||
imageY: params.image_y,
|
||||
},
|
||||
{
|
||||
cdp: deps.cdp,
|
||||
signal: deps.signal,
|
||||
},
|
||||
)
|
||||
: await (async () => {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
return {
|
||||
point: geometry.actionPoint,
|
||||
usedRef: node.usedRef,
|
||||
usedSelector: node.usedSelector,
|
||||
moveSettleMs: 0,
|
||||
};
|
||||
})();
|
||||
if (isRpcError(pointerTarget)) return pointerTarget;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
const button: MouseButton = params.button ?? "left";
|
||||
const clickCount = params.click_count ?? 1;
|
||||
if (clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be greater than zero" };
|
||||
}
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
const overlayBlocking = await checkOverlayAtPoint(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
pointerTarget.point.x,
|
||||
pointerTarget.point.y,
|
||||
);
|
||||
let automationBypassEnabled = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
@@ -279,47 +425,16 @@ export async function handleClick(
|
||||
}
|
||||
|
||||
try {
|
||||
// Move first so hover state activates, then press → release.
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
modifiers,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, {
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
signal: deps.signal,
|
||||
moveSettleMs: pointerTarget.moveSettleMs,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err);
|
||||
}
|
||||
if (dispatch === "cancelled") {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -337,10 +452,17 @@ export async function handleClick(
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
used_ref: pointerTarget.usedRef,
|
||||
used_selector: pointerTarget.usedSelector,
|
||||
x: pointerTarget.point.x,
|
||||
y: pointerTarget.point.y,
|
||||
...(pointerTarget.capture
|
||||
? {
|
||||
capture_id: pointerTarget.capture.id,
|
||||
image_x: pointerTarget.capture.imageX,
|
||||
image_y: pointerTarget.capture.imageY,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -598,8 +720,9 @@ export async function handleFill(
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
const input = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(input)) return input;
|
||||
const nodeCdp = input.cdp;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -607,7 +730,7 @@ export async function handleFill(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
backendNodeId: node.backendNodeId,
|
||||
backendNodeId: input.backendNodeId,
|
||||
},
|
||||
);
|
||||
if (!described.node || !isFillable(described.node)) {
|
||||
@@ -620,15 +743,15 @@ export async function handleFill(
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
input.target,
|
||||
input.backendNodeId,
|
||||
input.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -640,7 +763,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -671,7 +794,7 @@ export async function handleFill(
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
@@ -684,6 +807,28 @@ export async function handleFill(
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
// Give controlled inputs one bounded browser-render turn to accept or
|
||||
// reject the dispatched value before reporting success.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-input-sync */
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, 100);
|
||||
if (typeof requestAnimationFrame !== 'function') { finish(); return; }
|
||||
requestAnimationFrame(() => requestAnimationFrame(finish));
|
||||
});
|
||||
}`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -691,11 +836,56 @@ export async function handleFill(
|
||||
};
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
let actualValue: string;
|
||||
try {
|
||||
const readback = await nodeCdp.send<{
|
||||
result?: { value?: { supported?: boolean; value?: string } };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-input-readback */
|
||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||
return { supported: true, value: this.value };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
return { supported: true, value: this.innerText ?? this.textContent ?? '' };
|
||||
}
|
||||
return { supported: false, value: '' };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const state = readback.result?.value;
|
||||
actualValue = typeof state?.value === "string" ? state.value : "";
|
||||
const applied =
|
||||
state?.supported === true &&
|
||||
(clearBefore
|
||||
? actualValue === params.value
|
||||
: params.value.length === 0 || actualValue.includes(params.value));
|
||||
if (!applied) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the page did not retain the requested input value",
|
||||
{ expected_value_length: params.value.length, actual_value_length: actualValue.length },
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
used_ref: input.usedRef,
|
||||
used_selector: input.usedSelector,
|
||||
value_length: actualValue.length,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -870,31 +1060,38 @@ export async function handlePress(
|
||||
};
|
||||
}
|
||||
|
||||
let input: InputDispatchContext | RpcError;
|
||||
// Optional focus before key dispatch.
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(explicitInput)) return explicitInput;
|
||||
input = explicitInput;
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
explicitInput.target,
|
||||
explicitInput.backendNodeId,
|
||||
explicitInput.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||
backendNodeId: explicitInput.backendNodeId,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
} else {
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId);
|
||||
if (isRpcError(input)) return input;
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -908,7 +1105,7 @@ export async function handlePress(
|
||||
try {
|
||||
let cancelled = false;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
@@ -922,7 +1119,7 @@ export async function handlePress(
|
||||
typeof descriptor.text === "string" &&
|
||||
descriptor.text.length > 0
|
||||
) {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "char",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
@@ -939,7 +1136,7 @@ export async function handlePress(
|
||||
cancelled = true;
|
||||
}
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import type { MouseButton } from "@/transport/types";
|
||||
import type { Point } from "./geometry";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
interface MouseClickDispatchOptions {
|
||||
button: MouseButton;
|
||||
clickCount: number;
|
||||
modifiers: number;
|
||||
signal?: AbortSignal;
|
||||
moveSettleMs?: number;
|
||||
}
|
||||
|
||||
type MouseClickDispatchResult = "completed" | "cancelled";
|
||||
|
||||
function waitForSettle(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
|
||||
if (ms <= 0) return Promise.resolve(signal?.aborted !== true);
|
||||
if (signal?.aborted) return Promise.resolve(false);
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (completed: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve(completed);
|
||||
};
|
||||
const timer = setTimeout(() => finish(true), ms);
|
||||
const onAbort = () => finish(false);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch one coherent CDP mouse click in top-viewport CSS coordinates.
|
||||
*
|
||||
* Keeping the sequence here prevents DOM and Surface interaction paths from
|
||||
* duplicating cancellation and best-effort release behavior. Callers that
|
||||
* target asynchronously hit-tested content may request a settle interval.
|
||||
*/
|
||||
export async function dispatchMouseClick(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
point: Point,
|
||||
options: MouseClickDispatchOptions,
|
||||
): Promise<MouseClickDispatchResult> {
|
||||
const { button, clickCount, modifiers, signal, moveSettleMs = 0 } = options;
|
||||
let pressed = false;
|
||||
const release = () =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
modifiers,
|
||||
});
|
||||
if (!(await waitForSettle(moveSettleMs, signal))) return "cancelled";
|
||||
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
pressed = true;
|
||||
|
||||
if (signal?.aborted) {
|
||||
await release();
|
||||
pressed = false;
|
||||
return "cancelled";
|
||||
}
|
||||
|
||||
await release();
|
||||
pressed = false;
|
||||
return "completed";
|
||||
} catch (error) {
|
||||
if (pressed) {
|
||||
try {
|
||||
await release();
|
||||
} catch (releaseError) {
|
||||
console.debug("[bsk mouse-input] best-effort mouse release failed", releaseError);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
type Rect,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
@@ -47,6 +48,8 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { surfaceVisibleRect } from "./surface-coordinate";
|
||||
import { captureSurfaceEnvironment } from "./surface-target";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
@@ -59,6 +62,11 @@ import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "./vom/rendered-surfaces";
|
||||
import {
|
||||
buildSemanticGraph,
|
||||
buildSemanticVomScene,
|
||||
@@ -90,6 +98,9 @@ export const normaliseRef = sharedNormaliseRef;
|
||||
// screenshot — `tool.screenshot`
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
|
||||
|
||||
/**
|
||||
* Strip the `data:image/...;base64,` prefix from a Chrome
|
||||
* `captureVisibleTab` dataURL and return the raw base64 payload.
|
||||
@@ -182,18 +193,42 @@ async function captureElementScreenshot(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
visualSurface = false,
|
||||
observedVisibleRect?: Rect,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
): Promise<
|
||||
{ image_base64: string; width: number; height: number; topViewportRect: Rect } | RpcError
|
||||
> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: true },
|
||||
{ scrollIntoView: !visualSurface },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
const topViewportRect = surfaceVisibleRect(
|
||||
geometry.topBounds,
|
||||
visualSurface ? observedVisibleRect : undefined,
|
||||
);
|
||||
if (!topViewportRect) {
|
||||
return { code: "permission_denied", message: "surface is no longer visible" };
|
||||
}
|
||||
const rect = {
|
||||
x: topViewportRect.x,
|
||||
y: topViewportRect.y,
|
||||
width: topViewportRect.w,
|
||||
height: topViewportRect.h,
|
||||
};
|
||||
const scale = visualSurface
|
||||
? Math.min(
|
||||
1,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
|
||||
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
|
||||
)
|
||||
: 1;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
@@ -203,7 +238,7 @@ async function captureElementScreenshot(
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale: 1,
|
||||
scale,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
@@ -215,7 +250,7 @@ async function captureElementScreenshot(
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
return { image_base64, width: dims.width, height: dims.height, topViewportRect };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -310,7 +345,7 @@ export async function handleScreenshot(
|
||||
if (!deps.cdp) {
|
||||
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -330,18 +365,54 @@ export async function handleScreenshot(
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
node.kind === "surface",
|
||||
node.visibleRect,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const capture =
|
||||
node.kind === "surface"
|
||||
? await (async () => {
|
||||
const environment = await captureSurfaceEnvironment(cdp, target.tabId, node.frameId);
|
||||
if (isRpcError(environment)) return environment;
|
||||
return ctx.surfaceCaptures.create({
|
||||
sessionId: ctx.sessionId,
|
||||
tabId: target.tabId,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: node.refKey,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
backendNodeId: node.backendNodeId,
|
||||
observationGeneration: node.generation,
|
||||
},
|
||||
topViewportRect: captured.topViewportRect,
|
||||
imageWidth: captured.width,
|
||||
imageHeight: captured.height,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
})()
|
||||
: null;
|
||||
if (capture && isRpcError(capture)) return capture;
|
||||
return withShotDialogs({
|
||||
image_base64: captured.image_base64,
|
||||
width: captured.width,
|
||||
height: captured.height,
|
||||
format: "png",
|
||||
tab_id: target.tabId,
|
||||
...(capture
|
||||
? {
|
||||
capture: {
|
||||
id: capture.id,
|
||||
surface_ref: `@${capture.surface.ref}`,
|
||||
coordinate_space: "capture-image-pixel" as const,
|
||||
expires_at: capture.expiresAt,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1023,6 +1094,7 @@ async function captureForVom(
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
renderedSurfaceDiscovery?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
@@ -1064,7 +1136,20 @@ export async function captureVomObservation(
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
const observedScene = options.renderedSurfaceDiscovery
|
||||
? projectRenderedSurfaces(
|
||||
decoratedScene,
|
||||
normalizedDocuments,
|
||||
clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
normalizedDocuments,
|
||||
captured.viewport,
|
||||
captured.excludedBackendNodeIds,
|
||||
),
|
||||
),
|
||||
)
|
||||
: decoratedScene;
|
||||
const rendered = renderVom(observedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
@@ -1115,6 +1200,7 @@ async function handleVomObservation(
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
renderedSurfaceDiscovery: toolName === "observe",
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
@@ -1132,6 +1218,9 @@ async function handleVomObservation(
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
|
||||
...(ref.kind ? { kind: ref.kind } : {}),
|
||||
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
|
||||
@@ -243,6 +243,7 @@ export async function handleSessionStop(
|
||||
|
||||
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
||||
ctx.refStore.clear();
|
||||
ctx.surfaceCaptures.clear();
|
||||
|
||||
clearRecordingForSession(params.session_id);
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
|
||||
@@ -12,6 +13,10 @@ export interface SnapshotRefLookup {
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
@@ -37,6 +42,10 @@ export function lookupSnapshotRef(
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
|
||||
kind: entry.kind,
|
||||
capabilities: entry.capabilities,
|
||||
generation: entry.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -49,6 +58,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
requiredCapability?: RefCapability,
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
@@ -58,5 +68,13 @@ export function resolveSnapshotRef(
|
||||
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
|
||||
);
|
||||
}
|
||||
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"ref_capability_denied",
|
||||
`ref ${ref} does not support ${requiredCapability}`,
|
||||
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
|
||||
);
|
||||
}
|
||||
return looked;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { Point, Region, ViewportRect } from "./geometry";
|
||||
|
||||
const RECT_EPSILON = 0.01;
|
||||
|
||||
export function surfaceVisibleRect(
|
||||
liveRect: ViewportRect,
|
||||
observedRect: Rect | undefined,
|
||||
): Rect | null {
|
||||
const observed = observedRect ?? {
|
||||
x: liveRect.x,
|
||||
y: liveRect.y,
|
||||
w: liveRect.width,
|
||||
h: liveRect.height,
|
||||
};
|
||||
const x = Math.max(liveRect.x, observed.x);
|
||||
const y = Math.max(liveRect.y, observed.y);
|
||||
const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w);
|
||||
const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h);
|
||||
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
|
||||
}
|
||||
|
||||
export function sameRect(a: Rect, b: Rect): boolean {
|
||||
return (
|
||||
Math.abs(a.x - b.x) <= RECT_EPSILON &&
|
||||
Math.abs(a.y - b.y) <= RECT_EPSILON &&
|
||||
Math.abs(a.w - b.w) <= RECT_EPSILON &&
|
||||
Math.abs(a.h - b.h) <= RECT_EPSILON
|
||||
);
|
||||
}
|
||||
|
||||
export function mapImagePointToViewport(
|
||||
rect: Rect,
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
imageX: number,
|
||||
imageY: number,
|
||||
): Point | null {
|
||||
if (
|
||||
!Number.isFinite(imageX) ||
|
||||
!Number.isFinite(imageY) ||
|
||||
!Number.isSafeInteger(imageWidth) ||
|
||||
!Number.isSafeInteger(imageHeight) ||
|
||||
imageWidth <= 0 ||
|
||||
imageHeight <= 0 ||
|
||||
imageX < 0 ||
|
||||
imageY < 0 ||
|
||||
imageX >= imageWidth ||
|
||||
imageY >= imageHeight ||
|
||||
!Number.isFinite(rect.x) ||
|
||||
!Number.isFinite(rect.y) ||
|
||||
!Number.isFinite(rect.w) ||
|
||||
!Number.isFinite(rect.h) ||
|
||||
rect.w <= 0 ||
|
||||
rect.h <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
x: rect.x + (imageX / imageWidth) * rect.w,
|
||||
y: rect.y + (imageY / imageHeight) * rect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function pointInPolygon(point: Point, polygon: Point[]): boolean {
|
||||
for (let index = 0; index < polygon.length; index += 1) {
|
||||
const a = polygon[index];
|
||||
const b = polygon[(index + 1) % polygon.length];
|
||||
const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x);
|
||||
const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9;
|
||||
const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9;
|
||||
if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true;
|
||||
}
|
||||
let inside = false;
|
||||
for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) {
|
||||
const a = polygon[index];
|
||||
const b = polygon[previous];
|
||||
const intersects =
|
||||
a.y > point.y !== b.y > point.y &&
|
||||
point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x;
|
||||
if (intersects) inside = !inside;
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
export function pointInRegion(point: Point, region: Region): boolean {
|
||||
return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon));
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { type CdpRunner, isRpcError, normaliseRef } from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
mapImagePointToViewport,
|
||||
pointInRegion,
|
||||
sameRect,
|
||||
surfaceVisibleRect,
|
||||
} from "./surface-coordinate";
|
||||
|
||||
export interface SurfaceCaptureEnvironment {
|
||||
navigationIdentity: string;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
}
|
||||
|
||||
export interface SurfaceTargetResolverDeps {
|
||||
cdp: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface SurfacePointerTargetInput {
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
captureId?: string;
|
||||
imageX?: number;
|
||||
imageY?: number;
|
||||
}
|
||||
|
||||
export interface ResolvedSurfacePointerTarget {
|
||||
point: { x: number; y: number };
|
||||
usedRef: string;
|
||||
moveSettleMs: number;
|
||||
capture: {
|
||||
id: string;
|
||||
imageX: number;
|
||||
imageY: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Canvas-style controls commonly update their pointer hit-test on the next
|
||||
// animation frame. Leave a scheduling boundary after moving and before pressing.
|
||||
export const SURFACE_POINTER_MOVE_SETTLE_MS = 32;
|
||||
|
||||
function stale(message: string): RpcError {
|
||||
return rpcError("permission_denied", "surface_capture_stale", message);
|
||||
}
|
||||
|
||||
function finiteObject(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(finiteObject);
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value)
|
||||
.filter(
|
||||
([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)),
|
||||
)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, entry]) => [key, finiteObject(entry)]),
|
||||
);
|
||||
}
|
||||
|
||||
function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null {
|
||||
if (!frameId) return "top";
|
||||
const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
const path: Array<{
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
targetSessionId?: string;
|
||||
}> = [];
|
||||
const seen = new Set<string>();
|
||||
let current = frames.get(frameId);
|
||||
while (current) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push({
|
||||
frameId: current.frameId,
|
||||
...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}),
|
||||
...(current.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: current.ownerBackendNodeId }
|
||||
: {}),
|
||||
...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}),
|
||||
});
|
||||
if (!current.parentFrameId) return JSON.stringify(path);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function captureSurfaceEnvironment(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameId: string | undefined,
|
||||
): Promise<SurfaceCaptureEnvironment | RpcError> {
|
||||
try {
|
||||
const [tree, metrics, graph] = await Promise.all([
|
||||
cdp.send<{
|
||||
frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } };
|
||||
}>(tabId, "Page.getFrameTree", {}),
|
||||
cdp.send<Record<string, unknown>>(tabId, "Page.getLayoutMetrics", {}),
|
||||
frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null),
|
||||
]);
|
||||
const root = tree.frameTree?.frame;
|
||||
if (!root?.id || !root.loaderId) {
|
||||
return { code: "cdp_failed", message: "could not identify the current navigation" };
|
||||
}
|
||||
const frameProjectionSignature = frameId
|
||||
? graph
|
||||
? framePathSignature(graph, frameId)
|
||||
: null
|
||||
: "top";
|
||||
if (!frameProjectionSignature) {
|
||||
return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` };
|
||||
}
|
||||
return {
|
||||
navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }),
|
||||
viewportSignature: JSON.stringify(
|
||||
finiteObject({
|
||||
cssLayoutViewport: metrics.cssLayoutViewport,
|
||||
cssVisualViewport: metrics.cssVisualViewport,
|
||||
}),
|
||||
),
|
||||
frameProjectionSignature,
|
||||
};
|
||||
} catch (error) {
|
||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function validatePointParams(input: SurfacePointerTargetInput): RpcError | null {
|
||||
if (!input.captureId || typeof input.captureId !== "string") {
|
||||
return { code: "invalid_params", message: "surface point click requires capture_id" };
|
||||
}
|
||||
if (typeof input.imageX !== "number" || !Number.isFinite(input.imageX)) {
|
||||
return { code: "invalid_params", message: "image_x must be a finite number" };
|
||||
}
|
||||
if (typeof input.imageY !== "number" || !Number.isFinite(input.imageY)) {
|
||||
return { code: "invalid_params", message: "image_y must be a finite number" };
|
||||
}
|
||||
if (!input.ref || input.selector) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "surface point click requires one Surface ref and does not accept a selector",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError {
|
||||
const code = reason === "not_found" ? "not_found" : "permission_denied";
|
||||
return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`);
|
||||
}
|
||||
|
||||
function aborted(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted
|
||||
? { code: "cancelled", message: "surface target resolution aborted" }
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Resolve one fresh Surface capture coordinate into a validated top-viewport pointer target. */
|
||||
export async function resolveSurfacePointerTarget(
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number; url?: string },
|
||||
input: SurfacePointerTargetInput,
|
||||
deps: SurfaceTargetResolverDeps,
|
||||
): Promise<ResolvedSurfacePointerTarget | RpcError> {
|
||||
const invalid = validatePointParams(input);
|
||||
if (invalid) return invalid;
|
||||
const earlyAbort = aborted(deps.signal);
|
||||
if (earlyAbort) return earlyAbort;
|
||||
|
||||
const consumed = ctx.surfaceCaptures.consume(input.captureId as string);
|
||||
if (!consumed.ok) return captureConsumeError(consumed.reason);
|
||||
const capture = consumed.capture;
|
||||
const postConsumeAbort = aborted(deps.signal);
|
||||
if (postConsumeAbort) return postConsumeAbort;
|
||||
|
||||
if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) {
|
||||
return stale("surface capture belongs to a different session or tab");
|
||||
}
|
||||
if (normaliseRef(input.ref as string) !== capture.surface.ref) {
|
||||
return stale("surface ref does not match the capture");
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, input.ref as string, target.tabId, "screenshot");
|
||||
if (isRpcError(node)) return stale(node.message);
|
||||
if (
|
||||
node.kind !== "surface" ||
|
||||
node.backendNodeId !== capture.surface.backendNodeId ||
|
||||
node.frameId !== capture.surface.frameId ||
|
||||
node.generation !== capture.surface.observationGeneration
|
||||
) {
|
||||
return stale("surface observation generation or node identity changed");
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
try {
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
} catch (error) {
|
||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId);
|
||||
if (isRpcError(environment)) return environment;
|
||||
if (
|
||||
environment.navigationIdentity !== capture.navigationIdentity ||
|
||||
environment.viewportSignature !== capture.viewportSignature ||
|
||||
environment.frameProjectionSignature !== capture.frameProjectionSignature
|
||||
) {
|
||||
return stale("navigation, viewport, zoom, scroll, or frame projection changed");
|
||||
}
|
||||
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
},
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: false },
|
||||
);
|
||||
if (isRpcError(geometry)) return stale(geometry.message);
|
||||
const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect);
|
||||
if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) {
|
||||
return stale("surface visible region changed");
|
||||
}
|
||||
const point = mapImagePointToViewport(
|
||||
capture.topViewportRect,
|
||||
capture.imageWidth,
|
||||
capture.imageHeight,
|
||||
input.imageX as number,
|
||||
input.imageY as number,
|
||||
);
|
||||
if (!point || !pointInRegion(point, geometry.topVisibleRegions)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"surface_coordinate_invalid",
|
||||
"image coordinate is outside the captured Surface",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
point,
|
||||
usedRef: capture.surface.ref,
|
||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
capture: {
|
||||
id: capture.id,
|
||||
imageX: input.imageX as number,
|
||||
imageY: input.imageY as number,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -722,6 +722,44 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||
});
|
||||
|
||||
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
|
||||
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
|
||||
const i = (value: string) => S.indexOf(value);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 100],
|
||||
[10, 10, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1, 2],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const canvas = nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(canvas?.rendered).toBe(true);
|
||||
expect(canvas?.visuallySuppressed).toBe(true);
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import type { VomScene } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "../rendered-surfaces";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
|
||||
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "canvas",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
localRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
rendered: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function document(
|
||||
domNodes: CapturedNode[],
|
||||
axNodes: SemanticAxNode[] = [],
|
||||
): CapturedFrameDocument<SemanticAxNode> {
|
||||
return {
|
||||
frameId: domNodes[0]?.frameId ?? "main",
|
||||
contextScopeId: domNodes[0]?.frameId ?? "main",
|
||||
target: { tabId: 4 },
|
||||
domNodes,
|
||||
axNodes,
|
||||
};
|
||||
}
|
||||
|
||||
describe("rendered surface discovery", () => {
|
||||
it("finds prominent rendered canvases and excludes hidden descendants", () => {
|
||||
const visible = domNode(2, { parentBackendNodeId: 1 });
|
||||
const hidden = domNode(3, { parentBackendNodeId: 4 });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
visible,
|
||||
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
|
||||
hidden,
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
|
||||
expect(surfaces).toHaveLength(1);
|
||||
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
|
||||
});
|
||||
|
||||
it("keeps tiny canvases as individually addressable exact discoveries", () => {
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("keeps a surface when the canvas only has partial native accessibility content", () => {
|
||||
const axNodes = [
|
||||
{
|
||||
nodeId: "canvas",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "presentation" },
|
||||
childIds: ["button"],
|
||||
frameId: "main",
|
||||
},
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
frameId: "main",
|
||||
},
|
||||
] as SemanticAxNode[];
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
|
||||
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
|
||||
const canvas = domNode(2, {
|
||||
attrs: {
|
||||
id: "must-not-be-a-label",
|
||||
class: "must-not-be-a-label",
|
||||
"data-testid": "must-not-be-a-label",
|
||||
},
|
||||
});
|
||||
|
||||
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
|
||||
expect.not.objectContaining({
|
||||
label: expect.anything(),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats blank explicit Canvas names as unnamed", () => {
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
|
||||
});
|
||||
|
||||
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
|
||||
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([owner, canvas])],
|
||||
{ width: 1280, height: 720 },
|
||||
new Set([1]),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("excludes canvases suppressed by a fully transparent ancestor", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 900, h: 600 },
|
||||
visuallySuppressed: true,
|
||||
});
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 10, y: 20, w: 100, h: 70 },
|
||||
overflowX: "hidden",
|
||||
overflowY: "hidden",
|
||||
});
|
||||
const partial = domNode(2, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 55, y: 30, w: 140, h: 60 },
|
||||
});
|
||||
const fullyClipped = domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 140, y: 30, w: 80, h: 50 },
|
||||
});
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
|
||||
width: 1280,
|
||||
height: 720,
|
||||
}),
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 2,
|
||||
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rendered surface clustering and projection", () => {
|
||||
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
|
||||
const body = domNode(1, { tag: "body", rect: null, localRect: null });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
body,
|
||||
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
|
||||
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
const groups = clusterRenderedSurfaces(surfaces);
|
||||
const scene: VomScene = {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [
|
||||
{
|
||||
id: 10,
|
||||
parentId: null,
|
||||
backendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
role: "RootWebArea",
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]?.members).toHaveLength(2);
|
||||
expect(groups[0]?.label).toBeUndefined();
|
||||
expect(
|
||||
projectRenderedSurfaces(
|
||||
scene,
|
||||
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
|
||||
groups,
|
||||
),
|
||||
).toMatchObject({
|
||||
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge a contained canvas with a larger independent canvas", () => {
|
||||
const groups = clusterRenderedSurfaces([
|
||||
...discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
domNode(2, { parentBackendNodeId: 1 }),
|
||||
domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 30, y: 40, w: 200, h: 100 },
|
||||
}),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
]);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not merge coincident canvases owned by different containers", () => {
|
||||
const groups = clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(2, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(3, { parentBackendNodeId: 1 }),
|
||||
domNode(4, { parentBackendNodeId: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
@@ -39,6 +47,10 @@ export interface CapturedNode {
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
overflowX?: string;
|
||||
overflowY?: string;
|
||||
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
|
||||
visuallySuppressed?: boolean;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
@@ -796,6 +808,8 @@ function parseDocumentNodes(
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const overflowXCol = STYLE_COL["overflow-x"];
|
||||
const overflowYCol = STYLE_COL["overflow-y"];
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
@@ -821,6 +835,7 @@ function parseDocumentNodes(
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
@@ -840,7 +855,11 @@ function parseDocumentNodes(
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let overflowX = "visible";
|
||||
let overflowY = "visible";
|
||||
let rendered = false;
|
||||
let visuallySuppressed =
|
||||
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
@@ -863,12 +882,19 @@ function parseDocumentNodes(
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
|
||||
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
|
||||
visuallySuppressed ||=
|
||||
visibility === "hidden" ||
|
||||
visibility === "collapse" ||
|
||||
(Number.parseFloat(opacity) || 0) <= 0;
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
@@ -903,6 +929,9 @@ function parseDocumentNodes(
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
overflowX,
|
||||
overflowY,
|
||||
visuallySuppressed,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
|
||||
const STACK_IOU_RATIO = 0.9;
|
||||
const STACK_SIZE_RATIO = 0.9;
|
||||
|
||||
function area(rect: Rect): number {
|
||||
return Math.max(0, rect.w) * Math.max(0, rect.h);
|
||||
}
|
||||
|
||||
function intersectionArea(a: Rect, b: Rect): number {
|
||||
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
|
||||
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
|
||||
return width * height;
|
||||
}
|
||||
|
||||
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
||||
if (a.frameId !== b.frameId) return false;
|
||||
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
|
||||
const aArea = area(a.visibleRect);
|
||||
const bArea = area(b.visibleRect);
|
||||
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
|
||||
const union = aArea + bArea - intersection;
|
||||
const widthRatio =
|
||||
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
|
||||
const heightRatio =
|
||||
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
|
||||
return (
|
||||
union > 0 &&
|
||||
intersection / union >= STACK_IOU_RATIO &&
|
||||
widthRatio >= STACK_SIZE_RATIO &&
|
||||
heightRatio >= STACK_SIZE_RATIO
|
||||
);
|
||||
}
|
||||
|
||||
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
|
||||
const representative = [...members].sort(
|
||||
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
|
||||
)[0];
|
||||
return {
|
||||
frameId: representative.frameId,
|
||||
members,
|
||||
representative,
|
||||
label: members.map((surface) => surface.label).find((label) => label !== undefined),
|
||||
};
|
||||
}
|
||||
|
||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
|
||||
const groups: RenderedSurface[][] = [];
|
||||
for (const surface of surfaces) {
|
||||
const matching = groups.find((group) =>
|
||||
group.some((member) => sameVisualStack(member, surface)),
|
||||
);
|
||||
if (matching) matching.push(surface);
|
||||
else groups.push([surface]);
|
||||
}
|
||||
return groups
|
||||
.map(toGroup)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurface } from "./types";
|
||||
|
||||
function clean(value: string | undefined): string | undefined {
|
||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
|
||||
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
function intersectRect(a: Rect, b: Rect): Rect | null {
|
||||
const x = Math.max(a.x, b.x);
|
||||
const y = Math.max(a.y, b.y);
|
||||
const right = Math.min(a.x + a.w, b.x + b.w);
|
||||
const bottom = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, w: right - x, h: bottom - y };
|
||||
}
|
||||
|
||||
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
|
||||
|
||||
function clipToOverflowAncestors(
|
||||
node: CapturedNode,
|
||||
rect: Rect,
|
||||
nodesByBackend: ReadonlyMap<number, CapturedNode>,
|
||||
): Rect | null {
|
||||
let clipped: Rect | null = rect;
|
||||
let parentBackendNodeId = node.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
|
||||
const parent = nodesByBackend.get(parentBackendNodeId);
|
||||
if (!parent) break;
|
||||
if (parent.rect) {
|
||||
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
|
||||
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
|
||||
if (clipX || clipY) {
|
||||
const clipRect = {
|
||||
x: clipX ? parent.rect.x : clipped.x,
|
||||
y: clipY ? parent.rect.y : clipped.y,
|
||||
w: clipX ? parent.rect.w : clipped.w,
|
||||
h: clipY ? parent.rect.h : clipped.h,
|
||||
};
|
||||
clipped = intersectRect(clipped, clipRect);
|
||||
}
|
||||
}
|
||||
parentBackendNodeId = parent.parentBackendNodeId;
|
||||
guard += 1;
|
||||
}
|
||||
return clipped;
|
||||
}
|
||||
|
||||
function excludedByDomPolicy(
|
||||
node: CapturedNode,
|
||||
nodesByBackend: Map<number, CapturedNode>,
|
||||
excludedBackendNodeIds: ReadonlySet<number>,
|
||||
): boolean {
|
||||
let current: CapturedNode | undefined = node;
|
||||
let guard = 0;
|
||||
while (current && guard <= nodesByBackend.size) {
|
||||
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
|
||||
if (current.visuallySuppressed === true) return true;
|
||||
const attrs = current.attrs;
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
||||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
||||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
current =
|
||||
current.parentBackendNodeId === null
|
||||
? undefined
|
||||
: nodesByBackend.get(current.parentBackendNodeId);
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function discoverRenderedSurfaces(
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
viewport: Viewport,
|
||||
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
|
||||
): RenderedSurface[] {
|
||||
const surfaces: RenderedSurface[] = [];
|
||||
for (const document of documents) {
|
||||
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
|
||||
for (const node of document.domNodes) {
|
||||
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
|
||||
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
|
||||
const viewportRect = intersectViewport(node.rect, viewport);
|
||||
const visibleRect = viewportRect
|
||||
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
|
||||
: null;
|
||||
if (!visibleRect) continue;
|
||||
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
|
||||
surfaces.push({
|
||||
renderingKind: "canvas",
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
parentBackendNodeId: node.parentBackendNodeId,
|
||||
visibleRect,
|
||||
paintOrder: node.paintOrder,
|
||||
...(label ? { label } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return surfaces;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export { clusterRenderedSurfaces } from "./cluster";
|
||||
export { discoverRenderedSurfaces } from "./discovery";
|
||||
export { projectRenderedSurfaces } from "./project";
|
||||
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurfaceGroup } from "./types";
|
||||
|
||||
function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
interface ProjectionIndex {
|
||||
sceneByBackend: ReadonlyMap<string, number>;
|
||||
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
|
||||
domByFrame: ReadonlyMap<
|
||||
string,
|
||||
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
|
||||
>;
|
||||
rootSceneIdByFrame: ReadonlyMap<string, number>;
|
||||
}
|
||||
|
||||
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
|
||||
const document = index.documentByFrame.get(group.frameId);
|
||||
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
|
||||
let parentBackendNodeId = group.representative.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
|
||||
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
|
||||
if (sceneId !== undefined) return sceneId;
|
||||
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
|
||||
guard += 1;
|
||||
}
|
||||
|
||||
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
|
||||
const owner = index.sceneByBackend.get(
|
||||
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
|
||||
);
|
||||
if (owner !== undefined) return owner;
|
||||
}
|
||||
|
||||
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
|
||||
}
|
||||
|
||||
function buildProjectionIndex(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
): ProjectionIndex {
|
||||
const rootSceneIdByFrame = new Map<string, number>();
|
||||
for (const node of scene.nodes) {
|
||||
if (
|
||||
node.frameId &&
|
||||
!rootSceneIdByFrame.has(node.frameId) &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
|
||||
) {
|
||||
rootSceneIdByFrame.set(node.frameId, node.id);
|
||||
}
|
||||
}
|
||||
return {
|
||||
sceneByBackend: new Map(
|
||||
scene.nodes.flatMap((node) =>
|
||||
node.backendNodeId === undefined
|
||||
? []
|
||||
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
|
||||
),
|
||||
),
|
||||
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
|
||||
domByFrame: new Map(
|
||||
documents.map((document) => [
|
||||
document.frameId,
|
||||
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
|
||||
]),
|
||||
),
|
||||
rootSceneIdByFrame,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectRenderedSurfaces(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
groups: RenderedSurfaceGroup[],
|
||||
): VomScene {
|
||||
const index = buildProjectionIndex(scene, documents);
|
||||
const visualSurfaces: VomVisualSurface[] = [];
|
||||
|
||||
for (const group of groups) {
|
||||
const parentId = nearestSceneParent(index, group);
|
||||
visualSurfaces.push({
|
||||
parentId,
|
||||
backendNodeId: group.representative.backendNodeId,
|
||||
frameId: group.frameId,
|
||||
renderingKind: "canvas",
|
||||
visibleRect: group.representative.visibleRect,
|
||||
...(group.label ? { label: group.label } : {}),
|
||||
memberCount: group.members.length,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...scene,
|
||||
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface RenderedSurface {
|
||||
renderingKind: "canvas";
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
visibleRect: Rect;
|
||||
paintOrder: number;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface RenderedSurfaceGroup {
|
||||
frameId: string;
|
||||
members: RenderedSurface[];
|
||||
representative: RenderedSurface;
|
||||
label?: string;
|
||||
}
|
||||
@@ -23,9 +23,12 @@ export type ErrorCode =
|
||||
export type RpcErrorReason =
|
||||
| "agent_window_scope"
|
||||
| "element_not_visible"
|
||||
| "ref_capability_denied"
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "focus_target_unresolved"
|
||||
| "input_not_applied"
|
||||
| "target_not_select"
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
@@ -33,6 +36,11 @@ export type RpcErrorReason =
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "surface_capture_not_found"
|
||||
| "surface_capture_expired"
|
||||
| "surface_capture_consumed"
|
||||
| "surface_capture_stale"
|
||||
| "surface_coordinate_invalid"
|
||||
| "cleanup_failed";
|
||||
|
||||
export interface RpcErrorData {
|
||||
@@ -298,9 +306,17 @@ export interface ScreenshotResult {
|
||||
height: number;
|
||||
format: string;
|
||||
tab_id: number;
|
||||
capture?: SurfaceCaptureInfo;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface SurfaceCaptureInfo {
|
||||
id: string;
|
||||
surface_ref: string;
|
||||
coordinate_space: "capture-image-pixel";
|
||||
expires_at: number;
|
||||
}
|
||||
|
||||
export interface SnapshotParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
@@ -414,6 +430,9 @@ export interface ClickParams {
|
||||
click_count?: number;
|
||||
modifiers?: KeyModifier[];
|
||||
timeout_ms?: number;
|
||||
capture_id?: string;
|
||||
image_x?: number;
|
||||
image_y?: number;
|
||||
}
|
||||
|
||||
export interface ClickResult {
|
||||
@@ -422,6 +441,9 @@ export interface ClickResult {
|
||||
used_selector?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
capture_id?: string;
|
||||
image_x?: number;
|
||||
image_y?: number;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Rendered Surface browser fixtures
|
||||
|
||||
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
|
||||
|
||||
## Serve
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
python3 -m http.server 4173 --bind 127.0.0.1 \
|
||||
--directory apps/extension/test-fixtures/rendered-surfaces
|
||||
```
|
||||
|
||||
Use an Agent Window large enough to keep the primary matrix visible:
|
||||
|
||||
```bash
|
||||
bsk session start --width 1440 --height 1100
|
||||
bsk navigate http://127.0.0.1:4173/index.html --session <id>
|
||||
bsk observe --session <id>
|
||||
```
|
||||
|
||||
Always stop the session after the run.
|
||||
|
||||
## Pages
|
||||
|
||||
### `index.html`
|
||||
|
||||
Expected individual Surface labels:
|
||||
|
||||
- `fixture-basic-large`
|
||||
- `fixture-tiny-8px`
|
||||
- `fixture-pointer-events-none`
|
||||
- `fixture-partial-ax`
|
||||
- `fixture-contained-outer`
|
||||
- `fixture-contained-inner`
|
||||
- `fixture-different-parent-a`
|
||||
- `fixture-different-parent-b`
|
||||
|
||||
Expected one grouped Surface:
|
||||
|
||||
- `fixture-layered-group`, with `layers=2`
|
||||
|
||||
Expected absent:
|
||||
|
||||
- `fixture-display-none`
|
||||
- `fixture-visibility-hidden`
|
||||
- `fixture-opacity-zero`
|
||||
- `fixture-parent-opacity-zero`
|
||||
- `fixture-hidden-attribute`
|
||||
- `fixture-inert-parent`
|
||||
- `fixture-aria-hidden-parent`
|
||||
- `fixture-zero-size`
|
||||
- `fixture-offscreen-right`
|
||||
- `fixture-fully-overflow-clipped`
|
||||
|
||||
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
|
||||
|
||||
### `unlabeled.html`
|
||||
|
||||
This page contains eight Canvas elements but must render seven generic Surface markers because
|
||||
the two coincident layers form one group.
|
||||
|
||||
- Every marker uses the stable fallback label `canvas visual surface`.
|
||||
- Exactly one marker has `layers=2`.
|
||||
- The two contained canvases remain separate markers.
|
||||
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
|
||||
- Blank `aria-label` and `title` values behave as no label.
|
||||
- Fallback AX content does not suppress the corresponding Surface.
|
||||
|
||||
### `frames.html`
|
||||
|
||||
Expected Surface labels:
|
||||
|
||||
- `fixture-same-origin-frame`
|
||||
- `fixture-cross-origin-frame`
|
||||
- `fixture-nested-frame`
|
||||
|
||||
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
|
||||
|
||||
Each frame Canvas also exposes a `pointer status` live region. A successful
|
||||
screenshot-bound point click must report
|
||||
`down button=0 buttons=1 hoverReady=true; click received` and turn the Canvas
|
||||
yellow. The hover flag is set on the animation frame after `pointermove`, so the
|
||||
fixture verifies both child-document delivery and the scheduling boundary
|
||||
required before `pointerdown`, rather than merely validating projected
|
||||
coordinates or a successful CDP response.
|
||||
|
||||
### `viewport.html`
|
||||
|
||||
Run with a `900x700` Agent Window.
|
||||
|
||||
- `fixture-partial-viewport` is partly visible and must be represented.
|
||||
- `fixture-below-fold` is absent initially.
|
||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
||||
|
||||
### `input-frames.html`
|
||||
|
||||
This page exercises generic frame-aware input routing. It contains a top-level
|
||||
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
||||
frame includes an input, textarea, contenteditable editor, and a controlled
|
||||
input that resets its value on the next animation frame.
|
||||
|
||||
- `fill` must route focus, text insertion, DOM events, and readback through the
|
||||
ref's owning CDP target.
|
||||
- `press --ref` must focus and dispatch every key event through that same target.
|
||||
- Untargeted `press` must follow the actual focused target or refuse when focus
|
||||
ownership cannot be determined uniquely.
|
||||
- The rejecting input must return `input_not_applied`, never a successful
|
||||
requested-value length.
|
||||
|
||||
### `canvas-input.html`
|
||||
|
||||
This is the PR3 end-to-end fixture. Its cross-origin frame contains a
|
||||
visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
|
||||
focuses a standard textbox; ordinary `fill` must then write and read back its
|
||||
real value through the textbox's owning target. The fixture deliberately does
|
||||
not provide a Canvas-specific fill action.
|
||||
|
||||
### `dynamic.html`
|
||||
|
||||
Use the normal DOM buttons and re-observe after each action:
|
||||
|
||||
- Initial state: `fixture-dynamic` absent.
|
||||
- After **Insert canvas**: present.
|
||||
- After **Resize to zero**: absent.
|
||||
- After **Restore size**: present with a fresh ref.
|
||||
- After **Remove canvas**: absent.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `snapshot` must not emit any of these Surface markers.
|
||||
- `observe` must emit only currently visible Canvas surfaces.
|
||||
- Existing DOM/AX controls remain readable alongside Surface markers.
|
||||
- Missing labels never prevent discovery; naming and discovery are independent.
|
||||
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
|
||||
- No fixture URL, id, label, or layout constant may appear in production discovery code.
|
||||
@@ -0,0 +1,20 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Canvas to standard input fixture</title>
|
||||
<style>
|
||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||
iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Canvas to standard input fixture</h1>
|
||||
<p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
|
||||
<iframe
|
||||
title="cross-origin Canvas editor"
|
||||
src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
|
||||
></iframe>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,46 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface dynamic matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
|
||||
button { padding: 8px 12px; }
|
||||
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Dynamic Canvas lifecycle</h1>
|
||||
<div class="controls">
|
||||
<button id="insert" type="button">Insert canvas</button>
|
||||
<button id="zero" type="button">Resize to zero</button>
|
||||
<button id="restore" type="button">Restore size</button>
|
||||
<button id="remove" type="button">Remove canvas</button>
|
||||
</div>
|
||||
<div id="mount"></div>
|
||||
<script>
|
||||
const mount = document.querySelector("#mount");
|
||||
const getCanvas = () => mount.querySelector("canvas");
|
||||
document.querySelector("#insert").addEventListener("click", () => {
|
||||
if (getCanvas()) return;
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 260;
|
||||
canvas.height = 120;
|
||||
canvas.setAttribute("aria-label", "fixture-dynamic");
|
||||
mount.append(canvas);
|
||||
});
|
||||
document.querySelector("#zero").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
|
||||
});
|
||||
document.querySelector("#restore").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.removeAttribute("style");
|
||||
});
|
||||
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,80 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 12px; background: #fdfefe; }
|
||||
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
|
||||
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
|
||||
#pointer-status { margin: 8px 0 0; min-height: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="frame-canvas" width="220" height="90"></canvas>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const canvas = document.querySelector("#frame-canvas");
|
||||
if (params.get("canvas") === "0") canvas.remove();
|
||||
else {
|
||||
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
||||
if (params.get("editor") === "1") {
|
||||
const context = canvas.getContext("2d");
|
||||
context.font = "14px system-ui, sans-serif";
|
||||
context.textBaseline = "middle";
|
||||
context.strokeStyle = "#1e8449";
|
||||
context.fillStyle = "#145a32";
|
||||
for (let row = 0; row < 3; row += 1) {
|
||||
const top = row * 30;
|
||||
context.strokeRect(0.5, top + 0.5, 219, 29);
|
||||
context.fillText(`Row ${row + 1}`, 12, top + 15);
|
||||
}
|
||||
}
|
||||
if (params.get("interactive") === "1") {
|
||||
const status = document.createElement("p");
|
||||
status.id = "pointer-status";
|
||||
status.setAttribute("aria-live", "polite");
|
||||
status.textContent = "pointer status: idle";
|
||||
canvas.insertAdjacentElement("afterend", status);
|
||||
let hoverReady = false;
|
||||
canvas.addEventListener("pointermove", () => {
|
||||
hoverReady = false;
|
||||
requestAnimationFrame(() => {
|
||||
hoverReady = true;
|
||||
});
|
||||
});
|
||||
canvas.addEventListener("pointerdown", (event) => {
|
||||
status.textContent = `pointer status: down button=${event.button} buttons=${event.buttons} hoverReady=${hoverReady}`;
|
||||
if (event.buttons === 1 && hoverReady) canvas.style.background = "#f9e79f";
|
||||
});
|
||||
canvas.addEventListener("click", () => {
|
||||
status.textContent += "; click received";
|
||||
});
|
||||
if (params.get("editor") === "1") {
|
||||
canvas.addEventListener("click", (event) => {
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
|
||||
const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
|
||||
status.textContent += `; selected row=${row + 1}`;
|
||||
document.querySelector("#canvas-cell-editor")?.remove();
|
||||
if (row !== 2) return;
|
||||
const editor = document.createElement("input");
|
||||
editor.id = "canvas-cell-editor";
|
||||
editor.setAttribute("aria-label", "selected canvas cell editor");
|
||||
editor.placeholder = "Type into selected row 3";
|
||||
canvas.insertAdjacentElement("afterend", editor);
|
||||
editor.focus();
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested fixture frame";
|
||||
iframe.src = "frame.html?label=fixture-nested-frame&interactive=1";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,22 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface frame matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame matrix</h1>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame&interactive=1"></iframe>
|
||||
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame&interactive=1"></iframe>
|
||||
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,118 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface discovery matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
.same-place { position: relative; width: 180px; height: 90px; }
|
||||
.same-place > div { position: absolute; inset: 0; }
|
||||
.same-place canvas { width: 180px; height: 90px; }
|
||||
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
|
||||
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
|
||||
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
|
||||
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
|
||||
.tiny { width: 8px; height: 8px; border-width: 1px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Rendered Surface discovery matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Visible named canvas</h2>
|
||||
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Tiny canvas remains addressable</h2>
|
||||
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Visible but pointer-events none</h2>
|
||||
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partial AX/fallback content</h2>
|
||||
<canvas aria-label="fixture-partial-ax" width="180" height="80">
|
||||
<button type="button">Fallback button must not suppress the surface</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Two real rendering layers</h2>
|
||||
<div class="stack">
|
||||
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Contained canvases are independent</h2>
|
||||
<div class="contained">
|
||||
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
|
||||
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Same geometry, different parents</h2>
|
||||
<div class="same-place">
|
||||
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
|
||||
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partially overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fully overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>CSS-hidden variants</h2>
|
||||
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
|
||||
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
|
||||
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Hidden ancestor variants</h2>
|
||||
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
|
||||
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
|
||||
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
|
||||
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Zero-size and offscreen</h2>
|
||||
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
|
||||
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
|
||||
label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
|
||||
input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
|
||||
iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<label>Frame input <input aria-label="frame input" /></label>
|
||||
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
||||
<div contenteditable="true" aria-label="frame editor"></div>
|
||||
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const rejecting = document.querySelector("#rejecting");
|
||||
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested input frame";
|
||||
iframe.src = "input-frame.html";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Frame input routing matrix</title>
|
||||
<style>
|
||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
|
||||
input { padding: 6px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame input routing matrix</h1>
|
||||
<label>Top input <input aria-label="top input" /></label>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin inputs" src="input-frame.html"></iframe>
|
||||
<iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
|
||||
<iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,68 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Unlabeled Rendered Surface matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
.case p { margin: 0 0 8px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Unlabeled Rendered Surface matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Only engineering attributes</h2>
|
||||
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Blank explicit names</h2>
|
||||
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Nearby prose is not a Canvas name</h2>
|
||||
<p>must-not-be-inferred-as-a-surface-label</p>
|
||||
<canvas width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fallback AX content does not name or suppress the Canvas</h2>
|
||||
<canvas width="180" height="80">
|
||||
<button type="button">Fallback control remains separate</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed rendering layers still cluster</h2>
|
||||
<div class="stack">
|
||||
<canvas width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed contained canvases stay independent</h2>
|
||||
<div class="contained">
|
||||
<canvas width="190" height="100"></canvas>
|
||||
<canvas width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,22 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface viewport matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { min-height: 1800px; padding: 16px; }
|
||||
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
|
||||
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
|
||||
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Viewport matrix</h1>
|
||||
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
|
||||
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
|
||||
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -80,6 +80,16 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
||||
|
||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
|
||||
|
||||
```bash
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
@@ -174,7 +184,7 @@ bsk emulate --session <id> --off
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
@@ -200,7 +210,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
|
||||
@@ -136,6 +136,18 @@ pub struct ClickArgs {
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
|
||||
/// Capture id returned by `bsk screenshot --ref <surface>`.
|
||||
#[arg(long)]
|
||||
pub capture: Option<String>,
|
||||
|
||||
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
|
||||
#[arg(long = "image-x", allow_hyphen_values = true)]
|
||||
pub image_x: Option<f64>,
|
||||
|
||||
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
|
||||
#[arg(long = "image-y", allow_hyphen_values = true)]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -147,6 +159,14 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let point_arg_count = usize::from(args.capture.is_some())
|
||||
+ usize::from(args.image_x.is_some())
|
||||
+ usize::from(args.image_y.is_some());
|
||||
if point_arg_count != 0 && point_arg_count != 3 {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"--capture, --image-x, and --image-y must be given together"
|
||||
)));
|
||||
}
|
||||
let params = ClickParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
@@ -160,6 +180,9 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Some(modifiers)
|
||||
},
|
||||
timeout_ms: Some(args.timeout),
|
||||
capture_id: args.capture.clone(),
|
||||
image_x: args.image_x,
|
||||
image_y: args.image_y,
|
||||
};
|
||||
let reply: ClickResult = call(
|
||||
info.sock_path,
|
||||
@@ -183,6 +206,14 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
"click ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
if let Some(capture_id) = &reply.capture_id {
|
||||
println!(
|
||||
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
|
||||
capture_id,
|
||||
reply.image_x.unwrap_or_default(),
|
||||
reply.image_y.unwrap_or_default()
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,15 +40,23 @@ use bsk_protocol::ErrorCode;
|
||||
pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
|
||||
pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
|
||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
||||
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
||||
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
||||
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
|
||||
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
|
||||
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
|
||||
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
|
||||
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
|
||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||
}
|
||||
@@ -228,6 +236,27 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
|
||||
summary: "focused frame target could not be determined",
|
||||
hint: Some(
|
||||
"pass an explicit --ref or --selector so the key event can be routed safely",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
|
||||
summary: "the page did not retain the requested input",
|
||||
hint: Some(
|
||||
"refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||
summary: "snapshot ref does not support this operation",
|
||||
hint: Some(
|
||||
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
|
||||
summary: "target element is not a <select>",
|
||||
hint: Some(
|
||||
@@ -270,6 +299,41 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
|
||||
summary: "surface capture transaction was not found",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and use its capture id with coordinates from that image",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
|
||||
summary: "surface capture transaction has expired",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
|
||||
summary: "surface capture transaction was already used",
|
||||
hint: Some(
|
||||
"each capture id is single-use; take a fresh surface screenshot before another point action",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
|
||||
summary: "page state changed since the surface screenshot",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
|
||||
summary: "surface image coordinate is invalid",
|
||||
hint: Some(
|
||||
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(
|
||||
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
|
||||
reason::RECORD_START_PAGE_UNREACHABLE,
|
||||
@@ -377,6 +441,18 @@ mod tests {
|
||||
assert!(!info.summary.contains("sandbox"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_capability_denied_explains_visual_surface_refs() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(info.summary, "snapshot ref does not support this operation");
|
||||
assert!(
|
||||
info.hint.unwrap().contains("capture id"),
|
||||
"expected visual-surface-specific hint"
|
||||
);
|
||||
assert_eq!(info.exit_code, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_busy_overrides_timeout_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||
@@ -419,6 +495,25 @@ mod tests {
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_capture_stale_requests_a_fresh_screenshot() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"page state changed since the surface screenshot"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
|
||||
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
|
||||
assert_eq!(info.summary, "surface image coordinate is invalid");
|
||||
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_not_found_overrides_not_found_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||
@@ -453,4 +548,20 @@ mod tests {
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("exactly one"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn focus_target_unresolved_overrides_cdp_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert_eq!(info.summary, "focused frame target could not be determined");
|
||||
assert!(info.hint.unwrap().contains("--ref"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_not_applied_overrides_cdp_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert_eq!(info.summary, "the page did not retain the requested input");
|
||||
assert!(info.hint.unwrap().contains("rejecting"));
|
||||
}
|
||||
}
|
||||
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
.map_err(CliError::Local)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let json = serde_json::json!({
|
||||
let mut json = serde_json::json!({
|
||||
"tab_id": reply.tab_id,
|
||||
"width": reply.width,
|
||||
"height": reply.height,
|
||||
@@ -67,6 +67,10 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
"path": out_path.to_string_lossy(),
|
||||
"byte_size": bytes.len(),
|
||||
});
|
||||
if let Some(capture) = &reply.capture {
|
||||
json["capture"] = serde_json::to_value(capture)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json)
|
||||
@@ -75,6 +79,12 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
}
|
||||
Format::Human => {
|
||||
println!("{}", out_path.display());
|
||||
if let Some(capture) = &reply.capture {
|
||||
println!(
|
||||
"capture={} surface={} coordinate_space={} expires_at={}",
|
||||
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,6 +188,29 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_surface_capture_image_coordinates() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"click",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"160.5",
|
||||
"--image-y",
|
||||
"-1",
|
||||
]);
|
||||
let Command::Click(args) = cli.command else {
|
||||
panic!("expected click command");
|
||||
};
|
||||
assert_eq!(args.capture.as_deref(), Some("sc_test"));
|
||||
assert_eq!(args.image_x, Some(160.5));
|
||||
assert_eq!(args.image_y, Some(-1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
|
||||
@@ -238,6 +238,7 @@ async fn screenshot_returns_image_base64_with_dimensions() {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -286,6 +287,7 @@ async fn screenshot_forwards_ref_to_extension() {
|
||||
height: 60,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
|
||||
@@ -354,6 +354,9 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
used_selector: None,
|
||||
x: 12.5,
|
||||
y: 34.0,
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -373,6 +376,9 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -610,6 +616,9 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
click_count: None,
|
||||
modifiers: None,
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -16,6 +16,13 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"capture_id": {
|
||||
"description": "Short-lived capture id returned by a visual Surface screenshot.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"click_count": {
|
||||
"description": "Number of consecutive mouse presses (double-click = 2).",
|
||||
"type": [
|
||||
@@ -25,6 +32,22 @@
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
},
|
||||
"image_x": {
|
||||
"description": "X coordinate in the capture image's pixel coordinate space.",
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"image_y": {
|
||||
"description": "Y coordinate in the capture image's pixel coordinate space.",
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
|
||||
@@ -8,6 +8,12 @@
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"capture_id": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"description": "Native JS dialogs observed and auto-handled during this call.",
|
||||
"type": "array",
|
||||
@@ -15,6 +21,20 @@
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"image_x": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"image_y": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
|
||||
@@ -8,6 +8,16 @@
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"capture": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/SurfaceCaptureInfo"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -109,6 +119,31 @@
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"SurfaceCaptureInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"coordinate_space",
|
||||
"expires_at",
|
||||
"id",
|
||||
"surface_ref"
|
||||
],
|
||||
"properties": {
|
||||
"coordinate_space": {
|
||||
"type": "string"
|
||||
},
|
||||
"expires_at": {
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"surface_ref": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,15 @@ pub struct ClickParams {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
/// Short-lived capture id returned by a visual Surface screenshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture_id: Option<String>,
|
||||
/// X coordinate in the capture image's pixel coordinate space.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_x: Option<f64>,
|
||||
/// Y coordinate in the capture image's pixel coordinate space.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -80,6 +89,12 @@ pub struct ClickResult {
|
||||
/// agents can correlate with a follow-up `tool.screenshot`.
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_x: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_y: Option<f64>,
|
||||
/// Native JS dialogs observed and auto-handled during this call.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
@@ -288,6 +303,9 @@ mod tests {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: Some("sc_test".into()),
|
||||
image_x: Some(160.0),
|
||||
image_y: Some(90.0),
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||
|
||||
@@ -208,10 +208,20 @@ pub struct ScreenshotResult {
|
||||
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
|
||||
pub format: String,
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture: Option<SurfaceCaptureInfo>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SurfaceCaptureInfo {
|
||||
pub id: String,
|
||||
pub surface_ref: String,
|
||||
pub coordinate_space: String,
|
||||
pub expires_at: u64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -293,6 +303,12 @@ mod tests {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: Some(SurfaceCaptureInfo {
|
||||
id: "sc_test".into(),
|
||||
surface_ref: "@e3".into(),
|
||||
coordinate_space: "capture-image-pixel".into(),
|
||||
expires_at: 1_700_000_030_000,
|
||||
}),
|
||||
dialogs: vec![],
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
|
||||
@@ -20,22 +20,43 @@ extension keep owning the actual browser control — this package is a thin, wel
|
||||
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
|
||||
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
|
||||
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
|
||||
| `browser_hover` | `bsk hover` | Hover a ref or selector and allow hover-triggered UI to settle. |
|
||||
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
|
||||
| `browser_select` | `bsk select` | Select one or more option values on a `<select>`. |
|
||||
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
|
||||
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
|
||||
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
|
||||
| `browser_tab_list` | `bsk tab list` | List user and/or Agent Window tabs visible to an owned session. |
|
||||
| `browser_tab_create` | `bsk tab create` | Create a foreground or background tab in the Agent Window. |
|
||||
| `browser_tab_select` | `bsk tab select` | Focus an Agent Window tab. |
|
||||
| `browser_tab_close` | `bsk tab close` | Close an Agent Window tab. |
|
||||
| `browser_tab_borrow` | `bsk tab borrow` | Move a user tab into the Agent Window for controlled interaction. |
|
||||
| `browser_tab_return` | `bsk tab return` | Return a borrowed tab to its original user window. |
|
||||
| `browser_navigate_back` / `browser_navigate_forward` | `bsk navigate-back` / `bsk navigate-forward` | Traverse tab history and wait for a lifecycle phase. |
|
||||
| `browser_reload` | `bsk reload` | Reload the active tab, optionally bypassing cache. |
|
||||
| `browser_wait_for_navigation` | `bsk wait-for-navigation` | Wait explicitly for a page lifecycle event. |
|
||||
| `browser_request_help` | `bsk request-help` | Pause for login, captcha, OTP, confirmation, or another human browser step. |
|
||||
| `browser_get_html` | `bsk get-html` | Read raw document or ref-scoped HTML after semantic observation is insufficient. |
|
||||
| `browser_console` | `bsk console` | Read buffered console/log/exception entries without evaluating JavaScript. |
|
||||
| `browser_network` | `bsk network` | Read buffered network response/failure metadata. |
|
||||
| `browser_window_resize` | `bsk window resize` | Resize the owned session's Agent Window. |
|
||||
|
||||
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
|
||||
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
|
||||
DeepSeek Harness lifecycle before it can be added safely.
|
||||
|
||||
## Agent skill (progressive disclosure)
|
||||
|
||||
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
|
||||
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
|
||||
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
|
||||
tool. The body is assembled at build time from two parts, so there is exactly one source of
|
||||
truth: a dsh-specific prelude (`skill/prelude.md` — tool↔CLI map, owned-session semantics,
|
||||
plugin-only overrides) followed verbatim by the canonical CLI skill (`skill/SKILL.md` at the
|
||||
repo root — the same file `crates/bsk-cli/build.rs` mirrors for the CLI package; workflows,
|
||||
stop-when-done rules, refs usage, sandbox rules). Registration and every pre-step catalog
|
||||
snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
|
||||
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
|
||||
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
|
||||
concatenated: its command examples belong to a different execution interface and would bypass
|
||||
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
|
||||
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
|
||||
missing supported browser tools before embedding the Markdown. Registration and every pre-step
|
||||
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
|
||||
seam degrade silently.
|
||||
|
||||
## Multi-session model
|
||||
@@ -88,7 +109,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
|
||||
# lazyTools: true # reveal browser_* tools only after the skill is invoked
|
||||
```
|
||||
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the eleven
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the 28
|
||||
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
|
||||
`browser-skill` skill is actually invoked — the skill catalog entry is the only
|
||||
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
|
||||
@@ -152,7 +173,7 @@ shell's theme cannot bleed back in.
|
||||
card). Screenshots additionally attach the image itself when the host mounts an attachment store
|
||||
and the active model route declares image input; otherwise the PNG path is returned.
|
||||
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
|
||||
`lib/client.js`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
|
||||
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
|
||||
custom view keeps the terminal block (command + output) and, when the settled result carries an
|
||||
image block, resolves the durable attachment through the client session's authorized
|
||||
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
|
||||
@@ -183,8 +204,8 @@ as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
|
||||
Trigger it by pushing a tag that matches `package.json`'s `version`:
|
||||
|
||||
```sh
|
||||
git tag dsh-plugin-v0.1.0
|
||||
git push origin dsh-plugin-v0.1.0
|
||||
git tag dsh-plugin-v0.1.2
|
||||
git push origin dsh-plugin-v0.1.2
|
||||
```
|
||||
|
||||
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
|
||||
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model",
|
||||
"version": "0.1.1",
|
||||
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
|
||||
"version": "0.1.2",
|
||||
"license": "MIT",
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
@@ -107,6 +107,8 @@
|
||||
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-scope": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-skill": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/schemastery": "^3.18.1",
|
||||
"@tailwindcss/cli": "^4.3.3",
|
||||
|
||||
@@ -1,20 +1,15 @@
|
||||
// Assemble the plugin's skill content: a dsh-specific routing prelude, the
|
||||
// canonical CLI skill body verbatim (repo-root skill/SKILL.md — the same source
|
||||
// crates/bsk-cli's build.rs mirrors), then a dsh routing reminder postlude. The
|
||||
// original's YAML frontmatter is stripped: name/description are supplied by the
|
||||
// registration.
|
||||
// Embed the DeepSeek Harness-specific browser skill at build time. The model-facing
|
||||
// skill deliberately documents only injected browser_* tools; the repository's CLI
|
||||
// skill is a separate interface and must never be concatenated here.
|
||||
// Output: src/skill-content.generated.ts (gitignored, built before tsdown and
|
||||
// before vitest) so the markdown ships inlined in the bundle and registration
|
||||
// is a pure in-memory read — never a per-step disk access.
|
||||
// before vitest), so registration is a pure in-memory read.
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const pkg = join(here, "..");
|
||||
const prelude = readFileSync(join(pkg, "skill", "prelude.md"), "utf8");
|
||||
const upstream = readFileSync(join(pkg, "..", "..", "skill", "SKILL.md"), "utf8");
|
||||
const postlude = readFileSync(join(pkg, "skill", "postlude.md"), "utf8");
|
||||
const source = readFileSync(join(pkg, "skill", "SKILL.md"), "utf8");
|
||||
|
||||
function stripFrontmatter(markdown) {
|
||||
if (!markdown.startsWith("---\n")) return markdown;
|
||||
@@ -22,18 +17,69 @@ function stripFrontmatter(markdown) {
|
||||
return end === -1 ? markdown : markdown.slice(end + 5).replace(/^\s*\n/, "");
|
||||
}
|
||||
|
||||
const SUPPORTED_BROWSER_TOOLS = [
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_session_list",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_reload",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_snapshot",
|
||||
"browser_observe",
|
||||
"browser_get_html",
|
||||
"browser_screenshot",
|
||||
"browser_console",
|
||||
"browser_network",
|
||||
"browser_click",
|
||||
"browser_hover",
|
||||
"browser_fill",
|
||||
"browser_select",
|
||||
"browser_press",
|
||||
"browser_tab_list",
|
||||
"browser_tab_create",
|
||||
"browser_tab_select",
|
||||
"browser_tab_close",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_return",
|
||||
"browser_request_help",
|
||||
"browser_emulate",
|
||||
"browser_window_resize",
|
||||
];
|
||||
|
||||
function validateModelFacingSkill(markdown) {
|
||||
if (/\bbsk\b/i.test(markdown)) {
|
||||
throw new Error("DSH skill must not expose the internal CLI name");
|
||||
}
|
||||
if (/```(?:bash|sh|shell)\b/i.test(markdown)) {
|
||||
throw new Error("DSH skill must not contain command-line code blocks");
|
||||
}
|
||||
|
||||
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
|
||||
const supported = new Set(SUPPORTED_BROWSER_TOOLS);
|
||||
const unknown = [...mentioned].filter((name) => !supported.has(name));
|
||||
const missing = SUPPORTED_BROWSER_TOOLS.filter((name) => !mentioned.has(name));
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`DSH skill omits supported browser tools: ${missing.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
const NAME = "browser-skill";
|
||||
const DESCRIPTION =
|
||||
"Browser automation against the user's logged-in Chromium through this plugin's injected browser_* tools; never invoke bsk through bash or shell. " +
|
||||
"tools (bsk engine): visit and read pages, fill forms, click through flows, scrape data, " +
|
||||
"smoke-test UI changes. Use when a task needs a real browser; requires the bsk CLI and the " +
|
||||
"browser-skill extension connected.";
|
||||
"Browser automation against the user's logged-in Chromium through this plugin's injected " +
|
||||
"browser_* tools. Use to visit and read pages, fill forms, click through flows, inspect tabs, " +
|
||||
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
|
||||
|
||||
const content = `${prelude.trim()}\n\n${stripFrontmatter(upstream).trim()}\n\n${postlude.trim()}\n`;
|
||||
const content = `${stripFrontmatter(source).trim()}\n`;
|
||||
validateModelFacingSkill(content);
|
||||
|
||||
const banner =
|
||||
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
|
||||
"// Sources: skill/prelude.md + ../../../skill/SKILL.md + skill/postlude.md.\n";
|
||||
"// Source: skill/SKILL.md (DeepSeek Harness browser_* tool instructions).\n";
|
||||
|
||||
writeFileSync(
|
||||
join(pkg, "src", "skill-content.generated.ts"),
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use the injected browser_* tools to automate the user's logged-in Chromium
|
||||
browser in a managed Agent Window.
|
||||
---
|
||||
|
||||
# browser-skill for DeepSeek Harness
|
||||
|
||||
Drive the user's real Chromium browser through the structured `browser_*` tools provided by this
|
||||
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
|
||||
Agent Window. Use the tool schemas exactly as exposed by DeepSeek Harness; arguments are structured
|
||||
fields, not command-line flags.
|
||||
|
||||
## Tool availability
|
||||
|
||||
Loading this skill reveals the browser tools for the rest of the conversation. If the tools were
|
||||
hidden before this skill was loaded, start browser work only after the skill invocation succeeds.
|
||||
|
||||
All browser operations must use the injected tools directly. Do not translate the examples in this
|
||||
document into terminal commands or invoke another process to control the browser. Direct tool calls
|
||||
preserve session ownership, the live observation window, tool cards, cancellation, and cleanup.
|
||||
|
||||
## When to use
|
||||
|
||||
- Visit and read pages that require the user's existing browser session.
|
||||
- Fill forms, select options, click through bounded flows, or smoke-test a web UI.
|
||||
- Inspect tabs, page structure, screenshots, console messages, or network metadata.
|
||||
- Work with a user tab after explicitly borrowing it into the Agent Window.
|
||||
|
||||
Do not use browser automation for tasks that need only local files, a purpose-built API, or an
|
||||
explanation. Never extract credentials, cookies, tokens, or other secrets. Do not keep control of a
|
||||
personal tab longer than the immediate task requires.
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every browser task follows this lifecycle:
|
||||
|
||||
1. Call `browser_session_start` and retain its returned `sessionId`.
|
||||
2. Use that session for every browser operation. Pass `session` explicitly when more than one
|
||||
session exists.
|
||||
3. Call `browser_session_stop` when the bounded goal is reached or an error ends the task. Treat
|
||||
cleanup like a `finally` path unless the user explicitly asks to keep the session open.
|
||||
|
||||
Conceptual tool-call sequence:
|
||||
|
||||
```text
|
||||
browser_session_start({ url: "https://example.com" })
|
||||
browser_observe({ session: sessionId })
|
||||
browser_click({ session: sessionId, target: "@e4" })
|
||||
browser_observe({ session: sessionId })
|
||||
browser_session_stop({ session: sessionId })
|
||||
```
|
||||
|
||||
The plugin owns only sessions created by `browser_session_start`. `browser_session_list` reports
|
||||
only those owned sessions. Never guess or reuse a session id from another program. When `session`
|
||||
is omitted, most tools use the most recently started or used owned session; explicit ids are safer
|
||||
for multi-session work.
|
||||
|
||||
## Stop when the goal is met
|
||||
|
||||
Browser work is a bounded task, not open-ended browsing.
|
||||
|
||||
1. Define one observable success condition from the user's request.
|
||||
2. Take the shortest path: observe, act, then perform at most one relevant check.
|
||||
3. Stop immediately when success is visible. Do not refresh, navigate, click, or perform extra
|
||||
verification after success.
|
||||
4. If a human-only step appears, use `browser_request_help`. If an action fails twice without
|
||||
progress, pause instead of brute-forcing.
|
||||
5. When uncertain, make at most one fresh observation. If the state is still ambiguous, ask the
|
||||
user rather than continuing to click.
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Use `browser_navigate` to open a destination in the active Agent Window tab, then inspect the page
|
||||
before interacting:
|
||||
|
||||
1. `browser_observe` is the primary semantic view. It returns roles, states, text, and `@eN` refs,
|
||||
including hints about conditional hover or focus surfaces.
|
||||
2. `browser_snapshot` is the static accessibility-tree fallback when the semantic view is
|
||||
insufficient or a stricter tree is useful.
|
||||
3. Prefer a fresh `@eN` ref over a raw CSS selector for `browser_click`, `browser_hover`,
|
||||
`browser_fill`, `browser_select`, and targeted `browser_press` calls.
|
||||
4. Observe again after navigation or a meaningful DOM change before using another ref.
|
||||
|
||||
Refs invalidate after navigation and may also become stale after large DOM changes. Never carry a
|
||||
ref to a new page. If an observation marks a control as hover-triggered, call `browser_hover` on the
|
||||
trigger, observe again, and then act on the newly visible ref.
|
||||
|
||||
Use interaction tools according to their schemas:
|
||||
|
||||
- `browser_click`: click a ref or selector; optionally choose mouse button and click count.
|
||||
- `browser_hover`: move to a ref or selector and allow hover UI to settle; then observe again.
|
||||
- `browser_fill`: clear and fill a text-capable element; set `noClear` only when appending is
|
||||
intentional.
|
||||
- `browser_select`: pass `values` as an array, including for a single option.
|
||||
- `browser_press`: send a key or combo, optionally focusing `target` first.
|
||||
|
||||
Actions may trigger a navigation after the action itself returns. Use
|
||||
`browser_wait_for_navigation` only when that lifecycle wait is actually needed, then observe the
|
||||
new page.
|
||||
|
||||
## Observation and page reading priority
|
||||
|
||||
Escalate page reading only as needed:
|
||||
|
||||
1. `browser_observe` for normal semantic understanding and interaction refs.
|
||||
2. `browser_snapshot` for a stricter static accessibility tree.
|
||||
3. `browser_get_html` for hidden DOM, metadata, or exact markup that the semantic views cannot
|
||||
provide. A `ref` must come from a fresh observation; keep `maxBytes` bounded.
|
||||
4. `browser_screenshot` for layout, styling, canvas, image content, or another genuinely visual
|
||||
question. Pass a fresh `ref` to crop to one element, or omit it for the visible tab.
|
||||
|
||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls. Screenshots may
|
||||
return an inline image or a saved path depending on the deployment.
|
||||
|
||||
## Tabs and the Agent Window boundary
|
||||
|
||||
Write actions normally affect only Agent Window tabs.
|
||||
|
||||
- `browser_tab_list` lists `agent`, `user`, or `all` tabs visible to the owned session. Use
|
||||
`scope: "user"` to locate a user tab before borrowing it.
|
||||
- `browser_tab_create` creates an Agent Window tab. It is active by default; set `active: false`
|
||||
for a background tab.
|
||||
- `browser_tab_select` focuses an Agent Window tab.
|
||||
- `browser_tab_close` closes an Agent Window tab.
|
||||
- `browser_tab_borrow` moves a chosen user tab into the Agent Window before any write action.
|
||||
- `browser_tab_return` returns a borrowed tab to its original window and position. Return it as
|
||||
soon as the immediate task is finished; stopping the session is only the fallback cleanup.
|
||||
|
||||
Use only a `tabId` returned by `browser_tab_list` or `browser_tab_create`. Do not assume a tab id
|
||||
from page position or title.
|
||||
|
||||
## Navigation and lifecycle
|
||||
|
||||
- `browser_navigate` opens a URL and can wait for a requested lifecycle phase.
|
||||
- `browser_navigate_back` and `browser_navigate_forward` move one history entry.
|
||||
- `browser_reload` reloads the current tab; use `hard: true` only when bypassing cache is necessary.
|
||||
- `browser_wait_for_navigation` waits for a lifecycle event after an action that did not already
|
||||
wait. Avoid speculative waits when no navigation is expected.
|
||||
|
||||
After navigation, history traversal, or reload, discard old refs and call `browser_observe` or
|
||||
`browser_snapshot` again.
|
||||
|
||||
## Ask the user for browser help
|
||||
|
||||
Call `browser_request_help` for login, captcha, OTP, payment confirmation, consent, or another step
|
||||
that requires the user. Give a clear `prompt`; when the prompt refers to a concrete control, pass
|
||||
its fresh ref or selector in `targets` so the overlay can highlight it. Use `title` only when it
|
||||
adds clarity.
|
||||
|
||||
Optional `completionCriteria` can finish the pause automatically when the page has an explicit,
|
||||
stable success signal. It supports `any` or `all` conditions using `urlContains`, `urlMatches`,
|
||||
`selectorExists`, `selectorMissing`, `textExists`, or `textMissing`, plus `stableForMs`.
|
||||
|
||||
```text
|
||||
browser_request_help({
|
||||
session: sessionId,
|
||||
title: "Sign in required",
|
||||
prompt: "Complete sign-in, then return control after the account page appears.",
|
||||
targets: ["@e7"],
|
||||
timeoutMs: 300000,
|
||||
completionCriteria: {
|
||||
any: [{ urlContains: "/account" }, { selectorExists: "[data-testid='account-menu']" }],
|
||||
stableForMs: 1000
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
Interpret outcomes deliberately:
|
||||
|
||||
- `continued` or `completed`: the user returned control or the explicit detector matched. Observe
|
||||
again before reasoning about the new state or using refs.
|
||||
- `cancelled`: stop the requested flow.
|
||||
- `timed_out`: report the timeout; do not repeatedly reopen the prompt.
|
||||
- `disabled`: unattended mode does not allow help; proceed only if safe or stop gracefully.
|
||||
- `navigated`: legacy outcome; do not treat navigation alone as proof of success.
|
||||
|
||||
## Debugging without page scripting
|
||||
|
||||
Use the read-only debugging tools only when relevant to the user's task:
|
||||
|
||||
- `browser_console` reads buffered console messages, browser logs, and JavaScript exceptions.
|
||||
Set `includeStack: true` when stack frames are useful.
|
||||
- `browser_network` reads buffered response and failure metadata. It does not return request or
|
||||
response headers or bodies and does not modify traffic.
|
||||
|
||||
Both tools accept `tabId`, `since`, `limit`, and `maxTextChars`. Their result returns `nextSince`;
|
||||
pass that value as the next `since` cursor instead of repeatedly requesting the same entries.
|
||||
|
||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported in this
|
||||
DeepSeek Harness plugin. Do not invent tool names or route around this limitation. If the task
|
||||
strictly requires either capability, explain that it is not available in this phase.
|
||||
|
||||
## Window and device controls
|
||||
|
||||
- `browser_window_resize` changes the Agent Window's outer `width` and `height`; each dimension
|
||||
must be from 100 through 7680 CSS pixels.
|
||||
- `browser_emulate` applies a built-in device preset or explicit viewport settings to the active
|
||||
tab. Explicit `width` and `height` must be supplied together. `mobile: true` also requires both
|
||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per-tab and new tabs do not
|
||||
inherit it.
|
||||
|
||||
## Tool reference
|
||||
|
||||
### Sessions
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_session_start` | Start an owned Agent Window session, optionally with URL, size, browser instance, focus behavior, or device preset. |
|
||||
| `browser_session_stop` | Stop an owned session and close its Agent Window. |
|
||||
| `browser_session_list` | List owned sessions and identify the current one. |
|
||||
|
||||
### Navigation and reading
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_navigate` | Navigate the active tab to a URL. |
|
||||
| `browser_navigate_back` | Go back one history entry. |
|
||||
| `browser_navigate_forward` | Go forward one history entry. |
|
||||
| `browser_reload` | Reload the active tab, optionally bypassing cache. |
|
||||
| `browser_wait_for_navigation` | Wait for an expected page lifecycle event. |
|
||||
| `browser_observe` | Read the primary semantic page representation with refs. |
|
||||
| `browser_snapshot` | Read the static accessibility-tree fallback with refs. |
|
||||
| `browser_get_html` | Read raw document or ref-scoped HTML as a last resort. |
|
||||
| `browser_screenshot` | Capture the visible tab or a ref-cropped element. |
|
||||
|
||||
### Interaction
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_click` | Click a ref or CSS selector. |
|
||||
| `browser_hover` | Hover a ref or selector to reveal conditional UI. |
|
||||
| `browser_fill` | Fill a text-capable element. |
|
||||
| `browser_select` | Select one or more option values. |
|
||||
| `browser_press` | Send a keyboard key or combo. |
|
||||
|
||||
### Tabs
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_tab_list` | List visible user and Agent Window tabs. |
|
||||
| `browser_tab_create` | Create an Agent Window tab. |
|
||||
| `browser_tab_select` | Focus an Agent Window tab. |
|
||||
| `browser_tab_close` | Close an Agent Window tab. |
|
||||
| `browser_tab_borrow` | Move a user tab into the Agent Window. |
|
||||
| `browser_tab_return` | Return a borrowed tab to its original window. |
|
||||
|
||||
### Assistance, diagnostics, and display
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_request_help` | Pause for an explicit user browser step. |
|
||||
| `browser_console` | Read buffered console and exception entries. |
|
||||
| `browser_network` | Read buffered response and failure metadata. |
|
||||
| `browser_window_resize` | Resize the Agent Window. |
|
||||
| `browser_emulate` | Apply or clear per-tab device emulation. |
|
||||
|
||||
## Error handling and final checklist
|
||||
|
||||
On a stale ref, observe again and retry once with a fresh ref. On a closed or unknown tab, list
|
||||
tabs instead of guessing. On an unknown session, list owned sessions or start a new one; never try
|
||||
foreign ids. On timeout, decide whether one longer, purposeful wait is justified. Avoid repeated
|
||||
actions with unchanged state.
|
||||
|
||||
Before finishing a browser task, verify that:
|
||||
|
||||
- The user's observable goal is met, or the blocker is clearly reported.
|
||||
- Any borrowed tab has been returned when practical.
|
||||
- The owned session has been stopped unless the user explicitly asked to keep it open.
|
||||
- No further page action is taken after success.
|
||||
@@ -1,10 +0,0 @@
|
||||
---
|
||||
|
||||
## DSH routing reminder
|
||||
|
||||
The CLI-oriented reference above describes behavior, not the execution mechanism in
|
||||
DeepSeek Harness. For every browser operation, call the injected `browser_*` model
|
||||
tool. Do not translate these examples into `bash` commands and do not execute `bsk`
|
||||
directly; the plugin invokes `bsk` internally. In particular, always begin with
|
||||
`browser_session_start`, pass its returned session id to subsequent `browser_*`
|
||||
tools, and finish with `browser_session_stop`.
|
||||
@@ -1,71 +0,0 @@
|
||||
# browser-skill (dsh edition)
|
||||
|
||||
## DSH tool-routing requirement (highest priority)
|
||||
|
||||
This skill is running inside DeepSeek Harness, where the plugin has injected
|
||||
structured `browser_*` model tools. **Always call those injected `browser_*` tools
|
||||
for browser work. Never use `bash`, `shell`, `exec`, or another tool to invoke the
|
||||
`bsk` CLI directly.** The CLI command names shown below are only conceptual
|
||||
mappings for understanding the tools; they are not instructions to run commands.
|
||||
If a `browser_*` tool is available, using the CLI directly is incorrect because it
|
||||
bypasses plugin session ownership, observation UI, tool cards, and cleanup.
|
||||
|
||||
Use the injected tool whose name matches the operation, for example:
|
||||
`browser_session_start`, `browser_navigate`, `browser_observe`, `browser_snapshot`,
|
||||
`browser_click`, `browser_fill`, `browser_press`, `browser_screenshot`, and
|
||||
`browser_session_stop`. Pass arguments as the tool schema specifies (not CLI flags).
|
||||
|
||||
Drive the user's **real Chromium browser** (logins, cookies, Agent Window isolation)
|
||||
through this plugin's `browser_*` tools. Each injected tool is implemented by the
|
||||
plugin using the `bsk` engine internally — the model must not invoke that engine
|
||||
itself. The workflow, judgment rules, and safety constraints remain identical to
|
||||
the canonical CLI skill.
|
||||
|
||||
> **Tool availability**: invoking this skill (which you just did) makes the plugin's
|
||||
> `browser_*` tools available for the rest of the session — they are injected on
|
||||
> successful skill load. Until then they are deliberately hidden, so do not attempt
|
||||
> a browser action before loading this skill.
|
||||
|
||||
## Tool ↔ CLI map
|
||||
|
||||
The tools below are thin, structured wrappers over the bsk CLI verbs of the same
|
||||
name; `session` parameters take the 4-letter session id (no `--session` flag
|
||||
spelling here):
|
||||
|
||||
| dsh tool | bsk equivalent |
|
||||
|---|---|
|
||||
| `browser_session_start` / `browser_session_stop` / `browser_session_list` | `bsk session start/stop/list` |
|
||||
| `browser_navigate` | `bsk navigate` |
|
||||
| `browser_snapshot` / `browser_observe` | `bsk snapshot` / `bsk observe` |
|
||||
| `browser_click` / `browser_fill` / `browser_press` | `bsk click` / `bsk fill` / `bsk press` |
|
||||
| `browser_screenshot` | `bsk screenshot` |
|
||||
| `browser_emulate` | `bsk emulate` |
|
||||
|
||||
(`bsk console` / `bsk network` have no tool counterpart in this plugin yet.)
|
||||
|
||||
Plugin-specific semantics that override or narrow the CLI rules below:
|
||||
|
||||
- **Owned sessions only**: this plugin sees and acts on just the sessions IT
|
||||
started via `browser_session_start`. `browser_session_list` shows only those
|
||||
(with the current marker); passing a session id created by another program
|
||||
(or person) sharing the bsk daemon is refused outright — never retry those
|
||||
with the same id.
|
||||
- **Current session**: omitted `session` parameters resolve to the most
|
||||
recently started or used OWNED session. With several sessions in flight,
|
||||
pass `session` explicitly to avoid surprises.
|
||||
- **Screenshots**: when the model route accepts images, `browser_screenshot`
|
||||
returns the PNG inline as an image attachment; otherwise it returns the
|
||||
saved file path.
|
||||
- **`browser_emulate mobile`**: requires `width`+`height` — the bsk daemon
|
||||
refuses `--mobile` without viewport dimensions.
|
||||
- **Live observation**: in the dsh Web UI, every owned session is watched by
|
||||
the floating overlay (thumbnails, action, interrupt) — observational side
|
||||
traffic is free and never steals your command turn; just keep acting.
|
||||
|
||||
The canonical CLI skill body follows verbatim (source of truth: `skill/SKILL.md`
|
||||
in the BrowserSkill repo, mirrored by build into `crates/bsk-cli/skill/SKILL.md`).
|
||||
Read `bsk <verb> --session <id>` as "the `browser_<verb>` tool with
|
||||
`session: <id>`"; everything else — when to use, the stop-when-done rules, the
|
||||
interaction loop, refs, sandbox rules — applies unchanged.
|
||||
|
||||
---
|
||||
@@ -19,7 +19,7 @@ import { registerObservationRoutes } from "./observation-http";
|
||||
import { KeyedExecutor } from "./queue";
|
||||
import { type BskRunner, createBskRunner } from "./runner";
|
||||
import { SessionRegistry } from "./sessions";
|
||||
import { registerBskSkill } from "./skill";
|
||||
import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
|
||||
import { type PluginConfig, registerTools } from "./tools";
|
||||
|
||||
export const name = "dsh-plugin-browserskill";
|
||||
@@ -95,6 +95,11 @@ export function apply(
|
||||
// ONLY model-visible advertisement — the tool suite reveals itself on a
|
||||
// successful skill invocation (or a session whose history already has one).
|
||||
const unregisterSkill = registerBskSkill(ctx);
|
||||
// A same-name CLI skill left in ~/.agents is discovered in the nearer preset
|
||||
// layer and shadows the global registration above. Re-register through every
|
||||
// exact agent context at startup so DSH always loads the browser_* protocol
|
||||
// instructions; the shared CLI skill remains untouched for other agents.
|
||||
const disarmAgentSkill = armAgentScopedBskSkill(ctx);
|
||||
const removeSuite = resolved.lazyTools
|
||||
? armLazyTools(ctx, () =>
|
||||
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
|
||||
@@ -135,17 +140,15 @@ export function apply(
|
||||
ctx.effect(() => {
|
||||
return () => {
|
||||
removeSuite();
|
||||
disarmAgentSkill();
|
||||
unregisterSkill();
|
||||
removeRoutes();
|
||||
disarmArchiveCleanup();
|
||||
observation.dispose();
|
||||
runner.killAll();
|
||||
const stops = registry
|
||||
.ownedIds()
|
||||
.map((sessionId) =>
|
||||
runner.run(["session", "stop", sessionId], { timeoutMs: 15_000 }).catch(() => {}),
|
||||
);
|
||||
return Promise.all(stops).then(() => {});
|
||||
.map((sessionId) => observation.stopSession(sessionId).catch(() => false));
|
||||
return Promise.all(stops).then(() => observation.dispose());
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Lazy tool-schema injection ("progressive disclosure", final stage): with
|
||||
* `lazyTools` on (the default), the 11 browser_* tool schemas stay OUT of the
|
||||
* `lazyTools` on (the default), the 28 browser_* tool schemas stay OUT of the
|
||||
* system prompt until the `browser-skill` skill has actually been invoked —
|
||||
* the skill catalog entry is the only advertisement. One successful
|
||||
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
|
||||
@@ -96,7 +96,7 @@ export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]
|
||||
/**
|
||||
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
|
||||
* the reveal already happened — the tool suite itself.
|
||||
* @param registerSuite - registers the 11 browser_* tools, returns their disposer.
|
||||
* @param registerSuite - registers the 28 browser_* tools, returns their disposer.
|
||||
*/
|
||||
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
|
||||
let suiteDisposer: (() => void) | undefined;
|
||||
|
||||
@@ -18,7 +18,7 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { KeyedExecutor } from "./queue";
|
||||
import type { BskRunner } from "./runner";
|
||||
import { BskError, type BskRunner, parseBskJson, runWithSessionBusyRetry } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
/** One owned session's live observation record (wire-stable shape). */
|
||||
@@ -90,6 +90,10 @@ export class ObservationService {
|
||||
private readonly listeners = new Set<(event: ObservationEvent) => void>();
|
||||
private readonly captureTimers = new Map<string, unknown>();
|
||||
private readonly captureInFlight = new Set<string>();
|
||||
/** Captures intentionally cancelled to make way for foreground work. */
|
||||
private readonly capturePreempted = new Set<string>();
|
||||
/** Number of queued/running model-facing calls that currently outrank thumbnails. */
|
||||
private readonly foregroundDepth = new Map<string, number>();
|
||||
private readonly captureFailures = new Map<string, number>();
|
||||
private readonly lastActivity = new Map<string, number>();
|
||||
/** Ephemeral frame id → PNG bytes. Only the live ring members are present. */
|
||||
@@ -175,6 +179,8 @@ export class ObservationService {
|
||||
/** Drop a session (called from browser_session_stop). */
|
||||
removeSession(sessionId: string): void {
|
||||
this.cancelCapture(sessionId);
|
||||
this.foregroundDepth.delete(sessionId);
|
||||
this.capturePreempted.delete(sessionId);
|
||||
this.captureFailures.delete(sessionId);
|
||||
this.lastActivity.delete(sessionId);
|
||||
this.dropSessionFrames(sessionId);
|
||||
@@ -186,6 +192,37 @@ export class ObservationService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Give model-facing work priority over the best-effort thumbnail lane.
|
||||
* The first lease cancels a pending timer and gracefully interrupts an
|
||||
* active bsk screenshot; the last release schedules one fresh frame.
|
||||
*/
|
||||
acquireForeground(sessionId: string): () => void {
|
||||
if (!this.deps.options.enabled || this.disposed || !this.observations.has(sessionId)) {
|
||||
return () => {};
|
||||
}
|
||||
const depth = this.foregroundDepth.get(sessionId) ?? 0;
|
||||
this.foregroundDepth.set(sessionId, depth + 1);
|
||||
if (depth === 0) {
|
||||
this.cancelCapture(sessionId);
|
||||
if (this.deps.runner.killFor(`observation:${sessionId}`) > 0) {
|
||||
this.capturePreempted.add(sessionId);
|
||||
}
|
||||
}
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
const remaining = (this.foregroundDepth.get(sessionId) ?? 1) - 1;
|
||||
if (remaining > 0) {
|
||||
this.foregroundDepth.set(sessionId, remaining);
|
||||
return;
|
||||
}
|
||||
this.foregroundDepth.delete(sessionId);
|
||||
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId, 0);
|
||||
};
|
||||
}
|
||||
|
||||
/** Mark an action starting on a session (tool entry instrumentation). */
|
||||
beginAction(sessionId: string, action: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
@@ -213,7 +250,7 @@ export class ObservationService {
|
||||
if (error !== undefined) next.lastError = error;
|
||||
else delete next.lastError;
|
||||
this.put(next);
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
if (!this.foregroundDepth.has(sessionId)) this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Record the settled page URL (navigate success / start with url). */
|
||||
@@ -242,29 +279,48 @@ export class ObservationService {
|
||||
* button — same end state as `browser_session_stop`). Never waits behind a
|
||||
* hung in-flight command: tool children are killed first so the session's
|
||||
* keyed queue drains immediately, and no further captures queue up. A
|
||||
* session the daemon already forgot (dead strip entries) stops
|
||||
* idempotently — the goal state (entry gone) is identical.
|
||||
* @returns whether the session ended up stopped/removed.
|
||||
* session the daemon already forgot stops idempotently — the goal state
|
||||
* (entry gone) is identical.
|
||||
* @returns false only for a foreign session; bsk failures reject so callers
|
||||
* can preserve the structured error instead of silently leaving a ghost.
|
||||
*/
|
||||
async stopSession(sessionId: string): Promise<boolean> {
|
||||
async stopSession(sessionId: string, signal?: AbortSignal): Promise<boolean> {
|
||||
if (!this.deps.registry.isOwned(sessionId)) return false;
|
||||
this.cancelCapture(sessionId);
|
||||
this.deps.runner.killFor(sessionId);
|
||||
const result = await this.deps.queue.run(sessionId, () =>
|
||||
this.deps.runner.run(["session", "stop", sessionId], { timeoutMs: 30_000 }),
|
||||
);
|
||||
if (result.code !== 0) {
|
||||
let code: string | undefined;
|
||||
const releaseForeground = this.acquireForeground(sessionId);
|
||||
let actionError: string | undefined;
|
||||
this.beginAction(sessionId, "stopping");
|
||||
try {
|
||||
this.deps.runner.killFor(sessionId);
|
||||
const result = await this.deps.queue.run(
|
||||
sessionId,
|
||||
() =>
|
||||
runWithSessionBusyRetry(
|
||||
() =>
|
||||
this.deps.runner.run(["session", "stop", sessionId], {
|
||||
signal,
|
||||
timeoutMs: 30_000,
|
||||
tag: sessionId,
|
||||
}),
|
||||
signal,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
if (result.aborted) throw abortError();
|
||||
try {
|
||||
code = (JSON.parse(result.stdout) as { code?: string }).code;
|
||||
} catch {
|
||||
code = undefined;
|
||||
parseBskJson(result, "session stop");
|
||||
} catch (error) {
|
||||
if (!isSessionNotFoundError(error)) throw error;
|
||||
}
|
||||
if (code !== "session_not_found") return false;
|
||||
this.deps.registry.remove(sessionId);
|
||||
this.removeSession(sessionId);
|
||||
return true;
|
||||
} catch (error) {
|
||||
actionError = error instanceof Error ? error.message.split("\n")[0] : String(error);
|
||||
throw error;
|
||||
} finally {
|
||||
this.endAction(sessionId, actionError);
|
||||
releaseForeground();
|
||||
}
|
||||
this.deps.registry.remove(sessionId);
|
||||
this.removeSession(sessionId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -287,6 +343,8 @@ export class ObservationService {
|
||||
for (const sessionId of [...this.captureTimers.keys()]) this.cancelCapture(sessionId);
|
||||
this.captureTimers.clear();
|
||||
this.captureInFlight.clear();
|
||||
this.capturePreempted.clear();
|
||||
this.foregroundDepth.clear();
|
||||
this.captureFailures.clear();
|
||||
this.lastActivity.clear();
|
||||
this.frames.clear();
|
||||
@@ -313,6 +371,7 @@ export class ObservationService {
|
||||
/** Schedule the next capture for a session; `delayMs` 0 means "as soon as the event loop allows". */
|
||||
private scheduleCapture(sessionId: string, delayMs?: number): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
if (this.foregroundDepth.has(sessionId)) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined || entry.dead === true) return;
|
||||
this.cancelCapture(sessionId);
|
||||
@@ -349,12 +408,15 @@ export class ObservationService {
|
||||
const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
|
||||
let writtenPath = outPath;
|
||||
try {
|
||||
const result = await this.deps.queue.run(sessionId, () =>
|
||||
this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
|
||||
const result = await this.deps.queue.run(sessionId, () => {
|
||||
// A foreground lease may have arrived while this capture was queued.
|
||||
if (this.foregroundDepth.has(sessionId)) return Promise.resolve(undefined);
|
||||
return this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
|
||||
timeoutMs: 15_000,
|
||||
tag: `observation:${sessionId}`,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
if (result === undefined) return;
|
||||
if (result.code !== 0) {
|
||||
let code: string | undefined;
|
||||
try {
|
||||
@@ -362,8 +424,12 @@ export class ObservationService {
|
||||
} catch {
|
||||
code = undefined;
|
||||
}
|
||||
if (code === "session_not_found") {
|
||||
this.markDead(sessionId);
|
||||
if (isSessionNotFoundCode(code)) {
|
||||
// Observation entries are owned sessions. If the daemon has already
|
||||
// forgotten one (external close / restart), reconcile both stores so
|
||||
// the PiP cannot retain a ghost strip or keep retrying screenshots.
|
||||
this.deps.registry.remove(sessionId);
|
||||
this.removeSession(sessionId);
|
||||
return;
|
||||
}
|
||||
throw new Error(`screenshot exited ${result.code}`);
|
||||
@@ -378,12 +444,17 @@ export class ObservationService {
|
||||
this.setAvailable(true);
|
||||
this.publishFrame(sessionId, entry, data);
|
||||
} catch {
|
||||
// Silent by design: keep the previous frame, count toward backoff.
|
||||
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
|
||||
this.globalFailures += 1;
|
||||
if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
|
||||
// Foreground preemption is healthy scheduling, not browser/daemon
|
||||
// unavailability; do not let normal tool traffic trip capture backoff.
|
||||
if (!this.capturePreempted.delete(sessionId)) {
|
||||
// Silent by design: keep the previous frame, count toward backoff.
|
||||
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
|
||||
this.globalFailures += 1;
|
||||
if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
|
||||
}
|
||||
} finally {
|
||||
await unlink(writtenPath).catch(() => {});
|
||||
this.capturePreempted.delete(sessionId);
|
||||
this.captureInFlight.delete(sessionId);
|
||||
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
|
||||
}
|
||||
@@ -414,14 +485,20 @@ export class ObservationService {
|
||||
this.hashes.delete(sessionId);
|
||||
this.seqs.delete(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark a session dead: grey it out, stop asking for frames, keep the entry for the strip. */
|
||||
private markDead(sessionId: string): void {
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined || entry.dead === true) return;
|
||||
this.cancelCapture(sessionId);
|
||||
this.put({ ...entry, dead: true, action: "idle" });
|
||||
}
|
||||
function isSessionNotFoundCode(code: string | undefined): boolean {
|
||||
return code === "not_found" || code === "session_not_found";
|
||||
}
|
||||
|
||||
function isSessionNotFoundError(error: unknown): boolean {
|
||||
return error instanceof BskError && isSessionNotFoundCode(error.code);
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Map a bsk command label onto its observation action verb. */
|
||||
@@ -439,14 +516,48 @@ export function actionForLabel(label: string): string {
|
||||
return "observing";
|
||||
case "click":
|
||||
return "clicking";
|
||||
case "hover":
|
||||
return "hovering";
|
||||
case "fill":
|
||||
return "filling";
|
||||
case "select":
|
||||
return "selecting";
|
||||
case "press":
|
||||
return "pressing";
|
||||
case "screenshot":
|
||||
return "capturing";
|
||||
case "emulate":
|
||||
return "emulating";
|
||||
case "tab list":
|
||||
return "listing tabs";
|
||||
case "tab create":
|
||||
return "creating tab";
|
||||
case "tab close":
|
||||
return "closing tab";
|
||||
case "tab select":
|
||||
return "selecting tab";
|
||||
case "tab borrow":
|
||||
return "borrowing tab";
|
||||
case "tab return":
|
||||
return "returning tab";
|
||||
case "navigate-back":
|
||||
return "navigating back";
|
||||
case "navigate-forward":
|
||||
return "navigating forward";
|
||||
case "reload":
|
||||
return "reloading";
|
||||
case "wait-for-navigation":
|
||||
return "waiting for navigation";
|
||||
case "request-help":
|
||||
return "waiting for user";
|
||||
case "get-html":
|
||||
return "reading HTML";
|
||||
case "console":
|
||||
return "reading console";
|
||||
case "network":
|
||||
return "reading network";
|
||||
case "window resize":
|
||||
return "resizing window";
|
||||
default:
|
||||
return label;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { ToolDefinition, ToolResult, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
export type ToolRegistrar = (definition: ToolDefinition) => () => void;
|
||||
|
||||
type TerminalPresentation = { card: "terminal"; output: string; exitCode: number } | undefined;
|
||||
|
||||
/** Existing tool runtime seams reused by the phase-one capability modules. */
|
||||
export interface PhaseOneRuntime {
|
||||
run(
|
||||
exec: ToolRunContext,
|
||||
args: string[],
|
||||
label: string,
|
||||
observeSession?: string,
|
||||
runnerTimeoutMs?: number,
|
||||
): Promise<unknown>;
|
||||
commandLine(args: string[]): string;
|
||||
presentTerminalResult: (_args: never, result: ToolResult) => TerminalPresentation;
|
||||
}
|
||||
|
||||
export function requireNonEmpty(value: string, name: string): void {
|
||||
if (value.trim().length === 0) throw new Error(`${name} must be a non-empty string`);
|
||||
}
|
||||
|
||||
export function requirePositive(value: number | undefined, name: string): void {
|
||||
if (value !== undefined && value <= 0) throw new Error(`${name} must be greater than zero`);
|
||||
}
|
||||
|
||||
/** CLI positional target detection is shared by hover/select/request-help. */
|
||||
export function isSnapshotRef(target: string): boolean {
|
||||
return /^@?e\d+$/.test(target);
|
||||
}
|
||||
|
||||
/**
|
||||
* Give the child enough time to honour a command-level timeout plus IPC
|
||||
* settlement slack, without shortening the plugin's configured default.
|
||||
*/
|
||||
export function runnerTimeout(deps: ToolDeps, commandTimeoutMs: number | undefined): number {
|
||||
if (commandTimeoutMs === undefined) return deps.config.defaultTimeoutMs;
|
||||
return Math.max(deps.config.defaultTimeoutMs, commandTimeoutMs + 15_000);
|
||||
}
|
||||
|
||||
export function appendTarget(args: string[], target: string): void {
|
||||
args.push(target);
|
||||
}
|
||||
|
||||
export function appendTabId(args: string[], tabId: number | undefined): void {
|
||||
if (tabId !== undefined) args.push("--tab-id", String(tabId));
|
||||
}
|
||||
|
||||
export function appendWaitOptions(
|
||||
args: string[],
|
||||
waitUntil: string | undefined,
|
||||
timeoutMs: number | undefined,
|
||||
): void {
|
||||
if (waitUntil !== undefined) args.push("--wait-until", waitUntil);
|
||||
if (timeoutMs !== undefined) args.push("--timeout", `${timeoutMs}ms`);
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
import { defineTool } from "@deepseek-ai/dsh-tools";
|
||||
import {
|
||||
appendTabId,
|
||||
appendTarget,
|
||||
type PhaseOneRuntime,
|
||||
requireNonEmpty,
|
||||
requirePositive,
|
||||
runnerTimeout,
|
||||
type ToolRegistrar,
|
||||
} from "./phase-one-runtime";
|
||||
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM } from "./tool-params";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
|
||||
|
||||
/** Add hover and select without bypassing session ownership or observation. */
|
||||
export function registerPhaseOneInteractionTools(
|
||||
deps: ToolDeps,
|
||||
register: ToolRegistrar,
|
||||
runtime: PhaseOneRuntime,
|
||||
): void {
|
||||
const { registry } = deps;
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_hover",
|
||||
description:
|
||||
"Move the mouse over a snapshot ref or CSS selector to reveal hover-triggered UI. " +
|
||||
"Run browser_observe or browser_snapshot again afterwards to discover newly visible refs.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the element to hover.",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
modifiers: {
|
||||
type: "array",
|
||||
items: { type: "string", enum: MODIFIERS },
|
||||
description: "Keyboard modifiers held during the mouse move.",
|
||||
},
|
||||
settleMs: {
|
||||
type: "integer",
|
||||
description: "Milliseconds to wait for hover-triggered UI to settle (default: 200).",
|
||||
},
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
x: { type: "number", required: true },
|
||||
y: { type: "number", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] hovered at (${value.x}, ${value.y}) on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
requirePositive(args.settleMs, "settleMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_hover");
|
||||
const cmdArgs = ["hover", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
||||
cmdArgs.push("--modifiers", args.modifiers.join(","));
|
||||
}
|
||||
if (args.settleMs !== undefined) cmdArgs.push("--settle", `${args.settleMs}ms`);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
appendTarget(cmdArgs, args.target);
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
"hover",
|
||||
sessionId,
|
||||
runnerTimeout(deps, args.timeoutMs),
|
||||
)) as { tab_id: number; x: number; y: number };
|
||||
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"hover",
|
||||
args.target,
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: "Hover an element",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_select",
|
||||
description:
|
||||
"Set a select element's option values. Pass one value for a normal select or multiple " +
|
||||
"values for a multi-select; values replace the current selection.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the select element.",
|
||||
},
|
||||
values: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: { type: "string" },
|
||||
description: "Option value attributes to select; at least one value is required.",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
multiple: { type: "boolean", required: true },
|
||||
selectedValues: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: { type: "string" },
|
||||
},
|
||||
selectedLabels: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] selected ` +
|
||||
`${value.selectedValues.join(", ") || "(none)"} on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
if (args.values.length === 0) throw new Error("values must contain at least one option");
|
||||
const sessionId = registry.resolve(args.session, "browser_select");
|
||||
const cmdArgs = ["select", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
for (const value of args.values) cmdArgs.push("--value", value);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
appendTarget(cmdArgs, args.target);
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
"select",
|
||||
sessionId,
|
||||
runnerTimeout(deps, args.timeoutMs),
|
||||
)) as {
|
||||
tab_id: number;
|
||||
multiple: boolean;
|
||||
selected_values: string[];
|
||||
selected_labels: string[];
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
multiple: reply.multiple,
|
||||
selectedValues: reply.selected_values,
|
||||
selectedLabels: reply.selected_labels,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"select",
|
||||
args.target,
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: `Select ${args.values.length} option${args.values.length === 1 ? "" : "s"}`,
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
import { defineTool } from "@deepseek-ai/dsh-tools";
|
||||
import {
|
||||
appendTabId,
|
||||
appendWaitOptions,
|
||||
type PhaseOneRuntime,
|
||||
requirePositive,
|
||||
runnerTimeout,
|
||||
type ToolRegistrar,
|
||||
} from "./phase-one-runtime";
|
||||
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
interface RawHistoryResult {
|
||||
tab_id: number;
|
||||
previous_url?: string;
|
||||
final_url?: string;
|
||||
reached: string;
|
||||
error_text?: string;
|
||||
}
|
||||
|
||||
const HISTORY_OUTPUT = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
previousUrl: { type: "string" },
|
||||
finalUrl: { type: "string" },
|
||||
reached: { type: "string", required: true },
|
||||
errorText: { type: "string" },
|
||||
},
|
||||
} as const;
|
||||
|
||||
function mapHistory(sessionId: string, reply: RawHistoryResult) {
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
...(reply.previous_url !== undefined ? { previousUrl: reply.previous_url } : {}),
|
||||
...(reply.final_url !== undefined ? { finalUrl: reply.final_url } : {}),
|
||||
reached: reply.reached,
|
||||
...(reply.error_text !== undefined ? { errorText: reply.error_text } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Add browser history, reload, and explicit lifecycle waiting. */
|
||||
export function registerPhaseOneNavigationTools(
|
||||
deps: ToolDeps,
|
||||
register: ToolRegistrar,
|
||||
runtime: PhaseOneRuntime,
|
||||
): void {
|
||||
const { registry } = deps;
|
||||
|
||||
const registerHistory = (direction: "back" | "forward") => {
|
||||
const toolName = `browser_navigate_${direction}` as const;
|
||||
const command = `navigate-${direction}`;
|
||||
register(
|
||||
defineTool({
|
||||
name: toolName,
|
||||
description: `Navigate the active Agent Window tab ${direction} by one history entry.`,
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
waitUntil: WAIT_UNTIL_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
},
|
||||
output: {
|
||||
schema: HISTORY_OUTPUT,
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] navigated ${direction} on tab ${value.tabId}` +
|
||||
(value.finalUrl !== undefined ? ` to ${value.finalUrl}` : "") +
|
||||
` (reached: ${value.reached})` +
|
||||
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, toolName);
|
||||
const cmdArgs = [command, "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
command,
|
||||
sessionId,
|
||||
runnerTimeout(deps, args.timeoutMs),
|
||||
)) as RawHistoryResult;
|
||||
if (reply.final_url !== undefined) deps.observation.setUrl(sessionId, reply.final_url);
|
||||
return mapHistory(sessionId, reply);
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([command, "--session", args.session ?? "(current)"]),
|
||||
description: `Navigate ${direction}`,
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
};
|
||||
registerHistory("back");
|
||||
registerHistory("forward");
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_reload",
|
||||
description:
|
||||
"Reload the active Agent Window tab and wait for a lifecycle phase. Set hard=true to " +
|
||||
"bypass the HTTP cache.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
waitUntil: WAIT_UNTIL_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
hard: { type: "boolean", description: "Bypass the HTTP cache while reloading." },
|
||||
},
|
||||
output: {
|
||||
schema: HISTORY_OUTPUT,
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] reloaded tab ${value.tabId}` +
|
||||
(value.finalUrl !== undefined ? ` at ${value.finalUrl}` : "") +
|
||||
` (reached: ${value.reached})` +
|
||||
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_reload");
|
||||
const cmdArgs = ["reload", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
|
||||
if (args.hard === true) cmdArgs.push("--hard");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
"reload",
|
||||
sessionId,
|
||||
runnerTimeout(deps, args.timeoutMs),
|
||||
)) as RawHistoryResult;
|
||||
if (reply.final_url !== undefined) deps.observation.setUrl(sessionId, reply.final_url);
|
||||
return mapHistory(sessionId, reply);
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"reload",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.hard === true ? ["--hard"] : []),
|
||||
]),
|
||||
description: "Reload the active tab",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_wait_for_navigation",
|
||||
description:
|
||||
"Wait for a page lifecycle event on the active Agent Window tab. Use after an action " +
|
||||
"that may navigate when the action result itself does not wait for navigation.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
waitUntil: WAIT_UNTIL_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
reached: { type: "string", required: true },
|
||||
errorText: { type: "string" },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] navigation wait on tab ${value.tabId}: ${value.reached}` +
|
||||
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_wait_for_navigation");
|
||||
const timeoutMs = args.timeoutMs ?? 30_000;
|
||||
const cmdArgs = ["wait-for-navigation", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
|
||||
cmdArgs.push("--timeout", `${timeoutMs}ms`);
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
"wait-for-navigation",
|
||||
sessionId,
|
||||
runnerTimeout(deps, timeoutMs),
|
||||
)) as { tab_id: number; reached: string; error_text?: string };
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
reached: reply.reached,
|
||||
...(reply.error_text !== undefined ? { errorText: reply.error_text } : {}),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"wait-for-navigation",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: "Wait for page navigation",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,672 @@
|
||||
import { defineTool } from "@deepseek-ai/dsh-tools";
|
||||
import {
|
||||
appendTabId,
|
||||
isSnapshotRef,
|
||||
type PhaseOneRuntime,
|
||||
requireNonEmpty,
|
||||
requirePositive,
|
||||
runnerTimeout,
|
||||
type ToolRegistrar,
|
||||
} from "./phase-one-runtime";
|
||||
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM } from "./tool-params";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
interface HelpConditionInput {
|
||||
urlContains?: string;
|
||||
urlMatches?: string;
|
||||
selectorExists?: string;
|
||||
selectorMissing?: string;
|
||||
textExists?: string;
|
||||
textMissing?: string;
|
||||
}
|
||||
|
||||
interface CompletionCriteriaInput {
|
||||
any?: HelpConditionInput[];
|
||||
all?: HelpConditionInput[];
|
||||
stableForMs?: number;
|
||||
}
|
||||
|
||||
interface RawConsoleFrame {
|
||||
function_name?: string;
|
||||
url?: string;
|
||||
line?: number;
|
||||
column?: number;
|
||||
}
|
||||
|
||||
interface RawConsoleEntry {
|
||||
sequence: number;
|
||||
kind: string;
|
||||
level: string;
|
||||
text: string;
|
||||
url?: string;
|
||||
line?: number;
|
||||
column?: number;
|
||||
timestamp?: number;
|
||||
stack_trace?: RawConsoleFrame[];
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
interface RawNetworkEntry {
|
||||
sequence: number;
|
||||
kind: "response" | "failure";
|
||||
method?: string;
|
||||
url?: string;
|
||||
status?: number;
|
||||
status_text?: string;
|
||||
mime_type?: string;
|
||||
resource_type?: string;
|
||||
error_text?: string;
|
||||
timestamp?: number;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
const HELP_CONDITION_SCHEMA = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
urlContains: { type: "string" },
|
||||
urlMatches: { type: "string" },
|
||||
selectorExists: { type: "string" },
|
||||
selectorMissing: { type: "string" },
|
||||
textExists: { type: "string" },
|
||||
textMissing: { type: "string" },
|
||||
},
|
||||
} as const;
|
||||
|
||||
const DEBUG_PARAMETERS = {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
since: {
|
||||
type: "integer",
|
||||
description: "Return entries with a sequence strictly greater than this cursor.",
|
||||
},
|
||||
limit: {
|
||||
type: "integer",
|
||||
description: "Maximum entries to return (default: 50, daemon cap: 200).",
|
||||
},
|
||||
maxTextChars: {
|
||||
type: "integer",
|
||||
description: "Maximum characters per entry text/URL (default: 1000, daemon cap: 4096).",
|
||||
},
|
||||
} as const;
|
||||
|
||||
function mapCondition(condition: HelpConditionInput) {
|
||||
return {
|
||||
...(condition.urlContains !== undefined ? { url_contains: condition.urlContains } : {}),
|
||||
...(condition.urlMatches !== undefined ? { url_matches: condition.urlMatches } : {}),
|
||||
...(condition.selectorExists !== undefined
|
||||
? { selector_exists: condition.selectorExists }
|
||||
: {}),
|
||||
...(condition.selectorMissing !== undefined
|
||||
? { selector_missing: condition.selectorMissing }
|
||||
: {}),
|
||||
...(condition.textExists !== undefined ? { text_exists: condition.textExists } : {}),
|
||||
...(condition.textMissing !== undefined ? { text_missing: condition.textMissing } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function mapCompletionCriteria(criteria: CompletionCriteriaInput) {
|
||||
return {
|
||||
...(criteria.any !== undefined ? { any: criteria.any.map(mapCondition) } : {}),
|
||||
...(criteria.all !== undefined ? { all: criteria.all.map(mapCondition) } : {}),
|
||||
...(criteria.stableForMs !== undefined ? { stable_for_ms: criteria.stableForMs } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function validateDebugArgs(args: { since?: number; limit?: number; maxTextChars?: number }): void {
|
||||
if (args.since !== undefined && args.since < 0) {
|
||||
throw new Error("since must be zero or greater");
|
||||
}
|
||||
requirePositive(args.limit, "limit");
|
||||
requirePositive(args.maxTextChars, "maxTextChars");
|
||||
}
|
||||
|
||||
function appendDebugOptions(
|
||||
args: string[],
|
||||
options: { tabId?: number; since?: number; limit?: number; maxTextChars?: number },
|
||||
): void {
|
||||
appendTabId(args, options.tabId);
|
||||
if (options.since !== undefined) args.push("--since", String(options.since));
|
||||
if (options.limit !== undefined) args.push("--limit", String(options.limit));
|
||||
if (options.maxTextChars !== undefined) {
|
||||
args.push("--max-text-chars", String(options.maxTextChars));
|
||||
}
|
||||
}
|
||||
|
||||
/** Human help, raw HTML, diagnostics, and Agent Window sizing. */
|
||||
export function registerPhaseOneSupportTools(
|
||||
deps: ToolDeps,
|
||||
register: ToolRegistrar,
|
||||
runtime: PhaseOneRuntime,
|
||||
): void {
|
||||
const { registry } = deps;
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_request_help",
|
||||
description:
|
||||
"Pause browser automation and ask the user to complete an in-page step such as login, " +
|
||||
"captcha, OTP, or confirmation. After continued/completed, observe again before using refs.",
|
||||
parameters: {
|
||||
prompt: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Clear instructions shown to the user in the browser help overlay.",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
title: { type: "string", description: "Optional title for the help overlay." },
|
||||
targets: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
description: "Snapshot refs or CSS selectors to scroll to and highlight for the user.",
|
||||
},
|
||||
timeoutMs: {
|
||||
...TIMEOUT_MS_PARAM,
|
||||
description: "Maximum wait for the user in milliseconds (default: 300000).",
|
||||
},
|
||||
completionCriteria: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
description: "Optional explicit success detector for automatic completion.",
|
||||
properties: {
|
||||
any: { type: "array", items: HELP_CONDITION_SCHEMA },
|
||||
all: { type: "array", items: HELP_CONDITION_SCHEMA },
|
||||
stableForMs: {
|
||||
type: "integer",
|
||||
description: "How long criteria must remain true before completing.",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
outcome: {
|
||||
type: "string",
|
||||
required: true,
|
||||
enum: ["continued", "cancelled", "timed_out", "completed", "navigated", "disabled"],
|
||||
},
|
||||
completedBy: { type: "string" },
|
||||
note: { type: "string" },
|
||||
resolvedTargets: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
matched: { type: "boolean", required: true },
|
||||
ref: { type: "string" },
|
||||
selector: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] user help outcome on tab ${value.tabId}: ${value.outcome}` +
|
||||
(value.note !== undefined ? ` — ${value.note}` : "") +
|
||||
(value.resolvedTargets !== undefined
|
||||
? `\ntargets: ${value.resolvedTargets
|
||||
.map(
|
||||
(target) =>
|
||||
`${target.ref ?? target.selector ?? "(unknown)"}=${target.matched ? "matched" : "not found"}`,
|
||||
)
|
||||
.join(", ")}`
|
||||
: ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requireNonEmpty(args.prompt, "prompt");
|
||||
if (args.title !== undefined) requireNonEmpty(args.title, "title");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
for (const target of args.targets ?? []) requireNonEmpty(target, "target");
|
||||
const criteria = args.completionCriteria as CompletionCriteriaInput | undefined;
|
||||
if (criteria?.stableForMs !== undefined && criteria.stableForMs < 0) {
|
||||
throw new Error("completionCriteria.stableForMs must be zero or greater");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_request_help");
|
||||
const timeoutMs = args.timeoutMs ?? 300_000;
|
||||
const cmdArgs = [
|
||||
"request-help",
|
||||
"--session",
|
||||
sessionId,
|
||||
"--prompt",
|
||||
args.prompt,
|
||||
"--timeout",
|
||||
`${timeoutMs}ms`,
|
||||
];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.title !== undefined) cmdArgs.push("--title", args.title);
|
||||
for (const target of args.targets ?? []) cmdArgs.push("--target", target);
|
||||
if (criteria !== undefined) {
|
||||
cmdArgs.push("--completion-criteria", JSON.stringify(mapCompletionCriteria(criteria)));
|
||||
}
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
"request-help",
|
||||
sessionId,
|
||||
runnerTimeout(deps, timeoutMs),
|
||||
)) as {
|
||||
outcome: "continued" | "cancelled" | "timed_out" | "completed" | "navigated" | "disabled";
|
||||
completed_by?: string;
|
||||
note?: string;
|
||||
tab_id: number;
|
||||
resolved_targets?: { matched: boolean; ref?: string; selector?: string }[];
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
outcome: reply.outcome,
|
||||
...(reply.completed_by !== undefined ? { completedBy: reply.completed_by } : {}),
|
||||
...(reply.note !== undefined ? { note: reply.note } : {}),
|
||||
...(reply.resolved_targets !== undefined
|
||||
? { resolvedTargets: reply.resolved_targets }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine(["request-help", "--session", args.session ?? "(current)"]),
|
||||
description: "Ask the user for browser help",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_get_html",
|
||||
description:
|
||||
"Read raw DOM HTML from the active Agent Window tab, optionally scoped to a fresh " +
|
||||
"snapshot ref. Use only when browser_observe/browser_snapshot cannot answer the question.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
ref: { type: "string", description: "Fresh snapshot ref that scopes the HTML subtree." },
|
||||
maxBytes: {
|
||||
type: "integer",
|
||||
description: "Maximum returned HTML bytes before truncation (default: 524288).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
html: { type: "string", required: true },
|
||||
truncated: { type: "boolean", required: true },
|
||||
byteSize: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.html +
|
||||
(value.truncated ? `\n(HTML truncated; original size ${value.byteSize} bytes)` : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
if (args.ref !== undefined) {
|
||||
requireNonEmpty(args.ref, "ref");
|
||||
if (!isSnapshotRef(args.ref)) throw new Error("ref must be a snapshot ref such as @e3");
|
||||
}
|
||||
requirePositive(args.maxBytes, "maxBytes");
|
||||
const sessionId = registry.resolve(args.session, "browser_get_html");
|
||||
const cmdArgs = ["get-html", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
|
||||
if (args.maxBytes !== undefined) cmdArgs.push("--max-bytes", String(args.maxBytes));
|
||||
const reply = (await runtime.run(exec, cmdArgs, "get-html", sessionId)) as {
|
||||
tab_id: number;
|
||||
html: string;
|
||||
truncated?: boolean;
|
||||
byte_size: number;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
html: reply.html,
|
||||
truncated: reply.truncated ?? false,
|
||||
byteSize: reply.byte_size,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"get-html",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.ref !== undefined ? ["--ref", args.ref] : []),
|
||||
]),
|
||||
description: "Read raw page HTML",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_console",
|
||||
description:
|
||||
"Read buffered console messages, browser log entries, and JavaScript exceptions from a tab. " +
|
||||
"This is read-only and does not evaluate JavaScript.",
|
||||
parameters: {
|
||||
...DEBUG_PARAMETERS,
|
||||
includeStack: {
|
||||
type: "boolean",
|
||||
description: "Include structured stack frames in returned console entries.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
entries: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sequence: { type: "integer", required: true },
|
||||
kind: { type: "string", required: true },
|
||||
level: { type: "string", required: true },
|
||||
text: { type: "string", required: true },
|
||||
url: { type: "string" },
|
||||
line: { type: "integer" },
|
||||
column: { type: "integer" },
|
||||
timestamp: { type: "number" },
|
||||
stackTrace: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
functionName: { type: "string" },
|
||||
url: { type: "string" },
|
||||
line: { type: "integer" },
|
||||
column: { type: "integer" },
|
||||
},
|
||||
},
|
||||
},
|
||||
truncated: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
nextSince: { type: "integer", required: true },
|
||||
truncated: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.entries.length === 0
|
||||
? "(no console messages captured)"
|
||||
: value.entries
|
||||
.map((entry) => {
|
||||
const location =
|
||||
entry.url === undefined
|
||||
? ""
|
||||
: ` ${entry.url}${entry.line !== undefined ? `:${entry.line}` : ""}${entry.column !== undefined ? `:${entry.column}` : ""}`;
|
||||
const stack = entry.stackTrace
|
||||
.map(
|
||||
(frame) =>
|
||||
`\n at ${frame.functionName ?? "<anonymous>"} ${frame.url ?? ""}` +
|
||||
`${frame.line !== undefined ? `:${frame.line}` : ""}` +
|
||||
`${frame.column !== undefined ? `:${frame.column}` : ""}`,
|
||||
)
|
||||
.join("");
|
||||
return `#${entry.sequence} ${entry.level} ${entry.kind}${location} ${entry.text}${stack}`;
|
||||
})
|
||||
.join("\n") +
|
||||
(value.truncated
|
||||
? `\n(output truncated; continue with since=${value.nextSince})`
|
||||
: ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
validateDebugArgs(args);
|
||||
const sessionId = registry.resolve(args.session, "browser_console");
|
||||
const cmdArgs = ["console", "--session", sessionId];
|
||||
appendDebugOptions(cmdArgs, args);
|
||||
if (args.includeStack === true) cmdArgs.push("--include-stack");
|
||||
const reply = (await runtime.run(exec, cmdArgs, "console", sessionId)) as {
|
||||
tab_id: number;
|
||||
entries?: RawConsoleEntry[];
|
||||
next_since: number;
|
||||
truncated?: boolean;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
entries: (reply.entries ?? []).map((entry) => ({
|
||||
sequence: entry.sequence,
|
||||
kind: entry.kind,
|
||||
level: entry.level,
|
||||
text: entry.text,
|
||||
...(entry.url !== undefined ? { url: entry.url } : {}),
|
||||
...(entry.line !== undefined ? { line: entry.line } : {}),
|
||||
...(entry.column !== undefined ? { column: entry.column } : {}),
|
||||
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
|
||||
stackTrace: (entry.stack_trace ?? []).map((frame) => ({
|
||||
...(frame.function_name !== undefined ? { functionName: frame.function_name } : {}),
|
||||
...(frame.url !== undefined ? { url: frame.url } : {}),
|
||||
...(frame.line !== undefined ? { line: frame.line } : {}),
|
||||
...(frame.column !== undefined ? { column: frame.column } : {}),
|
||||
})),
|
||||
truncated: entry.truncated ?? false,
|
||||
})),
|
||||
nextSince: reply.next_since,
|
||||
truncated: reply.truncated ?? false,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine(["console", "--session", args.session ?? "(current)"]),
|
||||
description: "Read browser console messages",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_network",
|
||||
description:
|
||||
"Read buffered network responses and failures from a tab. Returns metadata only; request " +
|
||||
"and response headers and bodies are not captured.",
|
||||
parameters: DEBUG_PARAMETERS,
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
entries: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sequence: { type: "integer", required: true },
|
||||
kind: { type: "string", required: true, enum: ["response", "failure"] },
|
||||
method: { type: "string" },
|
||||
url: { type: "string" },
|
||||
status: { type: "integer" },
|
||||
statusText: { type: "string" },
|
||||
mimeType: { type: "string" },
|
||||
resourceType: { type: "string" },
|
||||
errorText: { type: "string" },
|
||||
timestamp: { type: "number" },
|
||||
truncated: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
nextSince: { type: "integer", required: true },
|
||||
truncated: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.entries.length === 0
|
||||
? "(no network activity captured)"
|
||||
: value.entries
|
||||
.map((entry) =>
|
||||
entry.kind === "failure"
|
||||
? `#${entry.sequence} FAILED ${entry.method ?? "?"} ${entry.url ?? "(unknown)"} — ${entry.errorText ?? "failed"}`
|
||||
: `#${entry.sequence} ${entry.status ?? "?"} ${entry.method ?? "?"} ${entry.url ?? "(unknown)"}`,
|
||||
)
|
||||
.join("\n") +
|
||||
(value.truncated
|
||||
? `\n(output truncated; continue with since=${value.nextSince})`
|
||||
: ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
validateDebugArgs(args);
|
||||
const sessionId = registry.resolve(args.session, "browser_network");
|
||||
const cmdArgs = ["network", "--session", sessionId];
|
||||
appendDebugOptions(cmdArgs, args);
|
||||
const reply = (await runtime.run(exec, cmdArgs, "network", sessionId)) as {
|
||||
tab_id: number;
|
||||
entries?: RawNetworkEntry[];
|
||||
next_since: number;
|
||||
truncated?: boolean;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
entries: (reply.entries ?? []).map((entry) => ({
|
||||
sequence: entry.sequence,
|
||||
kind: entry.kind,
|
||||
...(entry.method !== undefined ? { method: entry.method } : {}),
|
||||
...(entry.url !== undefined ? { url: entry.url } : {}),
|
||||
...(entry.status !== undefined ? { status: entry.status } : {}),
|
||||
...(entry.status_text !== undefined ? { statusText: entry.status_text } : {}),
|
||||
...(entry.mime_type !== undefined ? { mimeType: entry.mime_type } : {}),
|
||||
...(entry.resource_type !== undefined ? { resourceType: entry.resource_type } : {}),
|
||||
...(entry.error_text !== undefined ? { errorText: entry.error_text } : {}),
|
||||
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
|
||||
truncated: entry.truncated ?? false,
|
||||
})),
|
||||
nextSince: reply.next_since,
|
||||
truncated: reply.truncated ?? false,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine(["network", "--session", args.session ?? "(current)"]),
|
||||
description: "Read browser network activity",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_window_resize",
|
||||
description: "Resize the owned session's Agent Window using outer CSS-pixel dimensions.",
|
||||
parameters: {
|
||||
width: {
|
||||
type: "integer",
|
||||
required: true,
|
||||
description: "Agent Window outer width in CSS pixels (100..=7680).",
|
||||
},
|
||||
height: {
|
||||
type: "integer",
|
||||
required: true,
|
||||
description: "Agent Window outer height in CSS pixels (100..=7680).",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
windowId: { type: "integer", required: true },
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] resized window ${value.windowId} to ${value.width}x${value.height}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.width < 100 || args.width > 7680 || args.height < 100 || args.height > 7680) {
|
||||
throw new Error("width and height must each be in the range 100..=7680");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_window_resize");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
[
|
||||
"window",
|
||||
"resize",
|
||||
"--session",
|
||||
sessionId,
|
||||
"--width",
|
||||
String(args.width),
|
||||
"--height",
|
||||
String(args.height),
|
||||
],
|
||||
"window resize",
|
||||
sessionId,
|
||||
)) as { window_id: number; width: number; height: number };
|
||||
return {
|
||||
session: sessionId,
|
||||
windowId: reply.window_id,
|
||||
width: reply.width,
|
||||
height: reply.height,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"window",
|
||||
"resize",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
"--width",
|
||||
String(args.width),
|
||||
"--height",
|
||||
String(args.height),
|
||||
]),
|
||||
description: "Resize the Agent Window",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
import { defineTool } from "@deepseek-ai/dsh-tools";
|
||||
import type { PhaseOneRuntime, ToolRegistrar } from "./phase-one-runtime";
|
||||
import { requireNonEmpty } from "./phase-one-runtime";
|
||||
import { SESSION_PARAM } from "./tool-params";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
const TAB_ID_REQUIRED = {
|
||||
type: "integer",
|
||||
required: true,
|
||||
description: "Chrome tab id returned by browser_tab_list or browser_tab_create.",
|
||||
} as const;
|
||||
|
||||
interface RawTabInfo {
|
||||
tab_id: number;
|
||||
title?: string;
|
||||
url?: string;
|
||||
window_id?: number;
|
||||
active?: boolean;
|
||||
scope?: "user" | "agent";
|
||||
}
|
||||
|
||||
/** Add Agent Window tab management while preserving the owned-session boundary. */
|
||||
export function registerPhaseOneTabTools(
|
||||
deps: ToolDeps,
|
||||
register: ToolRegistrar,
|
||||
runtime: PhaseOneRuntime,
|
||||
): void {
|
||||
const { registry } = deps;
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_list",
|
||||
description:
|
||||
"List tabs visible to an owned browser session. Other sessions' Agent Windows remain " +
|
||||
"hidden; use scope=user to find a user tab before borrowing it.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
scope: {
|
||||
type: "string",
|
||||
enum: ["user", "agent", "all"],
|
||||
description: "Which tabs to list (default: all).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabs: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
tabId: { type: "integer", required: true },
|
||||
title: { type: "string" },
|
||||
url: { type: "string" },
|
||||
windowId: { type: "integer" },
|
||||
active: { type: "boolean" },
|
||||
scope: { type: "string", enum: ["user", "agent"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.tabs.length === 0
|
||||
? `[session ${value.session}] no matching tabs`
|
||||
: value.tabs
|
||||
.map(
|
||||
(tab) =>
|
||||
`${tab.tabId} [${tab.scope ?? "unknown"}]${tab.active === true ? " [active]" : ""} ` +
|
||||
`${tab.title ?? "(untitled)"} — ${tab.url ?? "(unknown URL)"}`,
|
||||
)
|
||||
.join("\n"),
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_list");
|
||||
const cmdArgs = ["tab", "list", "--session", sessionId];
|
||||
if (args.scope !== undefined) cmdArgs.push("--scope", args.scope);
|
||||
const reply = (await runtime.run(exec, cmdArgs, "tab list", sessionId)) as {
|
||||
tabs: RawTabInfo[];
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabs: reply.tabs.map((tab) => ({
|
||||
tabId: tab.tab_id,
|
||||
...(tab.title !== undefined ? { title: tab.title } : {}),
|
||||
...(tab.url !== undefined ? { url: tab.url } : {}),
|
||||
...(tab.window_id !== undefined ? { windowId: tab.window_id } : {}),
|
||||
...(tab.active !== undefined ? { active: tab.active } : {}),
|
||||
...(tab.scope !== undefined ? { scope: tab.scope } : {}),
|
||||
})),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"tab",
|
||||
"list",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.scope !== undefined ? ["--scope", args.scope] : []),
|
||||
]),
|
||||
description: "List browser tabs",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_create",
|
||||
description:
|
||||
"Create a tab inside the owned session's Agent Window. The tab is focused by default; " +
|
||||
"set active=false to open it in the background.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
url: { type: "string", description: "Initial URL (default: chrome://newtab/)." },
|
||||
active: { type: "boolean", description: "Whether to focus the new tab (default: true)." },
|
||||
index: { type: "integer", description: "Insertion index in the Agent Window tab strip." },
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
windowId: { type: "integer", required: true },
|
||||
url: { type: "string", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] created tab ${value.tabId} in window ${value.windowId} (${value.url || "pending URL"})`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url !== undefined) requireNonEmpty(args.url, "url");
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_create");
|
||||
const cmdArgs = ["tab", "create", "--session", sessionId];
|
||||
if (args.url !== undefined) cmdArgs.push("--url", args.url);
|
||||
if (args.active === false) cmdArgs.push("--no-active");
|
||||
if (args.index !== undefined) cmdArgs.push("--index", String(args.index));
|
||||
const reply = (await runtime.run(exec, cmdArgs, "tab create", sessionId)) as {
|
||||
tab_id: number;
|
||||
window_id: number;
|
||||
url: string;
|
||||
};
|
||||
if (args.active !== false && reply.url.length > 0) {
|
||||
deps.observation.setUrl(sessionId, reply.url);
|
||||
}
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
windowId: reply.window_id,
|
||||
url: reply.url,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"tab",
|
||||
"create",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.url !== undefined ? ["--url", args.url] : []),
|
||||
]),
|
||||
description: "Create an Agent Window tab",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_close",
|
||||
description: "Close a tab in the owned session's Agent Window.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{ type: "text", text: `[session ${value.session}] closed tab ${value.tabId}` },
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_close");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "close", String(args.tabId), "--session", sessionId],
|
||||
"tab close",
|
||||
sessionId,
|
||||
)) as { tab_id: number };
|
||||
return { session: sessionId, tabId: reply.tab_id };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"tab",
|
||||
"close",
|
||||
String(args.tabId),
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: "Close an Agent Window tab",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_select",
|
||||
description: "Focus a tab in the owned session's Agent Window.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
windowId: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] selected tab ${value.tabId} in window ${value.windowId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_select");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "select", String(args.tabId), "--session", sessionId],
|
||||
"tab select",
|
||||
sessionId,
|
||||
)) as { tab_id: number; window_id: number };
|
||||
return { session: sessionId, tabId: reply.tab_id, windowId: reply.window_id };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"tab",
|
||||
"select",
|
||||
String(args.tabId),
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: "Select an Agent Window tab",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_borrow",
|
||||
description:
|
||||
"Move a user-window tab into the owned session's Agent Window for controlled interaction. " +
|
||||
"Return it with browser_tab_return when finished; stopping the session also auto-returns it.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
originalWindowId: { type: "integer", required: true },
|
||||
originalIndex: { type: "integer", required: true },
|
||||
agentWindowId: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] borrowed tab ${value.tabId} from window ` +
|
||||
`${value.originalWindowId} into Agent Window ${value.agentWindowId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_borrow");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "borrow", String(args.tabId), "--session", sessionId],
|
||||
"tab borrow",
|
||||
sessionId,
|
||||
)) as {
|
||||
tab_id: number;
|
||||
original_window_id: number;
|
||||
original_index: number;
|
||||
agent_window_id: number;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
originalWindowId: reply.original_window_id,
|
||||
originalIndex: reply.original_index,
|
||||
agentWindowId: reply.agent_window_id,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"tab",
|
||||
"borrow",
|
||||
String(args.tabId),
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: "Borrow a user tab",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_return",
|
||||
description: "Return a borrowed tab to its original user window and tab-strip position.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
returnedToWindowId: { type: "integer", required: true },
|
||||
returnedToIndex: { type: "integer", required: true },
|
||||
fallback: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] returned tab ${value.tabId} to window ` +
|
||||
`${value.returnedToWindowId} at index ${value.returnedToIndex}` +
|
||||
(value.fallback ? " (original window unavailable; used fallback)" : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_return");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "return", String(args.tabId), "--session", sessionId],
|
||||
"tab return",
|
||||
sessionId,
|
||||
)) as {
|
||||
tab_id: number;
|
||||
returned_to_window_id: number;
|
||||
returned_to_index: number;
|
||||
fallback?: boolean;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
returnedToWindowId: reply.returned_to_window_id,
|
||||
returnedToIndex: reply.returned_to_index,
|
||||
fallback: reply.fallback ?? false,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
"tab",
|
||||
"return",
|
||||
String(args.tabId),
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: "Return a borrowed tab",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { PhaseOneRuntime, ToolRegistrar } from "./phase-one-runtime";
|
||||
import { registerPhaseOneInteractionTools } from "./phase-one-tools-interaction";
|
||||
import { registerPhaseOneNavigationTools } from "./phase-one-tools-navigation";
|
||||
import { registerPhaseOneSupportTools } from "./phase-one-tools-support";
|
||||
import { registerPhaseOneTabTools } from "./phase-one-tools-tabs";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
/** Register the first DSH capability-parity tranche on the existing runtime. */
|
||||
export function registerPhaseOneTools(
|
||||
deps: ToolDeps,
|
||||
register: ToolRegistrar,
|
||||
runtime: PhaseOneRuntime,
|
||||
): void {
|
||||
registerPhaseOneInteractionTools(deps, register, runtime);
|
||||
registerPhaseOneTabTools(deps, register, runtime);
|
||||
registerPhaseOneNavigationTools(deps, register, runtime);
|
||||
registerPhaseOneSupportTools(deps, register, runtime);
|
||||
}
|
||||
@@ -64,14 +64,17 @@ export interface BskRunner {
|
||||
killFor(tag: string): number;
|
||||
}
|
||||
|
||||
const KILL_GRACE_MS = 2000;
|
||||
// Business RPCs translate SIGINT into the daemon's cancel(rpc_id) protocol.
|
||||
// Give that bounded reconciliation path time to settle before the hard kill.
|
||||
const KILL_GRACE_MS = 3000;
|
||||
const SESSION_BUSY_RETRY_DELAY_MS = 100;
|
||||
|
||||
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
|
||||
const live = new Map<ChildProcess, string | undefined>();
|
||||
|
||||
function killChild(child: ChildProcess): void {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
child.kill("SIGINT");
|
||||
const force = setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
}, KILL_GRACE_MS);
|
||||
@@ -163,6 +166,60 @@ export function isCommandNotFound(error: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/** True only for the daemon's transient per-session reconciliation window. */
|
||||
export function isSessionBusyResult(result: BskRunResult): boolean {
|
||||
if (result.code === 0) return false;
|
||||
try {
|
||||
const body = JSON.parse(result.stdout) as BskErrorBody;
|
||||
const reason =
|
||||
typeof body.data === "object" && body.data !== null && "reason" in body.data
|
||||
? (body.data as { reason?: unknown }).reason
|
||||
: undefined;
|
||||
return reason === "session_busy";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry exactly once after the tiny daemon-settlement race that can follow a
|
||||
* graceful SIGINT cancellation. Other errors and persistent busy states stay
|
||||
* visible to callers.
|
||||
*/
|
||||
export async function runWithSessionBusyRetry(
|
||||
run: () => Promise<BskRunResult>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<BskRunResult> {
|
||||
const first = await run();
|
||||
if (!isSessionBusyResult(first)) return first;
|
||||
await abortableDelay(SESSION_BUSY_RETRY_DELAY_MS, signal);
|
||||
return run();
|
||||
}
|
||||
|
||||
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) return Promise.reject(abortError());
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(done, ms);
|
||||
timer.unref();
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(abortError());
|
||||
};
|
||||
function done() {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Install guidance shown when the bsk CLI cannot be spawned. */
|
||||
export function bskInstallMessage(bskPath: string): string {
|
||||
return (
|
||||
@@ -182,7 +239,7 @@ export function parseBskJson(result: BskRunResult, commandLabel: string): unknow
|
||||
throw new BskError(`bsk ${commandLabel} timed out`, { timedOut: true });
|
||||
}
|
||||
const body = result.stdout.trim();
|
||||
// Killed by our own interrupt (SIGTERM from killFor), not by the abort path:
|
||||
// Killed by our own interrupt (SIGINT from killFor), not by the abort path:
|
||||
// say so instead of doubling the generic label into the message.
|
||||
if (result.code === null && !result.aborted && !result.timedOut) {
|
||||
throw new BskError(`bsk ${commandLabel} was interrupted (process killed)`);
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
* Progressive disclosure of the BrowserSkill agent skill through the harness's
|
||||
* official skill seam (`ctx.skills`). The catalog entry (name + description)
|
||||
* is resident; the body is loaded only when the model invokes the `skill`
|
||||
* tool. The markdown is build-time assembled into a static module (dsh
|
||||
* prelude + the canonical CLI skill body verbatim), so registration and every
|
||||
* pre-step catalog snapshot are pure in-memory reads — no disk, no process,
|
||||
* no daemon.
|
||||
* tool. The dedicated DeepSeek Harness markdown is embedded into a static
|
||||
* module at build time, so registration and every pre-step catalog snapshot
|
||||
* are pure in-memory reads — no disk, no process, no daemon. The CLI-oriented
|
||||
* repository skill is intentionally a separate interface.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { Agent } from "@deepseek-ai/dsh-agent";
|
||||
import {
|
||||
BSK_SKILL_DESCRIPTION,
|
||||
BSK_SKILL_MARKDOWN,
|
||||
@@ -25,8 +26,13 @@ interface SkillsLike {
|
||||
}): () => void;
|
||||
}
|
||||
|
||||
/** Structural view of the live-agent registry (absent in bare compositions). */
|
||||
interface AgentsLike {
|
||||
list(): Agent[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish the BSK skill as an embedded runtime skill. Returns the unregistration
|
||||
* Publish the browser skill as an embedded runtime skill. Returns the unregistration
|
||||
* disposer. A composition without the `skills` service (or with dsh-tool-skill
|
||||
* retired) leaves the rest of the plugin unaffected — the no-op is silent.
|
||||
*/
|
||||
@@ -43,3 +49,54 @@ export function registerBskSkill(ctx: Context): () => void {
|
||||
source: "bundled",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Install the DSH-specific browser skill into every exact agent scope.
|
||||
*
|
||||
* DSH merges skill layers nearest-first (`agent -> preset -> global`). A legacy
|
||||
* CLI skill installed at `~/.agents/skills/browser-skill` is discovered by the
|
||||
* preset filesystem provider and therefore shadows a plugin-level registration.
|
||||
* Registering the embedded skill through `agent.ctx` makes the DSH protocol
|
||||
* contract authoritative for that agent without touching the shared CLI skill.
|
||||
*
|
||||
* New agents are handled at `agent/session-start`, the first supported startup
|
||||
* injection point and still before the first prompt assembly. Existing agents
|
||||
* are registered immediately so plugin reloads take effect without recreating
|
||||
* the conversation. Returns a disposer for all plugin-owned registrations.
|
||||
*/
|
||||
export function armAgentScopedBskSkill(ctx: Context): () => void {
|
||||
const registrations = new Map<Agent, () => void>();
|
||||
let active = true;
|
||||
|
||||
const registerForAgent = (agent: Agent): void => {
|
||||
if (!active || registrations.has(agent)) return;
|
||||
registrations.set(agent, registerBskSkill(agent.ctx));
|
||||
};
|
||||
|
||||
let stopSessionStart = () => {};
|
||||
let stopDisposed = () => {};
|
||||
if (typeof ctx.on === "function") {
|
||||
stopSessionStart = ctx.on("agent/session-start", ({ agent }) => {
|
||||
registerForAgent(agent);
|
||||
});
|
||||
stopDisposed = ctx.on("agent/disposed", ({ agent }) => {
|
||||
// Agent-scoped effects have already unwound at this lifecycle edge. Drop
|
||||
// the retained disposer without invoking it against a disposed context.
|
||||
registrations.delete(agent);
|
||||
});
|
||||
}
|
||||
|
||||
const agents = ctx.get("agents") as AgentsLike | undefined | null;
|
||||
if (agents != null && typeof agents.list === "function") {
|
||||
for (const agent of agents.list()) registerForAgent(agent);
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (!active) return;
|
||||
active = false;
|
||||
stopDisposed();
|
||||
stopSessionStart();
|
||||
for (const unregister of [...registrations.values()].reverse()) unregister();
|
||||
registrations.clear();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/** Shared model-facing parameter schemas for browser tools. */
|
||||
|
||||
export const SESSION_PARAM = {
|
||||
type: "string",
|
||||
description:
|
||||
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
|
||||
"Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
export const TAB_ID_PARAM = {
|
||||
type: "integer",
|
||||
description: "Target tab id. Omit to use the Agent Window's active tab.",
|
||||
} as const;
|
||||
|
||||
export const WAIT_UNTIL_PARAM = {
|
||||
type: "string",
|
||||
enum: ["load", "domcontentloaded", "networkidle", "commit"],
|
||||
description: "Page lifecycle phase to wait for (default: load).",
|
||||
} as const;
|
||||
|
||||
export const TIMEOUT_MS_PARAM = {
|
||||
type: "integer",
|
||||
description: "Command timeout in milliseconds; must be greater than zero.",
|
||||
} as const;
|
||||
@@ -20,9 +20,17 @@ import {
|
||||
import { ownerSessionIds } from "./archive-cleanup";
|
||||
import { trySaveScreenshot } from "./image";
|
||||
import { actionForLabel, type ObservationService } from "./observation";
|
||||
import { registerPhaseOneTools } from "./phase-one-tools";
|
||||
import type { KeyedExecutor } from "./queue";
|
||||
import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
|
||||
import {
|
||||
type BskRunner,
|
||||
bskInstallMessage,
|
||||
isCommandNotFound,
|
||||
parseBskJson,
|
||||
runWithSessionBusyRetry,
|
||||
} from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
import { SESSION_PARAM } from "./tool-params";
|
||||
|
||||
/** Plugin configuration resolved from the Schemastery schema in index.ts. */
|
||||
export interface PluginConfig {
|
||||
@@ -62,13 +70,6 @@ const DEVICE_PRESETS = [
|
||||
"galaxy-tab-s8",
|
||||
] as const;
|
||||
|
||||
const SESSION_PARAM = {
|
||||
type: "string",
|
||||
description:
|
||||
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
|
||||
"Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Run one bsk command and return its parsed JSON payload, or throw.
|
||||
* `observeSession` marks the run as model-facing work on that session: it is
|
||||
@@ -82,7 +83,10 @@ async function runBsk(
|
||||
args: string[],
|
||||
label: string,
|
||||
observeSession?: string,
|
||||
runnerTimeoutMs?: number,
|
||||
): Promise<unknown> {
|
||||
const releaseForeground =
|
||||
observeSession !== undefined ? deps.observation.acquireForeground(observeSession) : undefined;
|
||||
// beginAction must fire only when the task actually starts inside the
|
||||
// per-session queue — marking it while still queued would overwrite the
|
||||
// in-flight action's label (and flash it idle on a queued abort).
|
||||
@@ -96,11 +100,15 @@ async function runBsk(
|
||||
began = true;
|
||||
deps.observation.beginAction(observeSession, actionForLabel(label));
|
||||
}
|
||||
return deps.runner.run(args, {
|
||||
signal: exec.signal,
|
||||
timeoutMs: deps.config.defaultTimeoutMs,
|
||||
...(observeSession !== undefined ? { tag: observeSession } : {}),
|
||||
});
|
||||
return runWithSessionBusyRetry(
|
||||
() =>
|
||||
deps.runner.run(args, {
|
||||
signal: exec.signal,
|
||||
timeoutMs: runnerTimeoutMs ?? deps.config.defaultTimeoutMs,
|
||||
...(observeSession !== undefined ? { tag: observeSession } : {}),
|
||||
}),
|
||||
exec.signal,
|
||||
);
|
||||
};
|
||||
result =
|
||||
observeSession !== undefined
|
||||
@@ -125,6 +133,7 @@ async function runBsk(
|
||||
if (observeSession !== undefined && began) {
|
||||
deps.observation.endAction(observeSession, actionError);
|
||||
}
|
||||
releaseForeground?.();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,22 +280,15 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
}
|
||||
} catch (error) {
|
||||
// A half-initialized session must not leak: stop it before surfacing.
|
||||
// The stop funnels through the session's queue (an in-flight command
|
||||
// — e.g. an observation capture — would otherwise make the daemon
|
||||
// refuse it), with one retry for good measure.
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const stopped = await deps.queue.run(reply.session_id, () =>
|
||||
deps.runner.run(["session", "stop", reply.session_id], { timeoutMs: 30_000 }),
|
||||
);
|
||||
if (stopped.code === 0) break;
|
||||
} catch {
|
||||
// retry once, then give up quietly (registry/observation are
|
||||
// cleaned below either way; the daemon reaps orphans).
|
||||
}
|
||||
// Reuse the same capture-preempting, idempotent path as explicit and
|
||||
// overlay stops; local ownership is dropped even when daemon cleanup
|
||||
// itself fails, because this start never becomes usable to the model.
|
||||
try {
|
||||
await deps.observation.stopSession(reply.session_id);
|
||||
} catch {
|
||||
registry.remove(reply.session_id);
|
||||
deps.observation.removeSession(reply.session_id);
|
||||
}
|
||||
registry.remove(reply.session_id);
|
||||
deps.observation.removeSession(reply.session_id);
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
@@ -330,9 +332,15 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolveForStop(args.session);
|
||||
await runBsk(deps, exec, ["session", "stop", sessionId], "session stop", sessionId);
|
||||
registry.remove(sessionId);
|
||||
deps.observation.removeSession(sessionId);
|
||||
try {
|
||||
const stopped = await deps.observation.stopSession(sessionId, exec.signal);
|
||||
if (!stopped) throw new Error(`browser session ${sessionId} is not owned by this plugin`);
|
||||
} catch (error) {
|
||||
if (isCommandNotFound(error)) {
|
||||
throw new Error(bskInstallMessage(deps.config.bskPath));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { stopped: sessionId };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
@@ -489,7 +497,9 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
"probes) with @eN refs. Visual-only surfaces require image understanding and their refs are " +
|
||||
"screenshot-only; keep using any DOM/AX semantics rendered alongside them. " +
|
||||
"Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
@@ -561,7 +571,8 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
name: "browser_click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||
"last browser_snapshot/browser_observe, or a CSS selector. A visual Surface requires the " +
|
||||
"captureId and image coordinates returned by browser_screenshot.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -578,6 +589,23 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
type: "integer",
|
||||
description: "Number of consecutive presses (double-click = 2).",
|
||||
},
|
||||
modifiers: {
|
||||
type: "array",
|
||||
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
|
||||
description: "Keyboard modifiers held during the click.",
|
||||
},
|
||||
captureId: {
|
||||
type: "string",
|
||||
description: "Short-lived Surface capture id returned by browser_screenshot.",
|
||||
},
|
||||
imageX: {
|
||||
type: "number",
|
||||
description: "X coordinate in capture-image pixels; requires captureId and imageY.",
|
||||
},
|
||||
imageY: {
|
||||
type: "number",
|
||||
description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
@@ -588,6 +616,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
tabId: { type: "integer", required: true },
|
||||
x: { type: "number", required: true },
|
||||
y: { type: "number", required: true },
|
||||
captureId: { type: "string" },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
@@ -599,17 +628,44 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const pointArgCount =
|
||||
Number(args.captureId !== undefined) +
|
||||
Number(args.imageX !== undefined) +
|
||||
Number(args.imageY !== undefined);
|
||||
if (pointArgCount !== 0 && pointArgCount !== 3) {
|
||||
throw new Error("captureId, imageX, and imageY must be given together");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_click");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
||||
cmdArgs.push("--modifiers", args.modifiers.join(","));
|
||||
}
|
||||
if (args.captureId !== undefined) {
|
||||
cmdArgs.push(
|
||||
"--capture",
|
||||
args.captureId,
|
||||
"--image-x",
|
||||
String(args.imageX),
|
||||
"--image-y",
|
||||
String(args.imageY),
|
||||
);
|
||||
}
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
||||
tab_id: number;
|
||||
x: number;
|
||||
y: number;
|
||||
capture_id?: string;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
x: reply.x,
|
||||
y: reply.y,
|
||||
...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
@@ -752,12 +808,17 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
name: "browser_screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"This is the supported operation for visual surface refs emitted by browser_observe. Only use " +
|
||||
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
|
||||
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
|
||||
"not guess content or coordinates. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
ref: {
|
||||
type: "string",
|
||||
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||
description:
|
||||
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
@@ -771,6 +832,8 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
byteSize: { type: "integer", required: true },
|
||||
captureId: { type: "string" },
|
||||
captureExpiresAt: { type: "integer" },
|
||||
image: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
@@ -790,7 +853,11 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
value.image !== undefined
|
||||
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
||||
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
||||
const blocks: ContentBlock[] = [{ type: "text", text }];
|
||||
const captureText =
|
||||
value.captureId !== undefined
|
||||
? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
|
||||
: "";
|
||||
const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
|
||||
if (value.image !== undefined) {
|
||||
blocks.push({
|
||||
type: "image",
|
||||
@@ -829,6 +896,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
height: number;
|
||||
path: string;
|
||||
byte_size: number;
|
||||
capture?: { id: string; expires_at: number };
|
||||
};
|
||||
writtenPath = reply.path;
|
||||
const data = await readFile(reply.path);
|
||||
@@ -842,6 +910,9 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
width: reply.width,
|
||||
height: reply.height,
|
||||
byteSize: reply.byte_size,
|
||||
...(reply.capture !== undefined
|
||||
? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
|
||||
: {}),
|
||||
...(ref !== undefined
|
||||
? {
|
||||
image: {
|
||||
@@ -959,6 +1030,12 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
registerPhaseOneTools(deps, register, {
|
||||
run: (exec, args, label, observeSession, runnerTimeoutMs) =>
|
||||
runBsk(deps, exec, args, label, observeSession, runnerTimeoutMs),
|
||||
commandLine: (args) => cmdline(deps, args),
|
||||
presentTerminalResult,
|
||||
});
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
};
|
||||
|
||||
@@ -78,7 +78,7 @@ function fakeRunner(
|
||||
if (opts.stopNotFound) {
|
||||
return {
|
||||
code: 4,
|
||||
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
|
||||
stdout: JSON.stringify({ code: "not_found", message: "no such session" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
@@ -93,7 +93,7 @@ function fakeRunner(
|
||||
if (opts.screenshotNotFound) {
|
||||
return {
|
||||
code: 4,
|
||||
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
|
||||
stdout: JSON.stringify({ code: "not_found", message: "no such session" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
@@ -224,6 +224,57 @@ describe("state machine", () => {
|
||||
});
|
||||
|
||||
describe("thumbnail cadence", () => {
|
||||
it("gives foreground work priority and resumes with one fresh capture", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
let finishCapture: ((result: BskRunResult) => void) | undefined;
|
||||
const killed: string[] = [];
|
||||
const runner: FakeRunner = {
|
||||
calls: [],
|
||||
killed,
|
||||
run(args) {
|
||||
if (args[0] !== "screenshot") {
|
||||
return Promise.resolve({
|
||||
code: 0,
|
||||
stdout: "{}",
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
});
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
finishCapture = resolve;
|
||||
});
|
||||
},
|
||||
killAll() {},
|
||||
killFor(tag) {
|
||||
killed.push(tag);
|
||||
if (tag === "observation:s1" && finishCapture !== undefined) {
|
||||
finishCapture({
|
||||
code: 5,
|
||||
stdout: JSON.stringify({ code: "cancelled", message: "cancelled" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
});
|
||||
finishCapture = undefined;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
},
|
||||
};
|
||||
const { service } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(() => finishCapture !== undefined);
|
||||
|
||||
const release = service.acquireForeground("s1");
|
||||
expect(killed).toEqual(["observation:s1"]);
|
||||
await waitFor(() => scheduler.pending().length === 0);
|
||||
release();
|
||||
expect(service.isAvailable()).toBe(true);
|
||||
expect(scheduler.pending()).toEqual([0]);
|
||||
});
|
||||
|
||||
it("captures immediately on add and after action end, then fast-cadences while active", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
@@ -351,7 +402,7 @@ describe("stopSession", () => {
|
||||
expect(registry.isOwned("s1")).toBe(true);
|
||||
|
||||
await expect(service.stopSession("s1")).resolves.toBe(true);
|
||||
expect(runner.killed).toEqual(["s1"]);
|
||||
expect(runner.killed).toEqual(["observation:s1", "s1"]);
|
||||
const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
|
||||
expect(stop?.args).toEqual(["session", "stop", "s1"]);
|
||||
expect(registry.isOwned("s1")).toBe(false);
|
||||
@@ -380,13 +431,13 @@ describe("stopSession", () => {
|
||||
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
|
||||
});
|
||||
|
||||
it("returns false and keeps the entry when the stop itself fails", async () => {
|
||||
it("rejects and keeps the entry when the stop itself fails", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner({ stopFails: true });
|
||||
const { service } = setup({ registry, runner });
|
||||
service.addSession("s1");
|
||||
await expect(service.stopSession("s1")).resolves.toBe(false);
|
||||
await expect(service.stopSession("s1")).rejects.toThrow(/boom/);
|
||||
expect(registry.isOwned("s1")).toBe(true);
|
||||
expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
|
||||
});
|
||||
@@ -590,7 +641,7 @@ describe("HTTP/SSE interface", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("availability and dead sessions", () => {
|
||||
describe("availability and missing sessions", () => {
|
||||
it("flips availability off after repeated global failures and back on success", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner({ screenshotFails: true });
|
||||
@@ -610,23 +661,18 @@ describe("availability and dead sessions", () => {
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("marks a session dead on session_not_found and stops asking for frames", async () => {
|
||||
it("removes an owned ghost on not_found and stops asking for frames", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner({ screenshotNotFound: true });
|
||||
const { service, events } = setup({ runner, scheduler });
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const { service, events } = setup({ runner, scheduler, registry });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(() => events.some((e) => e.type === "upsert" && e.session?.dead === true));
|
||||
expect(service.getState()[0].dead).toBe(true);
|
||||
expect(service.getState()[0].action).toBe("idle");
|
||||
// No further captures are scheduled for a dead session.
|
||||
expect(scheduler.pending()).toEqual([]);
|
||||
// Instrumentation is dropped for dead sessions.
|
||||
service.beginAction("s1", "clicking");
|
||||
expect(service.getState()[0].action).toBe("idle");
|
||||
// A dead session leaves cleanly.
|
||||
service.removeSession("s1");
|
||||
await waitFor(() => events.some((e) => e.type === "remove"));
|
||||
expect(service.getState()).toEqual([]);
|
||||
expect(registry.isOwned("s1")).toBe(false);
|
||||
expect(scheduler.pending()).toEqual([]);
|
||||
});
|
||||
|
||||
it("clears lastError on the next action and keeps it on failure", () => {
|
||||
@@ -649,6 +695,9 @@ describe("actionForLabel", () => {
|
||||
expect(actionForLabel("navigate")).toBe("navigating");
|
||||
expect(actionForLabel("screenshot")).toBe("capturing");
|
||||
expect(actionForLabel("session start")).toBe("starting");
|
||||
expect(actionForLabel("tab borrow")).toBe("borrowing tab");
|
||||
expect(actionForLabel("request-help")).toBe("waiting for user");
|
||||
expect(actionForLabel("wait-for-navigation")).toBe("waiting for navigation");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type BskRunResult, createBskRunner, isCommandNotFound, parseBskJson } from "../src/runner";
|
||||
import {
|
||||
type BskRunResult,
|
||||
createBskRunner,
|
||||
isCommandNotFound,
|
||||
isSessionBusyResult,
|
||||
parseBskJson,
|
||||
runWithSessionBusyRetry,
|
||||
} from "../src/runner";
|
||||
|
||||
/** Minimal fake ChildProcess driven by the test. */
|
||||
class FakeChild extends EventEmitter {
|
||||
@@ -63,7 +70,7 @@ describe("createBskRunner", () => {
|
||||
controller.abort();
|
||||
const result = await promise;
|
||||
expect(result.aborted).toBe(true);
|
||||
expect(child.killedWith).toContain("SIGTERM");
|
||||
expect(child.killedWith).toContain("SIGINT");
|
||||
});
|
||||
|
||||
it("kills the child on timeout", async () => {
|
||||
@@ -80,7 +87,7 @@ describe("createBskRunner", () => {
|
||||
const promise = runner.run(["session", "list"]);
|
||||
runner.killAll();
|
||||
const result = await promise;
|
||||
expect(child.killedWith).toContain("SIGTERM");
|
||||
expect(child.killedWith).toContain("SIGINT");
|
||||
expect(result.code).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -140,3 +147,51 @@ describe("isCommandNotFound", () => {
|
||||
expect(isCommandNotFound(new Error("other"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("session busy reconciliation", () => {
|
||||
const busy: BskRunResult = {
|
||||
code: 4,
|
||||
stdout: JSON.stringify({
|
||||
code: "timeout",
|
||||
message: "session already has an unfinished command",
|
||||
data: { reason: "session_busy" },
|
||||
}),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
const ok: BskRunResult = {
|
||||
code: 0,
|
||||
stdout: "{}",
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
|
||||
it("recognizes only the structured session_busy reason", () => {
|
||||
expect(isSessionBusyResult(busy)).toBe(true);
|
||||
expect(isSessionBusyResult({ ...busy, stdout: '{"code":"timeout"}' })).toBe(false);
|
||||
expect(isSessionBusyResult(ok)).toBe(false);
|
||||
});
|
||||
|
||||
it("retries one transient busy result and returns the settled response", async () => {
|
||||
const replies = [busy, ok];
|
||||
let calls = 0;
|
||||
const result = await runWithSessionBusyRetry(async () => {
|
||||
calls += 1;
|
||||
return replies.shift() ?? ok;
|
||||
});
|
||||
expect(result).toBe(ok);
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
it("does not loop on a persistent busy result", async () => {
|
||||
let calls = 0;
|
||||
const result = await runWithSessionBusyRetry(async () => {
|
||||
calls += 1;
|
||||
return busy;
|
||||
});
|
||||
expect(result).toBe(busy);
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,18 @@
|
||||
// BSK skill injection: registration wiring, catalog content, progressive-load
|
||||
// Browser skill injection: registration wiring, catalog content, progressive-load
|
||||
// weight, and silent degradation without the skill seam.
|
||||
|
||||
import { Context } from "@deepseek-ai/cordis";
|
||||
import { createScope, scopeTarget } from "@deepseek-ai/dsh-scope";
|
||||
import { SkillRegistry } from "@deepseek-ai/dsh-skill";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { registerBskSkill } from "../src/skill";
|
||||
import { armAgentScopedBskSkill, registerBskSkill } from "../src/skill";
|
||||
|
||||
function fakeCtx(skills?: unknown) {
|
||||
return { get: (key: string) => (key === "skills" ? skills : undefined) } as never;
|
||||
}
|
||||
|
||||
describe("registerBskSkill", () => {
|
||||
it("registers the catalog entry with the assembled body", () => {
|
||||
it("registers the catalog entry with the DSH browser-tool body", () => {
|
||||
const registrations: { skill: Record<string, unknown>; disposer: () => void }[] = [];
|
||||
const skills = {
|
||||
register(skill: Record<string, unknown>) {
|
||||
@@ -26,23 +29,51 @@ describe("registerBskSkill", () => {
|
||||
expect(skill.name).toBe("browser-skill");
|
||||
expect(typeof skill.description).toBe("string");
|
||||
expect(String(skill.description)).toContain("browser_*");
|
||||
// body = dsh prelude + canonical CLI skill body + dsh postlude
|
||||
const content = String(skill.content);
|
||||
expect(content).toContain("browser_observe");
|
||||
expect(String(skill.description)).toContain("never invoke bsk through bash or shell");
|
||||
expect(content).toContain("Always call those injected `browser_*` tools");
|
||||
expect(content).toContain("Never use `bash`, `shell`, `exec`");
|
||||
expect(content).toContain("## DSH routing reminder");
|
||||
expect(content.trimEnd()).toMatch(
|
||||
/always begin with[\s\S]*`browser_session_start`[\s\S]*`browser_session_stop`\.$/,
|
||||
);
|
||||
expect(content).toContain("Owned sessions only");
|
||||
const mentionedTools = [...new Set(content.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? [])].sort();
|
||||
expect(mentionedTools).toEqual([
|
||||
"browser_click",
|
||||
"browser_console",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_get_html",
|
||||
"browser_hover",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_network",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_reload",
|
||||
"browser_request_help",
|
||||
"browser_screenshot",
|
||||
"browser_select",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_close",
|
||||
"browser_tab_create",
|
||||
"browser_tab_list",
|
||||
"browser_tab_return",
|
||||
"browser_tab_select",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_window_resize",
|
||||
]);
|
||||
expect(String(skill.description)).not.toMatch(/\bbsk\b/i);
|
||||
expect(content).not.toMatch(/\bbsk\b/i);
|
||||
expect(content).not.toMatch(/```(?:bash|sh|shell)\b/i);
|
||||
expect(content).not.toMatch(/--[a-z]/);
|
||||
expect(content).toContain("All browser operations must use the injected tools directly");
|
||||
expect(content).toContain("Mandatory workflow");
|
||||
expect(content).toContain("Refs invalidate after navigation");
|
||||
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
||||
expect(content.length).toBeGreaterThan(10_000);
|
||||
expect(skill.source).toBe("bundled");
|
||||
// the original YAML frontmatter must not leak into the body
|
||||
expect(content.includes("name: browser-skill\ndescription:")).toBe(false);
|
||||
// Source frontmatter is registration metadata and must not leak into the body.
|
||||
expect(content.startsWith("---")).toBe(false);
|
||||
expect(content).not.toContain("name: browser-skill\ndescription:");
|
||||
// disposer passthrough
|
||||
unregister();
|
||||
expect(skill.__disposed).toBe(true);
|
||||
@@ -72,3 +103,88 @@ describe("registerBskSkill", () => {
|
||||
expect(() => disposer()).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("armAgentScopedBskSkill", () => {
|
||||
it("overrides a nearer legacy CLI skill when the DSH agent starts", async () => {
|
||||
const root = new Context();
|
||||
const skillFiber = root.plugin(SkillRegistry);
|
||||
await skillFiber;
|
||||
|
||||
// The plugin's original registration is global.
|
||||
registerBskSkill(skillFiber.ctx);
|
||||
|
||||
// Model the standard preset's filesystem discovery of
|
||||
// ~/.agents/skills/browser-skill. Its nearer scope wins before the fix.
|
||||
const presetKey = { preset: "standard" };
|
||||
const presetScope = createScope(skillFiber.ctx, presetKey);
|
||||
presetScope.ctx.skills.register({
|
||||
name: "browser-skill",
|
||||
description: "Legacy BrowserSkill CLI instructions",
|
||||
content: "Use bsk shell commands.",
|
||||
source: "user-agents",
|
||||
});
|
||||
|
||||
const agentKey = { agent: "test-agent" };
|
||||
const agentScope = createScope(skillFiber.ctx, agentKey, { parent: presetKey });
|
||||
const agent = {
|
||||
id: "test-agent",
|
||||
ctx: agentScope.ctx,
|
||||
} as never;
|
||||
const agents = { list: () => [] };
|
||||
const pluginCtx = skillFiber.ctx.extend({ agents });
|
||||
|
||||
const before = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
|
||||
expect(before?.content).toBe("Use bsk shell commands.");
|
||||
expect(before?.source).toBe("user-agents");
|
||||
|
||||
const disarm = armAgentScopedBskSkill(pluginCtx);
|
||||
pluginCtx.emit(scopeTarget(agent, agentKey), "agent/session-start", {
|
||||
agent,
|
||||
source: "startup",
|
||||
});
|
||||
|
||||
const after = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
|
||||
expect(after?.content).toContain("All browser operations must use the injected tools directly");
|
||||
expect(after?.content).not.toMatch(/\bbsk\b/i);
|
||||
expect(after?.source).toBe("bundled");
|
||||
|
||||
disarm();
|
||||
await agentScope.dispose();
|
||||
await presetScope.dispose();
|
||||
await skillFiber.dispose();
|
||||
});
|
||||
|
||||
it("installs on existing agents for plugin reload and disposes registrations", () => {
|
||||
const registered: Record<string, unknown>[] = [];
|
||||
let disposed = 0;
|
||||
const agent = {
|
||||
ctx: fakeCtx({
|
||||
register(skill: Record<string, unknown>) {
|
||||
registered.push(skill);
|
||||
return () => {
|
||||
disposed += 1;
|
||||
};
|
||||
},
|
||||
}),
|
||||
} as never;
|
||||
const listeners = new Map<string, (payload: { agent: never }) => void>();
|
||||
const ctx = {
|
||||
get: (key: string) => (key === "agents" ? { list: () => [agent] } : undefined),
|
||||
on(event: string, listener: (payload: { agent: never }) => void) {
|
||||
listeners.set(event, listener);
|
||||
return () => listeners.delete(event);
|
||||
},
|
||||
} as never;
|
||||
|
||||
const disarm = armAgentScopedBskSkill(ctx);
|
||||
expect(registered).toHaveLength(1);
|
||||
|
||||
// A later lifecycle notification for the same agent must not duplicate it.
|
||||
listeners.get("agent/session-start")?.({ agent });
|
||||
expect(registered).toHaveLength(1);
|
||||
|
||||
disarm();
|
||||
expect(disposed).toBe(1);
|
||||
expect(listeners.size).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -142,16 +142,33 @@ describe("tool registration", () => {
|
||||
const { tools } = setup({});
|
||||
expect([...tools.keys()].sort()).toEqual([
|
||||
"browser_click",
|
||||
"browser_console",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_get_html",
|
||||
"browser_hover",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_network",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_reload",
|
||||
"browser_request_help",
|
||||
"browser_screenshot",
|
||||
"browser_select",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_close",
|
||||
"browser_tab_create",
|
||||
"browser_tab_list",
|
||||
"browser_tab_return",
|
||||
"browser_tab_select",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_window_resize",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -360,12 +377,12 @@ describe("interaction tools", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
};
|
||||
|
||||
it("browser_click maps target, button, and click count", async () => {
|
||||
it("browser_click maps target, button, click count, and modifiers", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const value = (await click?.execute(
|
||||
{ target: "@e1", button: "right", clickCount: 2 },
|
||||
{ target: "@e1", button: "right", clickCount: 2, modifiers: ["ctrl", "shift"] },
|
||||
makeExec(),
|
||||
)) as { session: string; x: number; y: number };
|
||||
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
|
||||
@@ -377,6 +394,43 @@ describe("interaction tools", () => {
|
||||
"right",
|
||||
"--click-count",
|
||||
"2",
|
||||
"--modifiers",
|
||||
"ctrl,shift",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_click forwards a complete Surface capture coordinate transaction", async () => {
|
||||
const { tools, calls } = setup({
|
||||
...responses,
|
||||
click: {
|
||||
tab_id: 7,
|
||||
used_ref: "e1",
|
||||
x: 110,
|
||||
y: 70,
|
||||
capture_id: "sc_test",
|
||||
image_x: 200,
|
||||
image_y: 100,
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const value = await click?.execute(
|
||||
{ target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
|
||||
makeExec(),
|
||||
);
|
||||
|
||||
expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
|
||||
expect(calls[1].args).toEqual([
|
||||
"click",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"200",
|
||||
"--image-y",
|
||||
"100",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
@@ -436,6 +490,336 @@ describe("interaction tools", () => {
|
||||
/mutually exclusive/,
|
||||
);
|
||||
});
|
||||
|
||||
it("browser_hover and browser_select map their structured arguments", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
hover: { tab_id: 7, used_ref: "@e3", x: 12, y: 24 },
|
||||
select: {
|
||||
tab_id: 7,
|
||||
used_ref: "@e4",
|
||||
multiple: true,
|
||||
selected_values: ["a", "b"],
|
||||
selected_labels: ["Alpha", "Beta"],
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const hovered = (await tools
|
||||
.get("browser_hover")
|
||||
?.execute(
|
||||
{ target: "@e3", modifiers: ["shift"], settleMs: 250, timeoutMs: 2000 },
|
||||
makeExec(),
|
||||
)) as { x: number; y: number };
|
||||
expect(hovered).toMatchObject({ x: 12, y: 24 });
|
||||
expect(calls[1].args).toEqual([
|
||||
"hover",
|
||||
"--session",
|
||||
"s1",
|
||||
"--modifiers",
|
||||
"shift",
|
||||
"--settle",
|
||||
"250ms",
|
||||
"--timeout",
|
||||
"2000ms",
|
||||
"@e3",
|
||||
]);
|
||||
|
||||
const selected = (await tools
|
||||
.get("browser_select")
|
||||
?.execute({ target: "@e4", values: ["a", "b"], tabId: 7 }, makeExec())) as {
|
||||
selectedValues: string[];
|
||||
selectedLabels: string[];
|
||||
};
|
||||
expect(selected).toMatchObject({
|
||||
selectedValues: ["a", "b"],
|
||||
selectedLabels: ["Alpha", "Beta"],
|
||||
});
|
||||
expect(calls[2].args).toEqual([
|
||||
"select",
|
||||
"--session",
|
||||
"s1",
|
||||
"--tab-id",
|
||||
"7",
|
||||
"--value",
|
||||
"a",
|
||||
"--value",
|
||||
"b",
|
||||
"@e4",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("phase-one tab tools", () => {
|
||||
it("maps list/create/close/select/borrow/return through the owned session", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"tab list": {
|
||||
tabs: [
|
||||
{
|
||||
tab_id: 7,
|
||||
title: "Example",
|
||||
url: "https://example.test",
|
||||
window_id: 2,
|
||||
active: true,
|
||||
scope: "agent",
|
||||
},
|
||||
],
|
||||
},
|
||||
"tab create": { tab_id: 8, window_id: 2, url: "https://new.test" },
|
||||
"tab close": { tab_id: 8 },
|
||||
"tab select": { tab_id: 7, window_id: 2 },
|
||||
"tab borrow": {
|
||||
tab_id: 20,
|
||||
original_window_id: 9,
|
||||
original_index: 1,
|
||||
agent_window_id: 2,
|
||||
},
|
||||
"tab return": {
|
||||
tab_id: 20,
|
||||
returned_to_window_id: 9,
|
||||
returned_to_index: 1,
|
||||
fallback: false,
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
|
||||
const listed = (await tools
|
||||
.get("browser_tab_list")
|
||||
?.execute({ scope: "agent" }, makeExec())) as { tabs: { tabId: number }[] };
|
||||
expect(listed.tabs[0].tabId).toBe(7);
|
||||
const created = (await tools
|
||||
.get("browser_tab_create")
|
||||
?.execute({ url: "https://new.test", active: false, index: 1 }, makeExec())) as {
|
||||
tabId: number;
|
||||
};
|
||||
expect(created.tabId).toBe(8);
|
||||
await tools.get("browser_tab_close")?.execute({ tabId: 8 }, makeExec());
|
||||
await tools.get("browser_tab_select")?.execute({ tabId: 7 }, makeExec());
|
||||
const borrowed = (await tools
|
||||
.get("browser_tab_borrow")
|
||||
?.execute({ tabId: 20 }, makeExec())) as { originalWindowId: number };
|
||||
expect(borrowed.originalWindowId).toBe(9);
|
||||
const returned = (await tools
|
||||
.get("browser_tab_return")
|
||||
?.execute({ tabId: 20 }, makeExec())) as { fallback: boolean };
|
||||
expect(returned.fallback).toBe(false);
|
||||
|
||||
expect(calls.slice(1).map((call) => call.args)).toEqual([
|
||||
["tab", "list", "--session", "s1", "--scope", "agent"],
|
||||
[
|
||||
"tab",
|
||||
"create",
|
||||
"--session",
|
||||
"s1",
|
||||
"--url",
|
||||
"https://new.test",
|
||||
"--no-active",
|
||||
"--index",
|
||||
"1",
|
||||
],
|
||||
["tab", "close", "8", "--session", "s1"],
|
||||
["tab", "select", "7", "--session", "s1"],
|
||||
["tab", "borrow", "20", "--session", "s1"],
|
||||
["tab", "return", "20", "--session", "s1"],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("phase-one navigation tools", () => {
|
||||
it("maps back/forward/reload/wait and preserves timeout budgets", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"navigate-back": {
|
||||
tab_id: 7,
|
||||
previous_url: "https://b.test",
|
||||
final_url: "https://a.test",
|
||||
reached: "load",
|
||||
},
|
||||
"navigate-forward": {
|
||||
tab_id: 7,
|
||||
previous_url: "https://a.test",
|
||||
final_url: "https://b.test",
|
||||
reached: "load",
|
||||
},
|
||||
reload: {
|
||||
tab_id: 7,
|
||||
previous_url: "https://b.test",
|
||||
final_url: "https://b.test",
|
||||
reached: "networkidle",
|
||||
},
|
||||
"wait-for-navigation": { tab_id: 7, reached: "timeout", error_text: "no navigation" },
|
||||
});
|
||||
await startSession(tools);
|
||||
await tools
|
||||
.get("browser_navigate_back")
|
||||
?.execute({ tabId: 7, waitUntil: "commit", timeoutMs: 1000 }, makeExec());
|
||||
await tools.get("browser_navigate_forward")?.execute({}, makeExec());
|
||||
await tools
|
||||
.get("browser_reload")
|
||||
?.execute({ hard: true, waitUntil: "networkidle" }, makeExec());
|
||||
const waited = (await tools
|
||||
.get("browser_wait_for_navigation")
|
||||
?.execute({ timeoutMs: 2500 }, makeExec())) as { reached: string; errorText: string };
|
||||
expect(waited).toMatchObject({ reached: "timeout", errorText: "no navigation" });
|
||||
expect(calls[1].args).toEqual([
|
||||
"navigate-back",
|
||||
"--session",
|
||||
"s1",
|
||||
"--tab-id",
|
||||
"7",
|
||||
"--wait-until",
|
||||
"commit",
|
||||
"--timeout",
|
||||
"1000ms",
|
||||
]);
|
||||
expect(calls[2].args).toEqual(["navigate-forward", "--session", "s1"]);
|
||||
expect(calls[3].args).toEqual([
|
||||
"reload",
|
||||
"--session",
|
||||
"s1",
|
||||
"--wait-until",
|
||||
"networkidle",
|
||||
"--hard",
|
||||
]);
|
||||
expect(calls[4].args).toEqual([
|
||||
"wait-for-navigation",
|
||||
"--session",
|
||||
"s1",
|
||||
"--timeout",
|
||||
"2500ms",
|
||||
]);
|
||||
expect(calls[4].options.timeoutMs).toBe(120_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("phase-one support tools", () => {
|
||||
it("maps request-help targets and structured completion criteria", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"request-help": {
|
||||
outcome: "completed",
|
||||
completed_by: "system",
|
||||
note: "signed in",
|
||||
tab_id: 7,
|
||||
resolved_targets: [{ matched: true, ref: "@e3" }],
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const value = (await tools.get("browser_request_help")?.execute(
|
||||
{
|
||||
prompt: "Please sign in",
|
||||
title: "Login required",
|
||||
targets: ["@e3", "#submit"],
|
||||
timeoutMs: 300_000,
|
||||
completionCriteria: {
|
||||
any: [{ urlContains: "/dashboard" }, { selectorExists: "[data-user]" }],
|
||||
stableForMs: 1000,
|
||||
},
|
||||
},
|
||||
makeExec(),
|
||||
)) as { outcome: string; note: string };
|
||||
expect(value).toMatchObject({ outcome: "completed", note: "signed in" });
|
||||
expect(calls[1].options.timeoutMs).toBe(315_000);
|
||||
expect(calls[1].args.slice(0, 8)).toEqual([
|
||||
"request-help",
|
||||
"--session",
|
||||
"s1",
|
||||
"--prompt",
|
||||
"Please sign in",
|
||||
"--timeout",
|
||||
"300000ms",
|
||||
"--title",
|
||||
]);
|
||||
const criteriaIndex = calls[1].args.indexOf("--completion-criteria");
|
||||
expect(JSON.parse(calls[1].args[criteriaIndex + 1])).toEqual({
|
||||
any: [{ url_contains: "/dashboard" }, { selector_exists: "[data-user]" }],
|
||||
stable_for_ms: 1000,
|
||||
});
|
||||
});
|
||||
|
||||
it("maps get-html, console, and network results without raw shell access", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"get-html": { tab_id: 7, html: "<main>Hi</main>", truncated: false, byte_size: 15 },
|
||||
console: {
|
||||
tab_id: 7,
|
||||
entries: [
|
||||
{
|
||||
sequence: 3,
|
||||
kind: "exception",
|
||||
level: "error",
|
||||
text: "boom",
|
||||
stack_trace: [{ function_name: "run", url: "https://a.test/app.js", line: 4 }],
|
||||
},
|
||||
],
|
||||
next_since: 3,
|
||||
truncated: false,
|
||||
},
|
||||
network: {
|
||||
tab_id: 7,
|
||||
entries: [
|
||||
{
|
||||
sequence: 5,
|
||||
kind: "response",
|
||||
method: "GET",
|
||||
url: "https://a.test/api",
|
||||
status: 200,
|
||||
mime_type: "application/json",
|
||||
},
|
||||
],
|
||||
next_since: 5,
|
||||
truncated: false,
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const html = (await tools
|
||||
.get("browser_get_html")
|
||||
?.execute({ ref: "@e2", maxBytes: 2048 }, makeExec())) as { html: string };
|
||||
expect(html.html).toBe("<main>Hi</main>");
|
||||
const consoleValue = (await tools
|
||||
.get("browser_console")
|
||||
?.execute({ since: 1, limit: 10, includeStack: true }, makeExec())) as {
|
||||
entries: { stackTrace: { functionName: string }[] }[];
|
||||
};
|
||||
expect(consoleValue.entries[0].stackTrace[0].functionName).toBe("run");
|
||||
const networkValue = (await tools
|
||||
.get("browser_network")
|
||||
?.execute({ tabId: 7, maxTextChars: 512 }, makeExec())) as {
|
||||
entries: { mimeType: string }[];
|
||||
};
|
||||
expect(networkValue.entries[0].mimeType).toBe("application/json");
|
||||
expect(calls.slice(1).map((call) => call.args)).toEqual([
|
||||
["get-html", "--session", "s1", "--ref", "@e2", "--max-bytes", "2048"],
|
||||
["console", "--session", "s1", "--since", "1", "--limit", "10", "--include-stack"],
|
||||
["network", "--session", "s1", "--tab-id", "7", "--max-text-chars", "512"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("resizes only within the CLI-supported range", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"window resize": { window_id: 2, width: 1280, height: 800 },
|
||||
});
|
||||
await startSession(tools);
|
||||
const resize = tools.get("browser_window_resize");
|
||||
const value = (await resize?.execute({ width: 1280, height: 800 }, makeExec())) as {
|
||||
windowId: number;
|
||||
};
|
||||
expect(value.windowId).toBe(2);
|
||||
expect(calls[1].args).toEqual([
|
||||
"window",
|
||||
"resize",
|
||||
"--session",
|
||||
"s1",
|
||||
"--width",
|
||||
"1280",
|
||||
"--height",
|
||||
"800",
|
||||
]);
|
||||
await expect(resize?.execute({ width: 99, height: 800 }, makeExec())).rejects.toThrow(
|
||||
/100\.\.=7680/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_screenshot", () => {
|
||||
@@ -467,6 +851,32 @@ describe("browser_screenshot", () => {
|
||||
expect(rendered?.every((block) => block.type === "text")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns Surface capture metadata alongside the screenshot", async () => {
|
||||
const path = pngFile();
|
||||
const { tools } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: {
|
||||
tab_id: 7,
|
||||
width: 800,
|
||||
height: 600,
|
||||
format: "png",
|
||||
path,
|
||||
byte_size: 8,
|
||||
capture: { id: "sc_test", expires_at: 1700000030000 },
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
|
||||
|
||||
expect(value).toMatchObject({
|
||||
captureId: "sc_test",
|
||||
captureExpiresAt: 1700000030000,
|
||||
});
|
||||
const rendered = screenshot?.output.render({ session: "s1" }, value as never);
|
||||
expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
|
||||
});
|
||||
|
||||
it("commits the image through the attachment store on an image-capable route", async () => {
|
||||
const path = pngFile();
|
||||
const services = {
|
||||
|
||||
@@ -29,6 +29,66 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("renders visual surfaces as screenshot-only refs", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 2,
|
||||
backendNodeId: 99,
|
||||
frameId: "child",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
label: "Data grid",
|
||||
memberCount: 2,
|
||||
},
|
||||
];
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
backendNodeId: 99,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("renders an unnamed visual surface with a stable generic label", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
name: "canvas visual surface",
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
@@ -15,6 +15,8 @@ export type {
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomRef,
|
||||
VomRefCapability,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
VomOptions,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
@@ -737,6 +738,53 @@ interface RenderState {
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
visualSurfaces: Map<number | null, VomVisualSurface[]>;
|
||||
}
|
||||
|
||||
function groupedByParent<T extends { parentId: number | null }>(
|
||||
items: T[],
|
||||
): Map<number | null, T[]> {
|
||||
const grouped = new Map<number | null, T[]>();
|
||||
for (const item of items) {
|
||||
const siblings = grouped.get(item.parentId) ?? [];
|
||||
siblings.push(item);
|
||||
grouped.set(item.parentId, siblings);
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
||||
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
|
||||
if (state.stopped) return;
|
||||
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
|
||||
if (depth > state.maxDepth) {
|
||||
state.truncated = true;
|
||||
continue;
|
||||
}
|
||||
const ref = `e${state.nextRef}`;
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
state.refs.push({
|
||||
ref,
|
||||
backendNodeId: surface.backendNodeId,
|
||||
frameId: surface.frameId,
|
||||
visibleRect: surface.visibleRect,
|
||||
role: "surface",
|
||||
name: label,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
||||
@@ -812,6 +860,8 @@ function renderTree(
|
||||
...(node.role ? { role: node.role } : {}),
|
||||
...(name ? { name } : {}),
|
||||
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
@@ -821,10 +871,12 @@ function renderTree(
|
||||
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
||||
|
||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||
emitVisualEntries(node.id, depth + 1, state);
|
||||
continue;
|
||||
}
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
}
|
||||
emitVisualEntries(parentId, depth, state);
|
||||
}
|
||||
|
||||
function renderNodes(
|
||||
@@ -833,6 +885,7 @@ function renderNodes(
|
||||
initialLines: string[],
|
||||
surfaces: CondSurface[] = [],
|
||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||
visualSurfaces: VomVisualSurface[] = [],
|
||||
): RenderState {
|
||||
const children = buildChildren(nodes);
|
||||
const state: RenderState = {
|
||||
@@ -851,6 +904,7 @@ function renderNodes(
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
visualSurfaces: groupedByParent(visualSurfaces),
|
||||
};
|
||||
|
||||
renderTree(children, null, 1, state);
|
||||
@@ -1056,7 +1110,16 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
"@layers 2 focus=L1",
|
||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||
];
|
||||
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
|
||||
const visibleSurface = (parentId: number | null): boolean =>
|
||||
parentId === null || included.has(parentId);
|
||||
const state = renderNodes(
|
||||
visibleNodes,
|
||||
options,
|
||||
header,
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||
);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
return {
|
||||
@@ -1084,6 +1147,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
],
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces,
|
||||
);
|
||||
|
||||
return {
|
||||
|
||||
@@ -61,6 +61,8 @@ export interface VomScene {
|
||||
rootFrameId?: string;
|
||||
surfaces?: CondSurface[];
|
||||
activeScopeBlocks?: ActiveScopeBlock[];
|
||||
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
|
||||
visualSurfaces?: VomVisualSurface[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
@@ -94,6 +96,22 @@ export interface VomRef {
|
||||
ref: string;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
/** Observation-time top-viewport crop for screenshot-only surfaces. */
|
||||
visibleRect?: Rect;
|
||||
kind?: "dom" | "surface";
|
||||
capabilities?: VomRefCapability[];
|
||||
}
|
||||
|
||||
export type VomRefCapability = "interact" | "screenshot";
|
||||
|
||||
export interface VomVisualSurface {
|
||||
parentId: number | null;
|
||||
backendNodeId: number;
|
||||
frameId: string;
|
||||
renderingKind: "canvas";
|
||||
visibleRect: Rect;
|
||||
label?: string;
|
||||
memberCount: number;
|
||||
}
|
||||
|
||||
export interface RenderedRef extends VomRef {
|
||||
|
||||
Generated
+6
@@ -114,6 +114,12 @@ importers:
|
||||
'@deepseek-ai/dsh-llm':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
|
||||
'@deepseek-ai/dsh-scope':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
||||
'@deepseek-ai/dsh-skill':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-llm@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))))(@deepseek-ai/dsh-scope@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
|
||||
'@deepseek-ai/dsh-tools':
|
||||
specifier: ^0.1.0-rc.6
|
||||
version: 0.1.0-rc.6(f8724372086ccc1457fc84e7becee2e0)
|
||||
|
||||
+12
-2
@@ -80,6 +80,16 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
||||
|
||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
|
||||
|
||||
```bash
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
@@ -174,7 +184,7 @@ bsk emulate --session <id> --off
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
@@ -200,7 +210,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
|
||||
Reference in new issue
Block a user