mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5cd43553ed | ||
|
|
9ad33801c7 |
No files matched your search
@@ -11,6 +11,10 @@ dist/
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Local browser evaluation artifacts
|
||||
evals/browser/results/
|
||||
evals/browser/agents.local.json
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
|
||||
@@ -208,6 +208,7 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -13,8 +13,6 @@ describe("RefStore", () => {
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
generation: 0,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
expect(s.size()).toBe(1);
|
||||
expect(s.isEmpty()).toBe(false);
|
||||
@@ -50,31 +48,4 @@ describe("RefStore", () => {
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit capabilities for visual surface refs", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
});
|
||||
|
||||
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, { tabId: 7, kind: "surface" });
|
||||
s.set("e2", 43, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
||||
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
|
||||
});
|
||||
});
|
||||
@@ -1,67 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
|
||||
|
||||
function input(): SurfaceCaptureInput {
|
||||
return {
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "main",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: 2,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 100,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
};
|
||||
}
|
||||
|
||||
describe("SurfaceCaptureStore", () => {
|
||||
it("creates metadata-only, short-lived captures and consumes them once", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 500,
|
||||
createId: () => "sc_test",
|
||||
});
|
||||
const capture = store.create(input());
|
||||
|
||||
expect(capture).toMatchObject({
|
||||
id: "sc_test",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 1_500,
|
||||
consumed: false,
|
||||
});
|
||||
expect(capture).not.toHaveProperty("imageBase64");
|
||||
expect(store.consume("sc_test")).toMatchObject({ ok: true });
|
||||
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
|
||||
|
||||
now = 1_600;
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects expired and unknown captures", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 10,
|
||||
createId: () => "sc_expired",
|
||||
});
|
||||
store.create(input());
|
||||
now = 1_010;
|
||||
|
||||
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
|
||||
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
|
||||
});
|
||||
|
||||
it("clears every capture on session cleanup", () => {
|
||||
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
|
||||
store.create(input());
|
||||
store.clear();
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,10 @@
|
||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||
import { RefStore } from "./ref-store";
|
||||
import { SurfaceCaptureStore } from "./surface-capture-store";
|
||||
|
||||
export interface SessionContext {
|
||||
sessionId: string;
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
surfaceCaptures: SurfaceCaptureStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
@@ -188,7 +186,6 @@ export class SessionManager {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
|
||||
borrowedTabs: new Map(),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
@@ -14,17 +12,12 @@ import type { Rect } from "@browser-skill/vom";
|
||||
* inside the `SessionContext`.
|
||||
*/
|
||||
export type BackendNodeId = number;
|
||||
export type RefCapability = "interact" | "screenshot";
|
||||
export type RefTargetKind = "dom" | "surface";
|
||||
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
@@ -35,19 +28,8 @@ export type RefInput =
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
};
|
||||
|
||||
function capabilitiesFor(
|
||||
kind: RefTargetKind,
|
||||
capabilities: RefCapability[] | undefined,
|
||||
): RefCapability[] {
|
||||
if (kind === "surface") return ["screenshot"];
|
||||
return capabilities ?? ["interact", "screenshot"];
|
||||
}
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
private generation = 0;
|
||||
@@ -88,20 +70,13 @@ export class RefStore {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
} = {},
|
||||
): void {
|
||||
const kind = opts.kind ?? "dom";
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, opts.capabilities),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -119,20 +94,14 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input,
|
||||
tabId: null,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
const kind = input.kind ?? "dom";
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, input.capabilities),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export const SURFACE_CAPTURE_TTL_MS = 30_000;
|
||||
|
||||
export interface SurfaceCapture {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
tabId: number;
|
||||
navigationIdentity: string;
|
||||
surface: {
|
||||
ref: string;
|
||||
frameId?: string;
|
||||
backendNodeId: number;
|
||||
observationGeneration: number;
|
||||
};
|
||||
topViewportRect: Rect;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
createdAt: number;
|
||||
expiresAt: number;
|
||||
consumed: boolean;
|
||||
}
|
||||
|
||||
export type SurfaceCaptureInput = Omit<
|
||||
SurfaceCapture,
|
||||
"id" | "createdAt" | "expiresAt" | "consumed"
|
||||
>;
|
||||
|
||||
export type SurfaceCaptureConsumeResult =
|
||||
| { ok: true; capture: SurfaceCapture }
|
||||
| { ok: false; reason: "not_found" | "expired" | "consumed" };
|
||||
|
||||
export interface SurfaceCaptureStoreOptions {
|
||||
now?: () => number;
|
||||
ttlMs?: number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
function randomCaptureId(): string {
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(12));
|
||||
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
||||
}
|
||||
|
||||
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
|
||||
export class SurfaceCaptureStore {
|
||||
private readonly captures = new Map<string, SurfaceCapture>();
|
||||
private readonly now: () => number;
|
||||
private readonly ttlMs: number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(options: SurfaceCaptureStoreOptions = {}) {
|
||||
this.now = options.now ?? Date.now;
|
||||
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
|
||||
this.createId = options.createId ?? randomCaptureId;
|
||||
}
|
||||
|
||||
create(input: SurfaceCaptureInput): SurfaceCapture {
|
||||
this.purgeExpired();
|
||||
const createdAt = this.now();
|
||||
const capture: SurfaceCapture = {
|
||||
...input,
|
||||
id: this.createId(),
|
||||
createdAt,
|
||||
expiresAt: createdAt + this.ttlMs,
|
||||
consumed: false,
|
||||
};
|
||||
this.captures.set(capture.id, capture);
|
||||
return capture;
|
||||
}
|
||||
|
||||
consume(id: string): SurfaceCaptureConsumeResult {
|
||||
const capture = this.captures.get(id);
|
||||
if (!capture) return { ok: false, reason: "not_found" };
|
||||
if (capture.expiresAt <= this.now()) {
|
||||
this.captures.delete(id);
|
||||
return { ok: false, reason: "expired" };
|
||||
}
|
||||
if (capture.consumed) return { ok: false, reason: "consumed" };
|
||||
capture.consumed = true;
|
||||
return { ok: true, capture };
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.captures.clear();
|
||||
}
|
||||
|
||||
size(): number {
|
||||
this.purgeExpired();
|
||||
return this.captures.size;
|
||||
}
|
||||
|
||||
private purgeExpired(): void {
|
||||
const now = this.now();
|
||||
for (const [id, capture] of this.captures) {
|
||||
if (capture.expiresAt <= now) this.captures.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
parseKeySpec,
|
||||
resolveKeyDescriptor,
|
||||
} from "../interaction";
|
||||
import { captureSurfaceEnvironment } from "../surface-target";
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
@@ -34,15 +33,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (
|
||||
!h &&
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression ?? "").includes(
|
||||
"bsk-focus-target-probe",
|
||||
)
|
||||
) {
|
||||
return { result: { value: { hasFocus: true, activeTag: "input" } } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -54,10 +44,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
getFrameGraph: vi.fn(async (tabId) => ({
|
||||
rootFrameId: "main",
|
||||
frames: [{ frameId: "main", target: { tabId } }],
|
||||
})),
|
||||
};
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
@@ -68,71 +54,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
return { cdp, tabsApi, sent };
|
||||
}
|
||||
|
||||
function fillRuntimeReadback(value: string) {
|
||||
return (params: unknown) => {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-input-readback")) {
|
||||
return { result: { value: { supported: true, value } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
};
|
||||
}
|
||||
|
||||
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Page.getFrameTree": () => ({
|
||||
frameTree: {
|
||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
||||
},
|
||||
}),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
scale: 1,
|
||||
},
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { overlayHostPresent: false, overlayHostConnected: false } },
|
||||
}),
|
||||
"Input.dispatchMouseEvent": (params) => {
|
||||
onMouse?.(params as Record<string, unknown>);
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const environment = await captureSurfaceEnvironment(fake.cdp, 4, undefined);
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing Surface ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
return { manager, context, fake, capture };
|
||||
}
|
||||
|
||||
describe("modifiersBitfield", () => {
|
||||
it("matches CDP's expected bit layout", () => {
|
||||
expect(modifiersBitfield([])).toBe(0);
|
||||
@@ -202,29 +123,6 @@ describe("handleClick", () => {
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||
});
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -269,74 +167,6 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
"left",
|
||||
"middle",
|
||||
"right",
|
||||
] as const)("uses the common click executor for a %s-button Surface click", async (button) => {
|
||||
const { manager, fake, capture } = await makeSurfaceClickFixture();
|
||||
const result = await handleClick(
|
||||
manager,
|
||||
{
|
||||
session_id: "aa11",
|
||||
ref: "@e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
button,
|
||||
click_count: 2,
|
||||
modifiers: ["ctrl", "shift"],
|
||||
},
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
used_ref: "e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
x: 35,
|
||||
y: 30,
|
||||
});
|
||||
const mouse = fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent");
|
||||
expect(mouse).toHaveLength(3);
|
||||
expect(mouse[1].params).toMatchObject({
|
||||
type: "mousePressed",
|
||||
button,
|
||||
clickCount: 2,
|
||||
modifiers: 2 | 8,
|
||||
});
|
||||
expect(mouse[2].params).toMatchObject({
|
||||
type: "mouseReleased",
|
||||
button,
|
||||
clickCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("consumes a Surface capture when the common click executor fails", async () => {
|
||||
let mouseCalls = 0;
|
||||
const { manager, fake, capture } = await makeSurfaceClickFixture((params) => {
|
||||
mouseCalls += 1;
|
||||
if (params.type === "mousePressed") throw new Error("input failed");
|
||||
});
|
||||
const params = {
|
||||
session_id: "aa11",
|
||||
ref: "@e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
};
|
||||
|
||||
expect(
|
||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
||||
).toMatchObject({ code: "cdp_failed" });
|
||||
expect(mouseCalls).toBe(2);
|
||||
expect(
|
||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -756,7 +586,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback("hello"),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -789,9 +619,6 @@ describe("handleFill", () => {
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (fn.includes("bsk-input-readback")) {
|
||||
return { result: { value: { supported: true, value: "existingx" } } };
|
||||
}
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
@@ -805,7 +632,7 @@ describe("handleFill", () => {
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.value_length).toBe(9);
|
||||
expect(res.value_length).toBe(1);
|
||||
});
|
||||
|
||||
it("treats contenteditable=true as fillable", async () => {
|
||||
@@ -823,7 +650,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback("rich"),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -834,88 +661,6 @@ describe("handleFill", () => {
|
||||
expect("code" in res).toBe(false);
|
||||
});
|
||||
|
||||
it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
childCalls.push(method);
|
||||
if (method === "DOM.describeNode") {
|
||||
return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
|
||||
}
|
||||
if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
|
||||
if (method === "Input.insertText") {
|
||||
expect(params).toEqual({ text: "中文输入" });
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
return fn.includes("bsk-input-readback")
|
||||
? { result: { value: { supported: true, value: "中文输入" } } }
|
||||
: { result: { type: "undefined" } };
|
||||
}
|
||||
throw new Error(`unexpected child call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleFill(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", value: "中文输入" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(JSON.stringify(res));
|
||||
expect(res.value_length).toBe(4);
|
||||
expect(childCalls).toContain("DOM.focus");
|
||||
expect(childCalls).toContain("Input.insertText");
|
||||
expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
|
||||
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns input_not_applied when a controlled input resets the value", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 8, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback(""),
|
||||
});
|
||||
|
||||
const res = await handleFill(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", value: "rejected" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -1110,125 +855,6 @@ describe("handlePress", () => {
|
||||
expect(res.key).toBe("Enter");
|
||||
});
|
||||
|
||||
it("routes ref-targeted key events through the OOPIF session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 555, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "Enter", ref: "@e1" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expectPressOk(res);
|
||||
expect(childCalls).toEqual([
|
||||
"DOM.scrollIntoViewIfNeeded",
|
||||
"DOM.focus",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "iframe" } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "a" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expectPressOk(res);
|
||||
expect(childCalls).toEqual([
|
||||
"Runtime.evaluate",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
});
|
||||
|
||||
it("refuses an untargeted press when focus ownership is ambiguous", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "input" } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
|
||||
],
|
||||
}));
|
||||
fake.cdp.sendToTarget = vi.fn(async () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "textarea" } },
|
||||
})) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "Enter" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "focus_target_unresolved", candidate_count: 2 },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before focus when abort fires after press scroll", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { dispatchMouseClick } from "../mouse-input";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
function fakeCdp(onSend?: (method: string, params: Record<string, unknown>) => void): {
|
||||
cdp: CdpRunner;
|
||||
events: Record<string, unknown>[];
|
||||
} {
|
||||
const events: Record<string, unknown>[] = [];
|
||||
return {
|
||||
events,
|
||||
cdp: {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
const event = params as Record<string, unknown>;
|
||||
events.push(event);
|
||||
onSend?.(method, event);
|
||||
return {} as never;
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("dispatchMouseClick", () => {
|
||||
it("emits one move, press, and release sequence", async () => {
|
||||
const { cdp, events } = fakeCdp();
|
||||
|
||||
expect(
|
||||
await dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 12, y: 34 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
),
|
||||
).toBe("completed");
|
||||
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "mouseMoved",
|
||||
x: 12,
|
||||
y: 34,
|
||||
modifiers: 0,
|
||||
},
|
||||
{
|
||||
type: "mousePressed",
|
||||
x: 12,
|
||||
y: 34,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
},
|
||||
{
|
||||
type: "mouseReleased",
|
||||
x: 12,
|
||||
y: 34,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("releases a pressed button when cancellation arrives", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, events } = fakeCdp((_method, event) => {
|
||||
if (event.type === "mousePressed") controller.abort();
|
||||
});
|
||||
|
||||
expect(
|
||||
await dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 1, y: 2 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
signal: controller.signal,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
),
|
||||
).toBe("cancelled");
|
||||
expect(events.at(-1)).toMatchObject({ type: "mouseReleased" });
|
||||
});
|
||||
|
||||
it("settles pointer-move hit testing before pressing", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const { cdp, events } = fakeCdp();
|
||||
const click = dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 1, y: 2 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
moveSettleMs: 32,
|
||||
},
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(31);
|
||||
expect(events.map((event) => event.type)).toEqual(["mouseMoved"]);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await click;
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"mouseMoved",
|
||||
"mousePressed",
|
||||
"mouseReleased",
|
||||
]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -67,9 +67,6 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler && method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -303,82 +300,6 @@ describe("handleScreenshot", () => {
|
||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
||||
});
|
||||
|
||||
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { width: number; height: number; scale: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
|
||||
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
|
||||
expect(res.capture).toMatchObject({
|
||||
surface_ref: "@e5",
|
||||
coordinate_space: "capture-image-pixel",
|
||||
});
|
||||
expect(ctx.surfaceCaptures.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("crops a visual surface screenshot to its observation-time visible region", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
||||
expect(res.capture?.id).toMatch(/^sc_/);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -2772,85 +2693,6 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Canvas app" },
|
||||
backendDOMNodeId: 100,
|
||||
};
|
||||
const strings = ["body", "canvas", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("canvas")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 30, 800, 500],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
conditionalSurfaceProbe: false,
|
||||
};
|
||||
|
||||
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
|
||||
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
|
||||
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 200,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
|
||||
expect(snapshot.text).not.toContain("visual surface");
|
||||
expect(snapshot.ref_count).toBe(0);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
|
||||
@@ -252,23 +252,12 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clears refs and Surface captures before window teardown", async () => {
|
||||
it("clears the RefStore before window teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Insert a fake ref so we can verify clear() ran.
|
||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||
ctx.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 },
|
||||
topViewportRect: { x: 0, y: 0, w: 100, h: 50 },
|
||||
imageWidth: 100,
|
||||
imageHeight: 50,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
});
|
||||
const state: FakeState = {
|
||||
tabs: new Map(),
|
||||
windowsClosed: new Set(),
|
||||
@@ -277,6 +266,5 @@ describe("handleSessionStop with auto-return", () => {
|
||||
const { tabs, windows } = makeApis(state);
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
expect(ctx.surfaceCaptures.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -24,16 +24,10 @@ describe("lookupSnapshotRef", () => {
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -59,8 +53,6 @@ describe("resolveSnapshotRef", () => {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
const expected = {
|
||||
@@ -68,9 +60,6 @@ describe("resolveSnapshotRef", () => {
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom" as const,
|
||||
capabilities: ["interact", "screenshot"] as const,
|
||||
generation: 0,
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
@@ -116,29 +105,6 @@ describe("resolveSnapshotRef", () => {
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an operation outside the ref's declared capabilities", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
|
||||
code: "permission_denied",
|
||||
message: "ref @e3 does not support interact",
|
||||
data: {
|
||||
reason: "ref_capability_denied",
|
||||
required_capability: "interact",
|
||||
kind: "surface",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,52 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
mapImagePointToViewport,
|
||||
pointInRegion,
|
||||
sameRect,
|
||||
surfaceVisibleRect,
|
||||
} from "../surface-coordinate";
|
||||
|
||||
describe("surface coordinate mapping", () => {
|
||||
it("maps capture image pixels through the actual screenshot dimensions", () => {
|
||||
expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({
|
||||
x: 200,
|
||||
y: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects invalid image coordinates and dimensions", () => {
|
||||
const rect = { x: 0, y: 0, w: 100, h: 50 };
|
||||
expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull();
|
||||
});
|
||||
|
||||
it("intersects live and observed visible regions and compares them strictly", () => {
|
||||
const clipped = surfaceVisibleRect(
|
||||
{ x: 0, y: 0, width: 200, height: 100 },
|
||||
{ x: 25, y: 30, w: 50, h: 40 },
|
||||
);
|
||||
expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 });
|
||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 25, y: 30, w: 50, h: 40 })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 26, y: 30, w: 50, h: 40 })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("checks the projected visible region rather than only its bounding box", () => {
|
||||
const triangle = [
|
||||
[
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 100, y: 0 },
|
||||
{ x: 0, y: 100 },
|
||||
],
|
||||
];
|
||||
expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true);
|
||||
expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,306 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import {
|
||||
captureSurfaceEnvironment,
|
||||
resolveSurfacePointerTarget,
|
||||
SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
} from "../surface-target";
|
||||
|
||||
function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeBrowser() {
|
||||
let loaderId = "loader-1";
|
||||
let pageY = 0;
|
||||
let quad = [10, 20, 110, 20, 110, 60, 10, 60];
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Page.getFrameTree") {
|
||||
return {
|
||||
frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } },
|
||||
} as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY,
|
||||
scale: 1,
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "DOM.getContentQuads") return { quads: [quad] } as never;
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
return {
|
||||
cdp,
|
||||
sent,
|
||||
setLoaderId: (value: string) => {
|
||||
loaderId = value;
|
||||
},
|
||||
setPageY: (value: number) => {
|
||||
pageY = value;
|
||||
},
|
||||
setQuad: (value: number[]) => {
|
||||
quad = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function setupCapture() {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
const browser = fakeBrowser();
|
||||
const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined);
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
return { manager, context, browser, capture };
|
||||
}
|
||||
|
||||
function pointParams(captureId: string) {
|
||||
return {
|
||||
ref: "@e3",
|
||||
captureId,
|
||||
imageX: 50,
|
||||
imageY: 20,
|
||||
};
|
||||
}
|
||||
|
||||
describe("Surface screenshot-bound pointer target", () => {
|
||||
it("maps one fresh capture coordinate without executing an action", async () => {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
usedRef: "e3",
|
||||
point: { x: 35, y: 30 },
|
||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
capture: { id: capture.id, imageX: 50, imageY: 20 },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
|
||||
const repeated = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(repeated).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_consumed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a new observation generation before sending input", async () => {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
context.refStore.replace([
|
||||
[
|
||||
"e3",
|
||||
{
|
||||
backendNodeId: 99,
|
||||
tabId: 4,
|
||||
kind: "surface" as const,
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_stale" },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects navigation, scroll, and visible geometry changes", async () => {
|
||||
for (const mutate of [
|
||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setLoaderId("loader-2"),
|
||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setPageY(20),
|
||||
(browser: ReturnType<typeof fakeBrowser>) =>
|
||||
browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]),
|
||||
]) {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
mutate(browser);
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_stale" },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("consumes the capture when coordinate validation fails", async () => {
|
||||
const invalid = await setupCapture();
|
||||
const outside = await resolveSurfacePointerTarget(
|
||||
invalid.context,
|
||||
{ tabId: 4 },
|
||||
{ ...pointParams(invalid.capture.id), imageX: 200 },
|
||||
{ cdp: invalid.browser.cdp },
|
||||
);
|
||||
expect(outside).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "surface_coordinate_invalid" },
|
||||
});
|
||||
expect(
|
||||
await resolveSurfacePointerTarget(
|
||||
invalid.context,
|
||||
{ tabId: 4 },
|
||||
pointParams(invalid.capture.id),
|
||||
{ cdp: invalid.browser.cdp },
|
||||
),
|
||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
||||
});
|
||||
|
||||
it("maps image coordinates through an OOPIF frame projection", async () => {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "surface",
|
||||
visibleRect: { x: 120, y: 110, w: 100, h: 40 },
|
||||
});
|
||||
const graph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 77,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Page.getFrameTree") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
scale: 1,
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never;
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {} as never;
|
||||
throw new Error(`unexpected root CDP call ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never;
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => graph),
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
const environment = await captureSurfaceEnvironment(cdp, 4, "child");
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "child",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 120, y: 110, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
{
|
||||
ref: "@e3",
|
||||
captureId: capture.id,
|
||||
imageX: 100,
|
||||
imageY: 40,
|
||||
},
|
||||
{ cdp },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({ point: { x: 170, y: 130 } });
|
||||
expect(sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,7 @@
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -32,7 +32,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import { dispatchMouseClick } from "./mouse-input";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -44,7 +43,6 @@ import {
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||
|
||||
export interface InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
@@ -161,7 +159,7 @@ async function resolveBackendNode(
|
||||
};
|
||||
}
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
@@ -217,131 +215,15 @@ async function resolveBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
interface InputDispatchContext {
|
||||
target: CdpTarget;
|
||||
cdp: CdpRunner;
|
||||
backendNodeId?: number;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
|
||||
function inputContextForNode(
|
||||
cdp: CdpRunner,
|
||||
node: Awaited<ReturnType<typeof resolveBackendNode>>,
|
||||
): (InputDispatchContext & { backendNodeId: number }) | RpcError {
|
||||
if (isRpcError(node)) return node;
|
||||
return {
|
||||
target: node.cdpTarget,
|
||||
cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
|
||||
backendNodeId: node.backendNodeId,
|
||||
frameId: node.frameId,
|
||||
usedRef: node.usedRef,
|
||||
usedSelector: node.usedSelector,
|
||||
};
|
||||
}
|
||||
|
||||
interface FocusProbe {
|
||||
hasFocus: boolean;
|
||||
activeTag: string;
|
||||
}
|
||||
|
||||
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
||||
async function resolveFocusedInputContext(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<InputDispatchContext | RpcError> {
|
||||
if (!cdp.getFrameGraph) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
"press cannot determine the focused frame target",
|
||||
);
|
||||
}
|
||||
|
||||
let targets: CdpTarget[];
|
||||
try {
|
||||
const graph = await cdp.getFrameGraph(tabId);
|
||||
const unique = new Map<string, CdpTarget>();
|
||||
unique.set(cdpTargetKey({ tabId }), { tabId });
|
||||
for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
|
||||
targets = [...unique.values()];
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
|
||||
await Promise.all(
|
||||
targets.map(async (candidate) => {
|
||||
try {
|
||||
const reply = await cdpRunnerForTarget(cdp, candidate).send<{
|
||||
result?: { value?: FocusProbe };
|
||||
}>(tabId, "Runtime.evaluate", {
|
||||
expression: `(() => {
|
||||
/* bsk-focus-target-probe */
|
||||
const active = document.activeElement;
|
||||
return {
|
||||
hasFocus: document.hasFocus(),
|
||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
|
||||
};
|
||||
})()`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const probe = reply.result?.value;
|
||||
if (probe?.hasFocus) focused.push({ target: candidate, probe });
|
||||
} catch {
|
||||
// A target may navigate or detach while the graph is being probed.
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// A top document whose active element is an iframe delegates focus to
|
||||
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
||||
// target while same-process frames continue to use the shared root target.
|
||||
const owners = focused.filter(
|
||||
({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
|
||||
);
|
||||
if (owners.length !== 1) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
owners.length === 0
|
||||
? "press found no unique focused frame target"
|
||||
: "press found multiple focused frame targets",
|
||||
{ candidate_count: owners.length },
|
||||
);
|
||||
}
|
||||
const target = owners[0].target;
|
||||
return { target, cdp: cdpRunnerForTarget(cdp, target) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface ResolvedClickPointerTarget {
|
||||
point: { x: number; y: number };
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
moveSettleMs: number;
|
||||
capture?: {
|
||||
id: string;
|
||||
imageX: number;
|
||||
imageY: number;
|
||||
};
|
||||
}
|
||||
|
||||
export async function handleClick(
|
||||
manager: SessionManager,
|
||||
params: ClickParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const hasSurfacePointParams =
|
||||
params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined;
|
||||
const ctxOrErr = lookupSession(manager, params, "click");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -353,67 +235,39 @@ export async function handleClick(
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const button: MouseButton = params.button ?? "left";
|
||||
const clickCount = params.click_count ?? 1;
|
||||
if (!Number.isSafeInteger(clickCount) || clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be a positive integer" };
|
||||
}
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const pointerTarget: ResolvedClickPointerTarget | RpcError = hasSurfacePointParams
|
||||
? await resolveSurfacePointerTarget(
|
||||
ctx,
|
||||
target,
|
||||
{
|
||||
ref: params.ref,
|
||||
selector: params.selector,
|
||||
captureId: params.capture_id,
|
||||
imageX: params.image_x,
|
||||
imageY: params.image_y,
|
||||
},
|
||||
{
|
||||
cdp: deps.cdp,
|
||||
signal: deps.signal,
|
||||
},
|
||||
)
|
||||
: await (async () => {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
return {
|
||||
point: geometry.actionPoint,
|
||||
usedRef: node.usedRef,
|
||||
usedSelector: node.usedSelector,
|
||||
moveSettleMs: 0,
|
||||
};
|
||||
})();
|
||||
if (isRpcError(pointerTarget)) return pointerTarget;
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
const overlayBlocking = await checkOverlayAtPoint(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
pointerTarget.point.x,
|
||||
pointerTarget.point.y,
|
||||
);
|
||||
const button: MouseButton = params.button ?? "left";
|
||||
const clickCount = params.click_count ?? 1;
|
||||
if (clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be greater than zero" };
|
||||
}
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
@@ -425,16 +279,47 @@ export async function handleClick(
|
||||
}
|
||||
|
||||
try {
|
||||
const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, {
|
||||
// Move first so hover state activates, then press → release.
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
modifiers,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
signal: deps.signal,
|
||||
moveSettleMs: pointerTarget.moveSettleMs,
|
||||
});
|
||||
if (dispatch === "cancelled") {
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err);
|
||||
}
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -452,17 +337,10 @@ export async function handleClick(
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: pointerTarget.usedRef,
|
||||
used_selector: pointerTarget.usedSelector,
|
||||
x: pointerTarget.point.x,
|
||||
y: pointerTarget.point.y,
|
||||
...(pointerTarget.capture
|
||||
? {
|
||||
capture_id: pointerTarget.capture.id,
|
||||
image_x: pointerTarget.capture.imageX,
|
||||
image_y: pointerTarget.capture.imageY,
|
||||
}
|
||||
: {}),
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -720,9 +598,8 @@ export async function handleFill(
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
const input = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(input)) return input;
|
||||
const nodeCdp = input.cdp;
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -730,7 +607,7 @@ export async function handleFill(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
backendNodeId: input.backendNodeId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
},
|
||||
);
|
||||
if (!described.node || !isFillable(described.node)) {
|
||||
@@ -743,15 +620,15 @@ export async function handleFill(
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
input.target,
|
||||
input.backendNodeId,
|
||||
input.frameId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -763,7 +640,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -794,7 +671,7 @@ export async function handleFill(
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
@@ -807,28 +684,6 @@ export async function handleFill(
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
// Give controlled inputs one bounded browser-render turn to accept or
|
||||
// reject the dispatched value before reporting success.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-input-sync */
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, 100);
|
||||
if (typeof requestAnimationFrame !== 'function') { finish(); return; }
|
||||
requestAnimationFrame(() => requestAnimationFrame(finish));
|
||||
});
|
||||
}`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -836,56 +691,11 @@ export async function handleFill(
|
||||
};
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
let actualValue: string;
|
||||
try {
|
||||
const readback = await nodeCdp.send<{
|
||||
result?: { value?: { supported?: boolean; value?: string } };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-input-readback */
|
||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||
return { supported: true, value: this.value };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
return { supported: true, value: this.innerText ?? this.textContent ?? '' };
|
||||
}
|
||||
return { supported: false, value: '' };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const state = readback.result?.value;
|
||||
actualValue = typeof state?.value === "string" ? state.value : "";
|
||||
const applied =
|
||||
state?.supported === true &&
|
||||
(clearBefore
|
||||
? actualValue === params.value
|
||||
: params.value.length === 0 || actualValue.includes(params.value));
|
||||
if (!applied) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the page did not retain the requested input value",
|
||||
{ expected_value_length: params.value.length, actual_value_length: actualValue.length },
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: input.usedRef,
|
||||
used_selector: input.usedSelector,
|
||||
value_length: actualValue.length,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1060,38 +870,31 @@ export async function handlePress(
|
||||
};
|
||||
}
|
||||
|
||||
let input: InputDispatchContext | RpcError;
|
||||
// Optional focus before key dispatch.
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(explicitInput)) return explicitInput;
|
||||
input = explicitInput;
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
explicitInput.target,
|
||||
explicitInput.backendNodeId,
|
||||
explicitInput.frameId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||
backendNodeId: explicitInput.backendNodeId,
|
||||
});
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
} else {
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId);
|
||||
if (isRpcError(input)) return input;
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -1105,7 +908,7 @@ export async function handlePress(
|
||||
try {
|
||||
let cancelled = false;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
@@ -1119,7 +922,7 @@ export async function handlePress(
|
||||
typeof descriptor.text === "string" &&
|
||||
descriptor.text.length > 0
|
||||
) {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "char",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
@@ -1136,7 +939,7 @@ export async function handlePress(
|
||||
cancelled = true;
|
||||
}
|
||||
}
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
import type { MouseButton } from "@/transport/types";
|
||||
import type { Point } from "./geometry";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
interface MouseClickDispatchOptions {
|
||||
button: MouseButton;
|
||||
clickCount: number;
|
||||
modifiers: number;
|
||||
signal?: AbortSignal;
|
||||
moveSettleMs?: number;
|
||||
}
|
||||
|
||||
type MouseClickDispatchResult = "completed" | "cancelled";
|
||||
|
||||
function waitForSettle(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
|
||||
if (ms <= 0) return Promise.resolve(signal?.aborted !== true);
|
||||
if (signal?.aborted) return Promise.resolve(false);
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (completed: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve(completed);
|
||||
};
|
||||
const timer = setTimeout(() => finish(true), ms);
|
||||
const onAbort = () => finish(false);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch one coherent CDP mouse click in top-viewport CSS coordinates.
|
||||
*
|
||||
* Keeping the sequence here prevents DOM and Surface interaction paths from
|
||||
* duplicating cancellation and best-effort release behavior. Callers that
|
||||
* target asynchronously hit-tested content may request a settle interval.
|
||||
*/
|
||||
export async function dispatchMouseClick(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
point: Point,
|
||||
options: MouseClickDispatchOptions,
|
||||
): Promise<MouseClickDispatchResult> {
|
||||
const { button, clickCount, modifiers, signal, moveSettleMs = 0 } = options;
|
||||
let pressed = false;
|
||||
const release = () =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
modifiers,
|
||||
});
|
||||
if (!(await waitForSettle(moveSettleMs, signal))) return "cancelled";
|
||||
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
pressed = true;
|
||||
|
||||
if (signal?.aborted) {
|
||||
await release();
|
||||
pressed = false;
|
||||
return "cancelled";
|
||||
}
|
||||
|
||||
await release();
|
||||
pressed = false;
|
||||
return "completed";
|
||||
} catch (error) {
|
||||
if (pressed) {
|
||||
try {
|
||||
await release();
|
||||
} catch (releaseError) {
|
||||
console.debug("[bsk mouse-input] best-effort mouse release failed", releaseError);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
type Rect,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
@@ -48,8 +47,6 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { surfaceVisibleRect } from "./surface-coordinate";
|
||||
import { captureSurfaceEnvironment } from "./surface-target";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
@@ -62,11 +59,6 @@ import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "./vom/rendered-surfaces";
|
||||
import {
|
||||
buildSemanticGraph,
|
||||
buildSemanticVomScene,
|
||||
@@ -98,9 +90,6 @@ export const normaliseRef = sharedNormaliseRef;
|
||||
// screenshot — `tool.screenshot`
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
|
||||
|
||||
/**
|
||||
* Strip the `data:image/...;base64,` prefix from a Chrome
|
||||
* `captureVisibleTab` dataURL and return the raw base64 payload.
|
||||
@@ -193,42 +182,18 @@ async function captureElementScreenshot(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
visualSurface = false,
|
||||
observedVisibleRect?: Rect,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
{ image_base64: string; width: number; height: number; topViewportRect: Rect } | RpcError
|
||||
> {
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: !visualSurface },
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const topViewportRect = surfaceVisibleRect(
|
||||
geometry.topBounds,
|
||||
visualSurface ? observedVisibleRect : undefined,
|
||||
);
|
||||
if (!topViewportRect) {
|
||||
return { code: "permission_denied", message: "surface is no longer visible" };
|
||||
}
|
||||
const rect = {
|
||||
x: topViewportRect.x,
|
||||
y: topViewportRect.y,
|
||||
width: topViewportRect.w,
|
||||
height: topViewportRect.h,
|
||||
};
|
||||
const scale = visualSurface
|
||||
? Math.min(
|
||||
1,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
|
||||
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
|
||||
)
|
||||
: 1;
|
||||
const rect = geometry.topBounds;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
@@ -238,7 +203,7 @@ async function captureElementScreenshot(
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
@@ -250,7 +215,7 @@ async function captureElementScreenshot(
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height, topViewportRect };
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -345,7 +310,7 @@ export async function handleScreenshot(
|
||||
if (!deps.cdp) {
|
||||
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -365,54 +330,18 @@ export async function handleScreenshot(
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
node.kind === "surface",
|
||||
node.visibleRect,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const capture =
|
||||
node.kind === "surface"
|
||||
? await (async () => {
|
||||
const environment = await captureSurfaceEnvironment(cdp, target.tabId, node.frameId);
|
||||
if (isRpcError(environment)) return environment;
|
||||
return ctx.surfaceCaptures.create({
|
||||
sessionId: ctx.sessionId,
|
||||
tabId: target.tabId,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: node.refKey,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
backendNodeId: node.backendNodeId,
|
||||
observationGeneration: node.generation,
|
||||
},
|
||||
topViewportRect: captured.topViewportRect,
|
||||
imageWidth: captured.width,
|
||||
imageHeight: captured.height,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
})()
|
||||
: null;
|
||||
if (capture && isRpcError(capture)) return capture;
|
||||
return withShotDialogs({
|
||||
image_base64: captured.image_base64,
|
||||
width: captured.width,
|
||||
height: captured.height,
|
||||
format: "png",
|
||||
tab_id: target.tabId,
|
||||
...(capture
|
||||
? {
|
||||
capture: {
|
||||
id: capture.id,
|
||||
surface_ref: `@${capture.surface.ref}`,
|
||||
coordinate_space: "capture-image-pixel" as const,
|
||||
expires_at: capture.expiresAt,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1094,7 +1023,6 @@ async function captureForVom(
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
renderedSurfaceDiscovery?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
@@ -1136,20 +1064,7 @@ export async function captureVomObservation(
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const observedScene = options.renderedSurfaceDiscovery
|
||||
? projectRenderedSurfaces(
|
||||
decoratedScene,
|
||||
normalizedDocuments,
|
||||
clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
normalizedDocuments,
|
||||
captured.viewport,
|
||||
captured.excludedBackendNodeIds,
|
||||
),
|
||||
),
|
||||
)
|
||||
: decoratedScene;
|
||||
const rendered = renderVom(observedScene, {
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
@@ -1200,7 +1115,6 @@ async function handleVomObservation(
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
renderedSurfaceDiscovery: toolName === "observe",
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
@@ -1218,9 +1132,6 @@ async function handleVomObservation(
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
|
||||
...(ref.kind ? { kind: ref.kind } : {}),
|
||||
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
|
||||
@@ -243,7 +243,6 @@ export async function handleSessionStop(
|
||||
|
||||
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
||||
ctx.refStore.clear();
|
||||
ctx.surfaceCaptures.clear();
|
||||
|
||||
clearRecordingForSession(params.session_id);
|
||||
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
|
||||
@@ -13,10 +12,6 @@ export interface SnapshotRefLookup {
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
@@ -42,10 +37,6 @@ export function lookupSnapshotRef(
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
|
||||
kind: entry.kind,
|
||||
capabilities: entry.capabilities,
|
||||
generation: entry.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -58,7 +49,6 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
requiredCapability?: RefCapability,
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
@@ -68,13 +58,5 @@ export function resolveSnapshotRef(
|
||||
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
|
||||
);
|
||||
}
|
||||
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"ref_capability_denied",
|
||||
`ref ${ref} does not support ${requiredCapability}`,
|
||||
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
|
||||
);
|
||||
}
|
||||
return looked;
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { Point, Region, ViewportRect } from "./geometry";
|
||||
|
||||
const RECT_EPSILON = 0.01;
|
||||
|
||||
export function surfaceVisibleRect(
|
||||
liveRect: ViewportRect,
|
||||
observedRect: Rect | undefined,
|
||||
): Rect | null {
|
||||
const observed = observedRect ?? {
|
||||
x: liveRect.x,
|
||||
y: liveRect.y,
|
||||
w: liveRect.width,
|
||||
h: liveRect.height,
|
||||
};
|
||||
const x = Math.max(liveRect.x, observed.x);
|
||||
const y = Math.max(liveRect.y, observed.y);
|
||||
const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w);
|
||||
const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h);
|
||||
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
|
||||
}
|
||||
|
||||
export function sameRect(a: Rect, b: Rect): boolean {
|
||||
return (
|
||||
Math.abs(a.x - b.x) <= RECT_EPSILON &&
|
||||
Math.abs(a.y - b.y) <= RECT_EPSILON &&
|
||||
Math.abs(a.w - b.w) <= RECT_EPSILON &&
|
||||
Math.abs(a.h - b.h) <= RECT_EPSILON
|
||||
);
|
||||
}
|
||||
|
||||
export function mapImagePointToViewport(
|
||||
rect: Rect,
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
imageX: number,
|
||||
imageY: number,
|
||||
): Point | null {
|
||||
if (
|
||||
!Number.isFinite(imageX) ||
|
||||
!Number.isFinite(imageY) ||
|
||||
!Number.isSafeInteger(imageWidth) ||
|
||||
!Number.isSafeInteger(imageHeight) ||
|
||||
imageWidth <= 0 ||
|
||||
imageHeight <= 0 ||
|
||||
imageX < 0 ||
|
||||
imageY < 0 ||
|
||||
imageX >= imageWidth ||
|
||||
imageY >= imageHeight ||
|
||||
!Number.isFinite(rect.x) ||
|
||||
!Number.isFinite(rect.y) ||
|
||||
!Number.isFinite(rect.w) ||
|
||||
!Number.isFinite(rect.h) ||
|
||||
rect.w <= 0 ||
|
||||
rect.h <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
x: rect.x + (imageX / imageWidth) * rect.w,
|
||||
y: rect.y + (imageY / imageHeight) * rect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function pointInPolygon(point: Point, polygon: Point[]): boolean {
|
||||
for (let index = 0; index < polygon.length; index += 1) {
|
||||
const a = polygon[index];
|
||||
const b = polygon[(index + 1) % polygon.length];
|
||||
const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x);
|
||||
const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9;
|
||||
const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9;
|
||||
if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true;
|
||||
}
|
||||
let inside = false;
|
||||
for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) {
|
||||
const a = polygon[index];
|
||||
const b = polygon[previous];
|
||||
const intersects =
|
||||
a.y > point.y !== b.y > point.y &&
|
||||
point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x;
|
||||
if (intersects) inside = !inside;
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
export function pointInRegion(point: Point, region: Region): boolean {
|
||||
return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon));
|
||||
}
|
||||
@@ -1,260 +0,0 @@
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { type CdpRunner, isRpcError, normaliseRef } from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
mapImagePointToViewport,
|
||||
pointInRegion,
|
||||
sameRect,
|
||||
surfaceVisibleRect,
|
||||
} from "./surface-coordinate";
|
||||
|
||||
export interface SurfaceCaptureEnvironment {
|
||||
navigationIdentity: string;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
}
|
||||
|
||||
export interface SurfaceTargetResolverDeps {
|
||||
cdp: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface SurfacePointerTargetInput {
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
captureId?: string;
|
||||
imageX?: number;
|
||||
imageY?: number;
|
||||
}
|
||||
|
||||
export interface ResolvedSurfacePointerTarget {
|
||||
point: { x: number; y: number };
|
||||
usedRef: string;
|
||||
moveSettleMs: number;
|
||||
capture: {
|
||||
id: string;
|
||||
imageX: number;
|
||||
imageY: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Canvas-style controls commonly update their pointer hit-test on the next
|
||||
// animation frame. Leave a scheduling boundary after moving and before pressing.
|
||||
export const SURFACE_POINTER_MOVE_SETTLE_MS = 32;
|
||||
|
||||
function stale(message: string): RpcError {
|
||||
return rpcError("permission_denied", "surface_capture_stale", message);
|
||||
}
|
||||
|
||||
function finiteObject(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(finiteObject);
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value)
|
||||
.filter(
|
||||
([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)),
|
||||
)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, entry]) => [key, finiteObject(entry)]),
|
||||
);
|
||||
}
|
||||
|
||||
function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null {
|
||||
if (!frameId) return "top";
|
||||
const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
const path: Array<{
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
targetSessionId?: string;
|
||||
}> = [];
|
||||
const seen = new Set<string>();
|
||||
let current = frames.get(frameId);
|
||||
while (current) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push({
|
||||
frameId: current.frameId,
|
||||
...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}),
|
||||
...(current.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: current.ownerBackendNodeId }
|
||||
: {}),
|
||||
...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}),
|
||||
});
|
||||
if (!current.parentFrameId) return JSON.stringify(path);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function captureSurfaceEnvironment(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameId: string | undefined,
|
||||
): Promise<SurfaceCaptureEnvironment | RpcError> {
|
||||
try {
|
||||
const [tree, metrics, graph] = await Promise.all([
|
||||
cdp.send<{
|
||||
frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } };
|
||||
}>(tabId, "Page.getFrameTree", {}),
|
||||
cdp.send<Record<string, unknown>>(tabId, "Page.getLayoutMetrics", {}),
|
||||
frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null),
|
||||
]);
|
||||
const root = tree.frameTree?.frame;
|
||||
if (!root?.id || !root.loaderId) {
|
||||
return { code: "cdp_failed", message: "could not identify the current navigation" };
|
||||
}
|
||||
const frameProjectionSignature = frameId
|
||||
? graph
|
||||
? framePathSignature(graph, frameId)
|
||||
: null
|
||||
: "top";
|
||||
if (!frameProjectionSignature) {
|
||||
return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` };
|
||||
}
|
||||
return {
|
||||
navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }),
|
||||
viewportSignature: JSON.stringify(
|
||||
finiteObject({
|
||||
cssLayoutViewport: metrics.cssLayoutViewport,
|
||||
cssVisualViewport: metrics.cssVisualViewport,
|
||||
}),
|
||||
),
|
||||
frameProjectionSignature,
|
||||
};
|
||||
} catch (error) {
|
||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function validatePointParams(input: SurfacePointerTargetInput): RpcError | null {
|
||||
if (!input.captureId || typeof input.captureId !== "string") {
|
||||
return { code: "invalid_params", message: "surface point click requires capture_id" };
|
||||
}
|
||||
if (typeof input.imageX !== "number" || !Number.isFinite(input.imageX)) {
|
||||
return { code: "invalid_params", message: "image_x must be a finite number" };
|
||||
}
|
||||
if (typeof input.imageY !== "number" || !Number.isFinite(input.imageY)) {
|
||||
return { code: "invalid_params", message: "image_y must be a finite number" };
|
||||
}
|
||||
if (!input.ref || input.selector) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "surface point click requires one Surface ref and does not accept a selector",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError {
|
||||
const code = reason === "not_found" ? "not_found" : "permission_denied";
|
||||
return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`);
|
||||
}
|
||||
|
||||
function aborted(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted
|
||||
? { code: "cancelled", message: "surface target resolution aborted" }
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Resolve one fresh Surface capture coordinate into a validated top-viewport pointer target. */
|
||||
export async function resolveSurfacePointerTarget(
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number; url?: string },
|
||||
input: SurfacePointerTargetInput,
|
||||
deps: SurfaceTargetResolverDeps,
|
||||
): Promise<ResolvedSurfacePointerTarget | RpcError> {
|
||||
const invalid = validatePointParams(input);
|
||||
if (invalid) return invalid;
|
||||
const earlyAbort = aborted(deps.signal);
|
||||
if (earlyAbort) return earlyAbort;
|
||||
|
||||
const consumed = ctx.surfaceCaptures.consume(input.captureId as string);
|
||||
if (!consumed.ok) return captureConsumeError(consumed.reason);
|
||||
const capture = consumed.capture;
|
||||
const postConsumeAbort = aborted(deps.signal);
|
||||
if (postConsumeAbort) return postConsumeAbort;
|
||||
|
||||
if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) {
|
||||
return stale("surface capture belongs to a different session or tab");
|
||||
}
|
||||
if (normaliseRef(input.ref as string) !== capture.surface.ref) {
|
||||
return stale("surface ref does not match the capture");
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, input.ref as string, target.tabId, "screenshot");
|
||||
if (isRpcError(node)) return stale(node.message);
|
||||
if (
|
||||
node.kind !== "surface" ||
|
||||
node.backendNodeId !== capture.surface.backendNodeId ||
|
||||
node.frameId !== capture.surface.frameId ||
|
||||
node.generation !== capture.surface.observationGeneration
|
||||
) {
|
||||
return stale("surface observation generation or node identity changed");
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
try {
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
} catch (error) {
|
||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId);
|
||||
if (isRpcError(environment)) return environment;
|
||||
if (
|
||||
environment.navigationIdentity !== capture.navigationIdentity ||
|
||||
environment.viewportSignature !== capture.viewportSignature ||
|
||||
environment.frameProjectionSignature !== capture.frameProjectionSignature
|
||||
) {
|
||||
return stale("navigation, viewport, zoom, scroll, or frame projection changed");
|
||||
}
|
||||
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
},
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: false },
|
||||
);
|
||||
if (isRpcError(geometry)) return stale(geometry.message);
|
||||
const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect);
|
||||
if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) {
|
||||
return stale("surface visible region changed");
|
||||
}
|
||||
const point = mapImagePointToViewport(
|
||||
capture.topViewportRect,
|
||||
capture.imageWidth,
|
||||
capture.imageHeight,
|
||||
input.imageX as number,
|
||||
input.imageY as number,
|
||||
);
|
||||
if (!point || !pointInRegion(point, geometry.topVisibleRegions)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"surface_coordinate_invalid",
|
||||
"image coordinate is outside the captured Surface",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
point,
|
||||
usedRef: capture.surface.ref,
|
||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
capture: {
|
||||
id: capture.id,
|
||||
imageX: input.imageX as number,
|
||||
imageY: input.imageY as number,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -722,44 +722,6 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||
});
|
||||
|
||||
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
|
||||
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
|
||||
const i = (value: string) => S.indexOf(value);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 100],
|
||||
[10, 10, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1, 2],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const canvas = nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(canvas?.rendered).toBe(true);
|
||||
expect(canvas?.visuallySuppressed).toBe(true);
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
import type { VomScene } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "../rendered-surfaces";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
|
||||
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "canvas",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
localRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
rendered: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function document(
|
||||
domNodes: CapturedNode[],
|
||||
axNodes: SemanticAxNode[] = [],
|
||||
): CapturedFrameDocument<SemanticAxNode> {
|
||||
return {
|
||||
frameId: domNodes[0]?.frameId ?? "main",
|
||||
contextScopeId: domNodes[0]?.frameId ?? "main",
|
||||
target: { tabId: 4 },
|
||||
domNodes,
|
||||
axNodes,
|
||||
};
|
||||
}
|
||||
|
||||
describe("rendered surface discovery", () => {
|
||||
it("finds prominent rendered canvases and excludes hidden descendants", () => {
|
||||
const visible = domNode(2, { parentBackendNodeId: 1 });
|
||||
const hidden = domNode(3, { parentBackendNodeId: 4 });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
visible,
|
||||
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
|
||||
hidden,
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
|
||||
expect(surfaces).toHaveLength(1);
|
||||
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
|
||||
});
|
||||
|
||||
it("keeps tiny canvases as individually addressable exact discoveries", () => {
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("keeps a surface when the canvas only has partial native accessibility content", () => {
|
||||
const axNodes = [
|
||||
{
|
||||
nodeId: "canvas",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "presentation" },
|
||||
childIds: ["button"],
|
||||
frameId: "main",
|
||||
},
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
frameId: "main",
|
||||
},
|
||||
] as SemanticAxNode[];
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
|
||||
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
|
||||
const canvas = domNode(2, {
|
||||
attrs: {
|
||||
id: "must-not-be-a-label",
|
||||
class: "must-not-be-a-label",
|
||||
"data-testid": "must-not-be-a-label",
|
||||
},
|
||||
});
|
||||
|
||||
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
|
||||
expect.not.objectContaining({
|
||||
label: expect.anything(),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats blank explicit Canvas names as unnamed", () => {
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
|
||||
});
|
||||
|
||||
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
|
||||
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([owner, canvas])],
|
||||
{ width: 1280, height: 720 },
|
||||
new Set([1]),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("excludes canvases suppressed by a fully transparent ancestor", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 900, h: 600 },
|
||||
visuallySuppressed: true,
|
||||
});
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 10, y: 20, w: 100, h: 70 },
|
||||
overflowX: "hidden",
|
||||
overflowY: "hidden",
|
||||
});
|
||||
const partial = domNode(2, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 55, y: 30, w: 140, h: 60 },
|
||||
});
|
||||
const fullyClipped = domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 140, y: 30, w: 80, h: 50 },
|
||||
});
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
|
||||
width: 1280,
|
||||
height: 720,
|
||||
}),
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 2,
|
||||
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rendered surface clustering and projection", () => {
|
||||
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
|
||||
const body = domNode(1, { tag: "body", rect: null, localRect: null });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
body,
|
||||
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
|
||||
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
const groups = clusterRenderedSurfaces(surfaces);
|
||||
const scene: VomScene = {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [
|
||||
{
|
||||
id: 10,
|
||||
parentId: null,
|
||||
backendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
role: "RootWebArea",
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]?.members).toHaveLength(2);
|
||||
expect(groups[0]?.label).toBeUndefined();
|
||||
expect(
|
||||
projectRenderedSurfaces(
|
||||
scene,
|
||||
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
|
||||
groups,
|
||||
),
|
||||
).toMatchObject({
|
||||
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge a contained canvas with a larger independent canvas", () => {
|
||||
const groups = clusterRenderedSurfaces([
|
||||
...discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
domNode(2, { parentBackendNodeId: 1 }),
|
||||
domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 30, y: 40, w: 200, h: 100 },
|
||||
}),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
]);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not merge coincident canvases owned by different containers", () => {
|
||||
const groups = clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(2, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(3, { parentBackendNodeId: 1 }),
|
||||
domNode(4, { parentBackendNodeId: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -11,15 +11,7 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
] as const;
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
@@ -47,10 +39,6 @@ export interface CapturedNode {
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
overflowX?: string;
|
||||
overflowY?: string;
|
||||
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
|
||||
visuallySuppressed?: boolean;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
@@ -808,8 +796,6 @@ function parseDocumentNodes(
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const overflowXCol = STYLE_COL["overflow-x"];
|
||||
const overflowYCol = STYLE_COL["overflow-y"];
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
@@ -835,7 +821,6 @@ function parseDocumentNodes(
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
@@ -855,11 +840,7 @@ function parseDocumentNodes(
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let overflowX = "visible";
|
||||
let overflowY = "visible";
|
||||
let rendered = false;
|
||||
let visuallySuppressed =
|
||||
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
@@ -882,19 +863,12 @@ function parseDocumentNodes(
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
|
||||
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
|
||||
visuallySuppressed ||=
|
||||
visibility === "hidden" ||
|
||||
visibility === "collapse" ||
|
||||
(Number.parseFloat(opacity) || 0) <= 0;
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
@@ -929,9 +903,6 @@ function parseDocumentNodes(
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
overflowX,
|
||||
overflowY,
|
||||
visuallySuppressed,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
|
||||
const STACK_IOU_RATIO = 0.9;
|
||||
const STACK_SIZE_RATIO = 0.9;
|
||||
|
||||
function area(rect: Rect): number {
|
||||
return Math.max(0, rect.w) * Math.max(0, rect.h);
|
||||
}
|
||||
|
||||
function intersectionArea(a: Rect, b: Rect): number {
|
||||
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
|
||||
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
|
||||
return width * height;
|
||||
}
|
||||
|
||||
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
||||
if (a.frameId !== b.frameId) return false;
|
||||
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
|
||||
const aArea = area(a.visibleRect);
|
||||
const bArea = area(b.visibleRect);
|
||||
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
|
||||
const union = aArea + bArea - intersection;
|
||||
const widthRatio =
|
||||
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
|
||||
const heightRatio =
|
||||
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
|
||||
return (
|
||||
union > 0 &&
|
||||
intersection / union >= STACK_IOU_RATIO &&
|
||||
widthRatio >= STACK_SIZE_RATIO &&
|
||||
heightRatio >= STACK_SIZE_RATIO
|
||||
);
|
||||
}
|
||||
|
||||
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
|
||||
const representative = [...members].sort(
|
||||
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
|
||||
)[0];
|
||||
return {
|
||||
frameId: representative.frameId,
|
||||
members,
|
||||
representative,
|
||||
label: members.map((surface) => surface.label).find((label) => label !== undefined),
|
||||
};
|
||||
}
|
||||
|
||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
|
||||
const groups: RenderedSurface[][] = [];
|
||||
for (const surface of surfaces) {
|
||||
const matching = groups.find((group) =>
|
||||
group.some((member) => sameVisualStack(member, surface)),
|
||||
);
|
||||
if (matching) matching.push(surface);
|
||||
else groups.push([surface]);
|
||||
}
|
||||
return groups
|
||||
.map(toGroup)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
||||
);
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurface } from "./types";
|
||||
|
||||
function clean(value: string | undefined): string | undefined {
|
||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
|
||||
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
function intersectRect(a: Rect, b: Rect): Rect | null {
|
||||
const x = Math.max(a.x, b.x);
|
||||
const y = Math.max(a.y, b.y);
|
||||
const right = Math.min(a.x + a.w, b.x + b.w);
|
||||
const bottom = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, w: right - x, h: bottom - y };
|
||||
}
|
||||
|
||||
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
|
||||
|
||||
function clipToOverflowAncestors(
|
||||
node: CapturedNode,
|
||||
rect: Rect,
|
||||
nodesByBackend: ReadonlyMap<number, CapturedNode>,
|
||||
): Rect | null {
|
||||
let clipped: Rect | null = rect;
|
||||
let parentBackendNodeId = node.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
|
||||
const parent = nodesByBackend.get(parentBackendNodeId);
|
||||
if (!parent) break;
|
||||
if (parent.rect) {
|
||||
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
|
||||
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
|
||||
if (clipX || clipY) {
|
||||
const clipRect = {
|
||||
x: clipX ? parent.rect.x : clipped.x,
|
||||
y: clipY ? parent.rect.y : clipped.y,
|
||||
w: clipX ? parent.rect.w : clipped.w,
|
||||
h: clipY ? parent.rect.h : clipped.h,
|
||||
};
|
||||
clipped = intersectRect(clipped, clipRect);
|
||||
}
|
||||
}
|
||||
parentBackendNodeId = parent.parentBackendNodeId;
|
||||
guard += 1;
|
||||
}
|
||||
return clipped;
|
||||
}
|
||||
|
||||
function excludedByDomPolicy(
|
||||
node: CapturedNode,
|
||||
nodesByBackend: Map<number, CapturedNode>,
|
||||
excludedBackendNodeIds: ReadonlySet<number>,
|
||||
): boolean {
|
||||
let current: CapturedNode | undefined = node;
|
||||
let guard = 0;
|
||||
while (current && guard <= nodesByBackend.size) {
|
||||
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
|
||||
if (current.visuallySuppressed === true) return true;
|
||||
const attrs = current.attrs;
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
||||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
||||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
current =
|
||||
current.parentBackendNodeId === null
|
||||
? undefined
|
||||
: nodesByBackend.get(current.parentBackendNodeId);
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function discoverRenderedSurfaces(
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
viewport: Viewport,
|
||||
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
|
||||
): RenderedSurface[] {
|
||||
const surfaces: RenderedSurface[] = [];
|
||||
for (const document of documents) {
|
||||
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
|
||||
for (const node of document.domNodes) {
|
||||
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
|
||||
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
|
||||
const viewportRect = intersectViewport(node.rect, viewport);
|
||||
const visibleRect = viewportRect
|
||||
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
|
||||
: null;
|
||||
if (!visibleRect) continue;
|
||||
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
|
||||
surfaces.push({
|
||||
renderingKind: "canvas",
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
parentBackendNodeId: node.parentBackendNodeId,
|
||||
visibleRect,
|
||||
paintOrder: node.paintOrder,
|
||||
...(label ? { label } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return surfaces;
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
export { clusterRenderedSurfaces } from "./cluster";
|
||||
export { discoverRenderedSurfaces } from "./discovery";
|
||||
export { projectRenderedSurfaces } from "./project";
|
||||
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
@@ -1,100 +0,0 @@
|
||||
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurfaceGroup } from "./types";
|
||||
|
||||
function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
interface ProjectionIndex {
|
||||
sceneByBackend: ReadonlyMap<string, number>;
|
||||
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
|
||||
domByFrame: ReadonlyMap<
|
||||
string,
|
||||
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
|
||||
>;
|
||||
rootSceneIdByFrame: ReadonlyMap<string, number>;
|
||||
}
|
||||
|
||||
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
|
||||
const document = index.documentByFrame.get(group.frameId);
|
||||
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
|
||||
let parentBackendNodeId = group.representative.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
|
||||
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
|
||||
if (sceneId !== undefined) return sceneId;
|
||||
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
|
||||
guard += 1;
|
||||
}
|
||||
|
||||
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
|
||||
const owner = index.sceneByBackend.get(
|
||||
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
|
||||
);
|
||||
if (owner !== undefined) return owner;
|
||||
}
|
||||
|
||||
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
|
||||
}
|
||||
|
||||
function buildProjectionIndex(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
): ProjectionIndex {
|
||||
const rootSceneIdByFrame = new Map<string, number>();
|
||||
for (const node of scene.nodes) {
|
||||
if (
|
||||
node.frameId &&
|
||||
!rootSceneIdByFrame.has(node.frameId) &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
|
||||
) {
|
||||
rootSceneIdByFrame.set(node.frameId, node.id);
|
||||
}
|
||||
}
|
||||
return {
|
||||
sceneByBackend: new Map(
|
||||
scene.nodes.flatMap((node) =>
|
||||
node.backendNodeId === undefined
|
||||
? []
|
||||
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
|
||||
),
|
||||
),
|
||||
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
|
||||
domByFrame: new Map(
|
||||
documents.map((document) => [
|
||||
document.frameId,
|
||||
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
|
||||
]),
|
||||
),
|
||||
rootSceneIdByFrame,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectRenderedSurfaces(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
groups: RenderedSurfaceGroup[],
|
||||
): VomScene {
|
||||
const index = buildProjectionIndex(scene, documents);
|
||||
const visualSurfaces: VomVisualSurface[] = [];
|
||||
|
||||
for (const group of groups) {
|
||||
const parentId = nearestSceneParent(index, group);
|
||||
visualSurfaces.push({
|
||||
parentId,
|
||||
backendNodeId: group.representative.backendNodeId,
|
||||
frameId: group.frameId,
|
||||
renderingKind: "canvas",
|
||||
visibleRect: group.representative.visibleRect,
|
||||
...(group.label ? { label: group.label } : {}),
|
||||
memberCount: group.members.length,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...scene,
|
||||
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
|
||||
};
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface RenderedSurface {
|
||||
renderingKind: "canvas";
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
visibleRect: Rect;
|
||||
paintOrder: number;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface RenderedSurfaceGroup {
|
||||
frameId: string;
|
||||
members: RenderedSurface[];
|
||||
representative: RenderedSurface;
|
||||
label?: string;
|
||||
}
|
||||
@@ -23,12 +23,9 @@ export type ErrorCode =
|
||||
export type RpcErrorReason =
|
||||
| "agent_window_scope"
|
||||
| "element_not_visible"
|
||||
| "ref_capability_denied"
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "focus_target_unresolved"
|
||||
| "input_not_applied"
|
||||
| "target_not_select"
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
@@ -36,11 +33,6 @@ export type RpcErrorReason =
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "surface_capture_not_found"
|
||||
| "surface_capture_expired"
|
||||
| "surface_capture_consumed"
|
||||
| "surface_capture_stale"
|
||||
| "surface_coordinate_invalid"
|
||||
| "cleanup_failed";
|
||||
|
||||
export interface RpcErrorData {
|
||||
@@ -306,17 +298,9 @@ export interface ScreenshotResult {
|
||||
height: number;
|
||||
format: string;
|
||||
tab_id: number;
|
||||
capture?: SurfaceCaptureInfo;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface SurfaceCaptureInfo {
|
||||
id: string;
|
||||
surface_ref: string;
|
||||
coordinate_space: "capture-image-pixel";
|
||||
expires_at: number;
|
||||
}
|
||||
|
||||
export interface SnapshotParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
@@ -430,9 +414,6 @@ export interface ClickParams {
|
||||
click_count?: number;
|
||||
modifiers?: KeyModifier[];
|
||||
timeout_ms?: number;
|
||||
capture_id?: string;
|
||||
image_x?: number;
|
||||
image_y?: number;
|
||||
}
|
||||
|
||||
export interface ClickResult {
|
||||
@@ -441,9 +422,6 @@ export interface ClickResult {
|
||||
used_selector?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
capture_id?: string;
|
||||
image_x?: number;
|
||||
image_y?: number;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
# Rendered Surface browser fixtures
|
||||
|
||||
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
|
||||
|
||||
## Serve
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
python3 -m http.server 4173 --bind 127.0.0.1 \
|
||||
--directory apps/extension/test-fixtures/rendered-surfaces
|
||||
```
|
||||
|
||||
Use an Agent Window large enough to keep the primary matrix visible:
|
||||
|
||||
```bash
|
||||
bsk session start --width 1440 --height 1100
|
||||
bsk navigate http://127.0.0.1:4173/index.html --session <id>
|
||||
bsk observe --session <id>
|
||||
```
|
||||
|
||||
Always stop the session after the run.
|
||||
|
||||
## Pages
|
||||
|
||||
### `index.html`
|
||||
|
||||
Expected individual Surface labels:
|
||||
|
||||
- `fixture-basic-large`
|
||||
- `fixture-tiny-8px`
|
||||
- `fixture-pointer-events-none`
|
||||
- `fixture-partial-ax`
|
||||
- `fixture-contained-outer`
|
||||
- `fixture-contained-inner`
|
||||
- `fixture-different-parent-a`
|
||||
- `fixture-different-parent-b`
|
||||
|
||||
Expected one grouped Surface:
|
||||
|
||||
- `fixture-layered-group`, with `layers=2`
|
||||
|
||||
Expected absent:
|
||||
|
||||
- `fixture-display-none`
|
||||
- `fixture-visibility-hidden`
|
||||
- `fixture-opacity-zero`
|
||||
- `fixture-parent-opacity-zero`
|
||||
- `fixture-hidden-attribute`
|
||||
- `fixture-inert-parent`
|
||||
- `fixture-aria-hidden-parent`
|
||||
- `fixture-zero-size`
|
||||
- `fixture-offscreen-right`
|
||||
- `fixture-fully-overflow-clipped`
|
||||
|
||||
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
|
||||
|
||||
### `unlabeled.html`
|
||||
|
||||
This page contains eight Canvas elements but must render seven generic Surface markers because
|
||||
the two coincident layers form one group.
|
||||
|
||||
- Every marker uses the stable fallback label `canvas visual surface`.
|
||||
- Exactly one marker has `layers=2`.
|
||||
- The two contained canvases remain separate markers.
|
||||
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
|
||||
- Blank `aria-label` and `title` values behave as no label.
|
||||
- Fallback AX content does not suppress the corresponding Surface.
|
||||
|
||||
### `frames.html`
|
||||
|
||||
Expected Surface labels:
|
||||
|
||||
- `fixture-same-origin-frame`
|
||||
- `fixture-cross-origin-frame`
|
||||
- `fixture-nested-frame`
|
||||
|
||||
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
|
||||
|
||||
Each frame Canvas also exposes a `pointer status` live region. A successful
|
||||
screenshot-bound point click must report
|
||||
`down button=0 buttons=1 hoverReady=true; click received` and turn the Canvas
|
||||
yellow. The hover flag is set on the animation frame after `pointermove`, so the
|
||||
fixture verifies both child-document delivery and the scheduling boundary
|
||||
required before `pointerdown`, rather than merely validating projected
|
||||
coordinates or a successful CDP response.
|
||||
|
||||
### `viewport.html`
|
||||
|
||||
Run with a `900x700` Agent Window.
|
||||
|
||||
- `fixture-partial-viewport` is partly visible and must be represented.
|
||||
- `fixture-below-fold` is absent initially.
|
||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
||||
|
||||
### `input-frames.html`
|
||||
|
||||
This page exercises generic frame-aware input routing. It contains a top-level
|
||||
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
||||
frame includes an input, textarea, contenteditable editor, and a controlled
|
||||
input that resets its value on the next animation frame.
|
||||
|
||||
- `fill` must route focus, text insertion, DOM events, and readback through the
|
||||
ref's owning CDP target.
|
||||
- `press --ref` must focus and dispatch every key event through that same target.
|
||||
- Untargeted `press` must follow the actual focused target or refuse when focus
|
||||
ownership cannot be determined uniquely.
|
||||
- The rejecting input must return `input_not_applied`, never a successful
|
||||
requested-value length.
|
||||
|
||||
### `canvas-input.html`
|
||||
|
||||
This is the PR3 end-to-end fixture. Its cross-origin frame contains a
|
||||
visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
|
||||
focuses a standard textbox; ordinary `fill` must then write and read back its
|
||||
real value through the textbox's owning target. The fixture deliberately does
|
||||
not provide a Canvas-specific fill action.
|
||||
|
||||
### `dynamic.html`
|
||||
|
||||
Use the normal DOM buttons and re-observe after each action:
|
||||
|
||||
- Initial state: `fixture-dynamic` absent.
|
||||
- After **Insert canvas**: present.
|
||||
- After **Resize to zero**: absent.
|
||||
- After **Restore size**: present with a fresh ref.
|
||||
- After **Remove canvas**: absent.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `snapshot` must not emit any of these Surface markers.
|
||||
- `observe` must emit only currently visible Canvas surfaces.
|
||||
- Existing DOM/AX controls remain readable alongside Surface markers.
|
||||
- Missing labels never prevent discovery; naming and discovery are independent.
|
||||
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
|
||||
- No fixture URL, id, label, or layout constant may appear in production discovery code.
|
||||
@@ -1,20 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Canvas to standard input fixture</title>
|
||||
<style>
|
||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||
iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Canvas to standard input fixture</h1>
|
||||
<p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
|
||||
<iframe
|
||||
title="cross-origin Canvas editor"
|
||||
src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
|
||||
></iframe>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,46 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface dynamic matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
|
||||
button { padding: 8px 12px; }
|
||||
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Dynamic Canvas lifecycle</h1>
|
||||
<div class="controls">
|
||||
<button id="insert" type="button">Insert canvas</button>
|
||||
<button id="zero" type="button">Resize to zero</button>
|
||||
<button id="restore" type="button">Restore size</button>
|
||||
<button id="remove" type="button">Remove canvas</button>
|
||||
</div>
|
||||
<div id="mount"></div>
|
||||
<script>
|
||||
const mount = document.querySelector("#mount");
|
||||
const getCanvas = () => mount.querySelector("canvas");
|
||||
document.querySelector("#insert").addEventListener("click", () => {
|
||||
if (getCanvas()) return;
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 260;
|
||||
canvas.height = 120;
|
||||
canvas.setAttribute("aria-label", "fixture-dynamic");
|
||||
mount.append(canvas);
|
||||
});
|
||||
document.querySelector("#zero").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
|
||||
});
|
||||
document.querySelector("#restore").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.removeAttribute("style");
|
||||
});
|
||||
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,80 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 12px; background: #fdfefe; }
|
||||
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
|
||||
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
|
||||
#pointer-status { margin: 8px 0 0; min-height: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="frame-canvas" width="220" height="90"></canvas>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const canvas = document.querySelector("#frame-canvas");
|
||||
if (params.get("canvas") === "0") canvas.remove();
|
||||
else {
|
||||
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
||||
if (params.get("editor") === "1") {
|
||||
const context = canvas.getContext("2d");
|
||||
context.font = "14px system-ui, sans-serif";
|
||||
context.textBaseline = "middle";
|
||||
context.strokeStyle = "#1e8449";
|
||||
context.fillStyle = "#145a32";
|
||||
for (let row = 0; row < 3; row += 1) {
|
||||
const top = row * 30;
|
||||
context.strokeRect(0.5, top + 0.5, 219, 29);
|
||||
context.fillText(`Row ${row + 1}`, 12, top + 15);
|
||||
}
|
||||
}
|
||||
if (params.get("interactive") === "1") {
|
||||
const status = document.createElement("p");
|
||||
status.id = "pointer-status";
|
||||
status.setAttribute("aria-live", "polite");
|
||||
status.textContent = "pointer status: idle";
|
||||
canvas.insertAdjacentElement("afterend", status);
|
||||
let hoverReady = false;
|
||||
canvas.addEventListener("pointermove", () => {
|
||||
hoverReady = false;
|
||||
requestAnimationFrame(() => {
|
||||
hoverReady = true;
|
||||
});
|
||||
});
|
||||
canvas.addEventListener("pointerdown", (event) => {
|
||||
status.textContent = `pointer status: down button=${event.button} buttons=${event.buttons} hoverReady=${hoverReady}`;
|
||||
if (event.buttons === 1 && hoverReady) canvas.style.background = "#f9e79f";
|
||||
});
|
||||
canvas.addEventListener("click", () => {
|
||||
status.textContent += "; click received";
|
||||
});
|
||||
if (params.get("editor") === "1") {
|
||||
canvas.addEventListener("click", (event) => {
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
|
||||
const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
|
||||
status.textContent += `; selected row=${row + 1}`;
|
||||
document.querySelector("#canvas-cell-editor")?.remove();
|
||||
if (row !== 2) return;
|
||||
const editor = document.createElement("input");
|
||||
editor.id = "canvas-cell-editor";
|
||||
editor.setAttribute("aria-label", "selected canvas cell editor");
|
||||
editor.placeholder = "Type into selected row 3";
|
||||
canvas.insertAdjacentElement("afterend", editor);
|
||||
editor.focus();
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested fixture frame";
|
||||
iframe.src = "frame.html?label=fixture-nested-frame&interactive=1";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface frame matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame matrix</h1>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame&interactive=1"></iframe>
|
||||
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame&interactive=1"></iframe>
|
||||
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,118 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface discovery matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
.same-place { position: relative; width: 180px; height: 90px; }
|
||||
.same-place > div { position: absolute; inset: 0; }
|
||||
.same-place canvas { width: 180px; height: 90px; }
|
||||
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
|
||||
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
|
||||
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
|
||||
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
|
||||
.tiny { width: 8px; height: 8px; border-width: 1px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Rendered Surface discovery matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Visible named canvas</h2>
|
||||
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Tiny canvas remains addressable</h2>
|
||||
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Visible but pointer-events none</h2>
|
||||
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partial AX/fallback content</h2>
|
||||
<canvas aria-label="fixture-partial-ax" width="180" height="80">
|
||||
<button type="button">Fallback button must not suppress the surface</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Two real rendering layers</h2>
|
||||
<div class="stack">
|
||||
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Contained canvases are independent</h2>
|
||||
<div class="contained">
|
||||
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
|
||||
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Same geometry, different parents</h2>
|
||||
<div class="same-place">
|
||||
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
|
||||
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partially overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fully overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>CSS-hidden variants</h2>
|
||||
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
|
||||
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
|
||||
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Hidden ancestor variants</h2>
|
||||
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
|
||||
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
|
||||
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
|
||||
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Zero-size and offscreen</h2>
|
||||
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
|
||||
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
|
||||
label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
|
||||
input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
|
||||
iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<label>Frame input <input aria-label="frame input" /></label>
|
||||
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
||||
<div contenteditable="true" aria-label="frame editor"></div>
|
||||
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const rejecting = document.querySelector("#rejecting");
|
||||
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested input frame";
|
||||
iframe.src = "input-frame.html";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,23 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Frame input routing matrix</title>
|
||||
<style>
|
||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
|
||||
input { padding: 6px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame input routing matrix</h1>
|
||||
<label>Top input <input aria-label="top input" /></label>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin inputs" src="input-frame.html"></iframe>
|
||||
<iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
|
||||
<iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,68 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Unlabeled Rendered Surface matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
.case p { margin: 0 0 8px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Unlabeled Rendered Surface matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Only engineering attributes</h2>
|
||||
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Blank explicit names</h2>
|
||||
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Nearby prose is not a Canvas name</h2>
|
||||
<p>must-not-be-inferred-as-a-surface-label</p>
|
||||
<canvas width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fallback AX content does not name or suppress the Canvas</h2>
|
||||
<canvas width="180" height="80">
|
||||
<button type="button">Fallback control remains separate</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed rendering layers still cluster</h2>
|
||||
<div class="stack">
|
||||
<canvas width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed contained canvases stay independent</h2>
|
||||
<div class="contained">
|
||||
<canvas width="190" height="100"></canvas>
|
||||
<canvas width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface viewport matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { min-height: 1800px; padding: 16px; }
|
||||
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
|
||||
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
|
||||
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Viewport matrix</h1>
|
||||
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
|
||||
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
|
||||
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
||||
"!evals/browser/results/**",
|
||||
"!**/target/**",
|
||||
"!**/.wxt/**",
|
||||
"!**/.output/**",
|
||||
|
||||
@@ -80,16 +80,6 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
||||
|
||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
|
||||
|
||||
```bash
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
@@ -184,7 +174,7 @@ bsk emulate --session <id> --off
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
@@ -210,7 +200,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
|
||||
@@ -136,18 +136,6 @@ pub struct ClickArgs {
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
|
||||
/// Capture id returned by `bsk screenshot --ref <surface>`.
|
||||
#[arg(long)]
|
||||
pub capture: Option<String>,
|
||||
|
||||
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
|
||||
#[arg(long = "image-x", allow_hyphen_values = true)]
|
||||
pub image_x: Option<f64>,
|
||||
|
||||
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
|
||||
#[arg(long = "image-y", allow_hyphen_values = true)]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -159,14 +147,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let point_arg_count = usize::from(args.capture.is_some())
|
||||
+ usize::from(args.image_x.is_some())
|
||||
+ usize::from(args.image_y.is_some());
|
||||
if point_arg_count != 0 && point_arg_count != 3 {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"--capture, --image-x, and --image-y must be given together"
|
||||
)));
|
||||
}
|
||||
let params = ClickParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
@@ -180,9 +160,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Some(modifiers)
|
||||
},
|
||||
timeout_ms: Some(args.timeout),
|
||||
capture_id: args.capture.clone(),
|
||||
image_x: args.image_x,
|
||||
image_y: args.image_y,
|
||||
};
|
||||
let reply: ClickResult = call(
|
||||
info.sock_path,
|
||||
@@ -206,14 +183,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
"click ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
if let Some(capture_id) = &reply.capture_id {
|
||||
println!(
|
||||
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
|
||||
capture_id,
|
||||
reply.image_x.unwrap_or_default(),
|
||||
reply.image_y.unwrap_or_default()
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,23 +40,15 @@ use bsk_protocol::ErrorCode;
|
||||
pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
|
||||
pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
|
||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
||||
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
||||
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
||||
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
|
||||
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
|
||||
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
|
||||
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
|
||||
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
|
||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||
}
|
||||
@@ -236,27 +228,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
|
||||
summary: "focused frame target could not be determined",
|
||||
hint: Some(
|
||||
"pass an explicit --ref or --selector so the key event can be routed safely",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
|
||||
summary: "the page did not retain the requested input",
|
||||
hint: Some(
|
||||
"refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||
summary: "snapshot ref does not support this operation",
|
||||
hint: Some(
|
||||
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
|
||||
summary: "target element is not a <select>",
|
||||
hint: Some(
|
||||
@@ -299,41 +270,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
|
||||
summary: "surface capture transaction was not found",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and use its capture id with coordinates from that image",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
|
||||
summary: "surface capture transaction has expired",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
|
||||
summary: "surface capture transaction was already used",
|
||||
hint: Some(
|
||||
"each capture id is single-use; take a fresh surface screenshot before another point action",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
|
||||
summary: "page state changed since the surface screenshot",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
|
||||
summary: "surface image coordinate is invalid",
|
||||
hint: Some(
|
||||
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(
|
||||
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
|
||||
reason::RECORD_START_PAGE_UNREACHABLE,
|
||||
@@ -441,18 +377,6 @@ mod tests {
|
||||
assert!(!info.summary.contains("sandbox"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_capability_denied_explains_visual_surface_refs() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(info.summary, "snapshot ref does not support this operation");
|
||||
assert!(
|
||||
info.hint.unwrap().contains("capture id"),
|
||||
"expected visual-surface-specific hint"
|
||||
);
|
||||
assert_eq!(info.exit_code, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_busy_overrides_timeout_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||
@@ -495,25 +419,6 @@ mod tests {
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_capture_stale_requests_a_fresh_screenshot() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"page state changed since the surface screenshot"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
|
||||
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
|
||||
assert_eq!(info.summary, "surface image coordinate is invalid");
|
||||
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_not_found_overrides_not_found_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||
@@ -548,20 +453,4 @@ mod tests {
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("exactly one"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn focus_target_unresolved_overrides_cdp_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert_eq!(info.summary, "focused frame target could not be determined");
|
||||
assert!(info.hint.unwrap().contains("--ref"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_not_applied_overrides_cdp_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert_eq!(info.summary, "the page did not retain the requested input");
|
||||
assert!(info.hint.unwrap().contains("rejecting"));
|
||||
}
|
||||
}
|
||||
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
.map_err(CliError::Local)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let mut json = serde_json::json!({
|
||||
let json = serde_json::json!({
|
||||
"tab_id": reply.tab_id,
|
||||
"width": reply.width,
|
||||
"height": reply.height,
|
||||
@@ -67,10 +67,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
"path": out_path.to_string_lossy(),
|
||||
"byte_size": bytes.len(),
|
||||
});
|
||||
if let Some(capture) = &reply.capture {
|
||||
json["capture"] = serde_json::to_value(capture)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json)
|
||||
@@ -79,12 +75,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
}
|
||||
Format::Human => {
|
||||
println!("{}", out_path.display());
|
||||
if let Some(capture) = &reply.capture {
|
||||
println!(
|
||||
"capture={} surface={} coordinate_space={} expires_at={}",
|
||||
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,29 +188,6 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_surface_capture_image_coordinates() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"click",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"160.5",
|
||||
"--image-y",
|
||||
"-1",
|
||||
]);
|
||||
let Command::Click(args) = cli.command else {
|
||||
panic!("expected click command");
|
||||
};
|
||||
assert_eq!(args.capture.as_deref(), Some("sc_test"));
|
||||
assert_eq!(args.image_x, Some(160.5));
|
||||
assert_eq!(args.image_y, Some(-1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
|
||||
@@ -238,7 +238,6 @@ async fn screenshot_returns_image_base64_with_dimensions() {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -287,7 +286,6 @@ async fn screenshot_forwards_ref_to_extension() {
|
||||
height: 60,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
|
||||
@@ -354,9 +354,6 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
used_selector: None,
|
||||
x: 12.5,
|
||||
y: 34.0,
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -376,9 +373,6 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -616,9 +610,6 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
click_count: None,
|
||||
modifiers: None,
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -16,13 +16,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"capture_id": {
|
||||
"description": "Short-lived capture id returned by a visual Surface screenshot.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"click_count": {
|
||||
"description": "Number of consecutive mouse presses (double-click = 2).",
|
||||
"type": [
|
||||
@@ -32,22 +25,6 @@
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
},
|
||||
"image_x": {
|
||||
"description": "X coordinate in the capture image's pixel coordinate space.",
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"image_y": {
|
||||
"description": "Y coordinate in the capture image's pixel coordinate space.",
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
|
||||
@@ -8,12 +8,6 @@
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"capture_id": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"description": "Native JS dialogs observed and auto-handled during this call.",
|
||||
"type": "array",
|
||||
@@ -21,20 +15,6 @@
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"image_x": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"image_y": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
|
||||
@@ -8,16 +8,6 @@
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"capture": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/SurfaceCaptureInfo"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -119,31 +109,6 @@
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"SurfaceCaptureInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"coordinate_space",
|
||||
"expires_at",
|
||||
"id",
|
||||
"surface_ref"
|
||||
],
|
||||
"properties": {
|
||||
"coordinate_space": {
|
||||
"type": "string"
|
||||
},
|
||||
"expires_at": {
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"surface_ref": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,15 +67,6 @@ pub struct ClickParams {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
/// Short-lived capture id returned by a visual Surface screenshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture_id: Option<String>,
|
||||
/// X coordinate in the capture image's pixel coordinate space.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_x: Option<f64>,
|
||||
/// Y coordinate in the capture image's pixel coordinate space.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -89,12 +80,6 @@ pub struct ClickResult {
|
||||
/// agents can correlate with a follow-up `tool.screenshot`.
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_x: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_y: Option<f64>,
|
||||
/// Native JS dialogs observed and auto-handled during this call.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
@@ -303,9 +288,6 @@ mod tests {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: Some("sc_test".into()),
|
||||
image_x: Some(160.0),
|
||||
image_y: Some(90.0),
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||
|
||||
@@ -208,20 +208,10 @@ pub struct ScreenshotResult {
|
||||
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
|
||||
pub format: String,
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture: Option<SurfaceCaptureInfo>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SurfaceCaptureInfo {
|
||||
pub id: String,
|
||||
pub surface_ref: String,
|
||||
pub coordinate_space: String,
|
||||
pub expires_at: u64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -303,12 +293,6 @@ mod tests {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: Some(SurfaceCaptureInfo {
|
||||
id: "sc_test".into(),
|
||||
surface_ref: "@e3".into(),
|
||||
coordinate_space: "capture-image-pixel".into(),
|
||||
expires_at: 1_700_000_030_000,
|
||||
}),
|
||||
dialogs: vec![],
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
# Browser capability evaluation corpus
|
||||
|
||||
This directory is a deterministic, local browser evaluation environment for BrowserSkill, DSH,
|
||||
and other command-line agents. Cases, fixture pages, direct `bsk` workflows, and assertions are
|
||||
data-driven and discovered automatically. Adding a case does not require editing a central switch.
|
||||
|
||||
[中文说明](README.zh-CN.md)
|
||||
|
||||
## Design goals
|
||||
|
||||
- Agent-neutral: prompts describe observable page goals so the same cases can run through different
|
||||
agents and adapters.
|
||||
- Deterministic: pages run locally and use stable markers and run-scoped events.
|
||||
- Extensible: manifests, fixture modules, and workflows are auto-discovered.
|
||||
- Reproducible: generated cases record a seed and derive every DOM variation from it.
|
||||
- Honest verification: page-observable results, response markers, and adapter evidence are reported
|
||||
separately. Missing adapter evidence is `unverified`, never silently treated as passed.
|
||||
- Privacy-safe: user badcases must be reduced to synthetic DOM and anonymized data before commit.
|
||||
|
||||
The corpus currently has six stable `core` cases and one seeded `matrix` case. The operation
|
||||
inventory contains 28 browser operations; that count is capability coverage, not the number of
|
||||
cases. Cases can grow independently and many cases may cover the same operation under different
|
||||
page conditions.
|
||||
|
||||
## Quick start
|
||||
|
||||
These commands validate manifests and run unit/integration tests without launching a browser:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser validate
|
||||
pnpm eval:browser list
|
||||
pnpm eval:browser coverage
|
||||
pnpm eval:browser:test
|
||||
```
|
||||
|
||||
Build the repository CLI and exercise the real CLI, daemon, extension, Agent Window, and pages:
|
||||
|
||||
```sh
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
Every smoke case starts a fresh browser session and stops it in a `finally` cleanup path. Reports
|
||||
and screenshots are written to the ignored `evals/browser/results/` directory.
|
||||
|
||||
`pnpm run eval:browser -- coverage` is also supported. The CLI intentionally accepts pnpm's leading
|
||||
`--` separator.
|
||||
|
||||
## Suites and selection
|
||||
|
||||
| Suite | Intended use | Default execution |
|
||||
| --- | --- | --- |
|
||||
| `core` | Small stable acceptance set for everyday smoke and CI | Yes |
|
||||
| `matrix` | Seeded DOM, timing, and layout variations | Explicit |
|
||||
| `regression` | Minimized user badcases tied to an issue or fix | Explicit |
|
||||
| `stress` | Higher volume or slower boundary testing | Explicit |
|
||||
| `manual` | Human-dependent capabilities that cannot be deterministic | Explicit |
|
||||
|
||||
`smoke` and `run-agent` default to `core`, so adding a stress or regression case cannot silently
|
||||
slow down existing CI. `list`, `coverage`, and `validate` inspect the full corpus by default.
|
||||
|
||||
```sh
|
||||
pnpm eval:browser list --suite core
|
||||
pnpm eval:browser list --tag form
|
||||
pnpm eval:browser smoke --case form-controls
|
||||
pnpm eval:browser smoke --case form-controls,generated-form
|
||||
pnpm eval:browser smoke --case all
|
||||
```
|
||||
|
||||
Filters from different dimensions are combined. Repeated/comma-separated suites or case ids are
|
||||
alternatives; every requested tag must be present. `--task` remains a backward-compatible alias for
|
||||
`--case`.
|
||||
|
||||
## Directory and discovery model
|
||||
|
||||
```text
|
||||
evals/browser/
|
||||
cases/
|
||||
core/*.case.json
|
||||
matrix/*.case.json
|
||||
regression/<case-id>/
|
||||
<case-id>.case.json
|
||||
<case-id>.fixture.mjs
|
||||
README.md
|
||||
fixtures/pages/*.fixture.mjs
|
||||
schemas/case.schema.json
|
||||
lib/
|
||||
tests/
|
||||
```
|
||||
|
||||
- Any `cases/**/*.case.json` file is loaded as a case manifest.
|
||||
- Any `fixtures/**/*.fixture.mjs` or `cases/**/*.fixture.mjs` file is registered as a fixture.
|
||||
- Manifests are ordered by suite, optional numeric `order`, then id.
|
||||
- `validate` rejects duplicate case ids, duplicate routes, missing fixture routes, unknown
|
||||
operations/actions, invalid workflow variables, and adapter assertions with no evidence producer.
|
||||
|
||||
## Case manifest
|
||||
|
||||
A case owns all information needed by both an agent evaluation and direct CLI smoke:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "example-case",
|
||||
"title": "Example browser behavior",
|
||||
"order": 10,
|
||||
"suite": "core",
|
||||
"tags": ["form", "keyboard"],
|
||||
"seed": 17,
|
||||
"fixture": { "startPath": "/example" },
|
||||
"prompts": { "en": "Open {url} ...", "zh-CN": "打开 {url}……" },
|
||||
"coverage": ["page.navigate", "interact.press"],
|
||||
"assertions": { "site": [], "response": [], "adapter": [] },
|
||||
"smoke": { "steps": [{ "action": "navigate" }] }
|
||||
}
|
||||
```
|
||||
|
||||
Prompt placeholders are `{url}`, `{baseUrl}`, `{runId}`, and `{seed}`. Agent adapter arguments also
|
||||
support `{prompt}`, `{caseId}` / `{taskId}`, and `{variant}`.
|
||||
|
||||
Assertions have three independent sources:
|
||||
|
||||
- `site`: events emitted by the local page, such as submitted values or a history navigation.
|
||||
- `response`: stable text that must appear in the agent or CLI output.
|
||||
- `adapter`: facts outside page JavaScript, such as a non-empty screenshot or confirmed session
|
||||
cleanup.
|
||||
|
||||
The declarative smoke workflow supports navigation/history, waits, observe/snapshot/HTML,
|
||||
screenshot/console/network, click/hover/fill/select/press, tabs, resize/emulation, and the manual
|
||||
ownership/help operations. A step may `saveAs` its JSON result and later reference values such as
|
||||
`{child.tab_id}`. A step may set `evidence` for an adapter assertion. See existing manifests for
|
||||
complete examples.
|
||||
|
||||
## Adding a normal case
|
||||
|
||||
1. Add a `*.fixture.mjs` module exporting `{ id, routes, render(context) }`. It can live in shared
|
||||
`fixtures/pages/` or beside a self-contained regression case.
|
||||
2. Add a `*.case.json` manifest with prompts, operation coverage, assertions, and smoke steps.
|
||||
3. Make all page events run-scoped using the shared `browserEval.send(...)` client helper.
|
||||
4. Run `pnpm eval:browser validate` and `pnpm eval:browser:test`.
|
||||
5. Run the new case against the real local build with `smoke --case <id>`.
|
||||
|
||||
Prefer one behavior or failure mechanism per case. Stable synthetic markers make failures easier to
|
||||
diagnose than assertions against incidental prose.
|
||||
|
||||
## Turning a user badcase into a regression
|
||||
|
||||
Create a self-contained starting point:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser scaffold reported-timeout \
|
||||
--title "Navigation settles after a late frame" \
|
||||
--source "issue-123"
|
||||
```
|
||||
|
||||
The command creates a manifest, fixture, and reproduction notes under
|
||||
`cases/regression/reported-timeout/`. Then reduce the report to the smallest relevant mechanism:
|
||||
|
||||
- Keep DOM structure, timing, redirects, frames, or browser state that triggers the bug.
|
||||
- Replace names, account data, URLs, tokens, screenshots, and page copy with synthetic values.
|
||||
- Do not commit HAR files, cookies, credentials, production HTML, or proprietary assets.
|
||||
- Record the issue/PR reference, seed when relevant, expected result, and the fixing PR.
|
||||
- Confirm the new case fails on the known-bad build and passes on the fixed build when practical.
|
||||
|
||||
## Seeded matrix cases
|
||||
|
||||
`generated-form` varies label association, nesting depth, hydration delay, decoy controls, field
|
||||
order, and element ids while keeping the task and oracle stable. Inspect a seed without a browser:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser generate --seed user-badcase-42
|
||||
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
|
||||
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
|
||||
pnpm eval:browser smoke --case generated-form --seed 4,7,14
|
||||
```
|
||||
|
||||
Repeated/comma-separated seeds run as independent results. Reports store every normalized seed, so
|
||||
a failure can be replayed exactly and promoted into a named regression case if it represents a
|
||||
distinct bug.
|
||||
|
||||
## Run any command-line agent
|
||||
|
||||
Copy `agents.example.json` to the ignored `agents.local.json` and adapt its command. Arguments are
|
||||
passed directly to the executable without a shell.
|
||||
|
||||
```sh
|
||||
pnpm eval:browser run-agent \
|
||||
--config evals/browser/agents.local.json \
|
||||
--agent dsh-local \
|
||||
--suite core \
|
||||
--repeat 3
|
||||
```
|
||||
|
||||
An adapter may define `command`, `args`, `cwd`, `env`, `timeoutMs`, `variant`, free-form `metadata`,
|
||||
and optional `metrics.errorPattern` / `metrics.toolCallPattern` regular expressions.
|
||||
|
||||
To compare 28 granular tools with domain tools, create two isolated adapter entries and run both
|
||||
against the same selected cases and seeds:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser run-agent \
|
||||
--config evals/browser/agents.local.json \
|
||||
--agent dsh-granular,dsh-domain \
|
||||
--suite core \
|
||||
--repeat 10
|
||||
```
|
||||
|
||||
Keep browser runs sequential to avoid session contention. Use fresh profiles and alternate adapter
|
||||
order in larger experiments to reduce warm-cache bias.
|
||||
|
||||
## Reading results
|
||||
|
||||
```sh
|
||||
pnpm eval:browser summarize evals/browser/results/*.json
|
||||
```
|
||||
|
||||
The summary groups by adapter and variant and reports pass rate, fully verified rate, process
|
||||
failures, matched error count, average tool calls, and average duration. The raw JSON retains case,
|
||||
suite, tags, seed, prompt, command output, page event count, and individual oracle checks.
|
||||
|
||||
A healthy run has:
|
||||
|
||||
- no execution timeout or non-zero exit;
|
||||
- no `executionError` in direct smoke;
|
||||
- `verification.status` equal to `passed` (or intentionally `passed-with-unverified` for an agent
|
||||
adapter that cannot expose external evidence);
|
||||
- every session cleanup assertion verified;
|
||||
- no unexpected increase in errors, tool calls, or duration versus the chosen baseline.
|
||||
|
||||
## Development baseline
|
||||
|
||||
Use these lanes as commit and PR gates:
|
||||
|
||||
| Change | Required checks |
|
||||
| --- | --- |
|
||||
| Any case, fixture, oracle, or harness change | `pnpm eval:browser:check` |
|
||||
| CLI, daemon, extension, DSH plugin, or browser-operation change | The check above, `cargo build -p bsk`, then real-browser `smoke --suite core` |
|
||||
| DOM discovery, form interaction, timing, or waiting change | The checks above, then `smoke --suite matrix --seed 4,7,14` |
|
||||
| A bug fix with a known reproduction | Add a `regression` case and run that case on the fixed build |
|
||||
|
||||
Before committing this harness, the complete acceptance sequence is:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser:check
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
All commands must exit zero. The smoke summary must have 100% pass and fully verified rates, zero
|
||||
execution failures/errors, and `bsk session list --json` must be `[]` afterward. Do not weaken an
|
||||
oracle or replace a stable marker merely to make a regression pass; behavior changes should update
|
||||
the case and expected baseline in the same reviewed change.
|
||||
|
||||
The direct smoke lane covers 25 of 28 operations. `tabs.borrow`, `tabs.return`, and
|
||||
`assist.request-help` stay manual because they require a real user tab or human interaction and
|
||||
would make the default suite destructive or non-deterministic.
|
||||
@@ -0,0 +1,197 @@
|
||||
# 浏览器能力测试语料库
|
||||
|
||||
这里是一套供 BrowserSkill、DSH 以及其他命令行 Agent 共用的本地确定性浏览器测试环境。
|
||||
case、测试页面、`bsk` 直连流程和断言都会自动发现;新增 case 不需要修改中央 `switch`。
|
||||
|
||||
[English](README.md)
|
||||
|
||||
## 核心原则
|
||||
|
||||
- 与 Agent 无关:prompt 只描述可观察的页面目标,同一套 case 可以被不同 Agent 和
|
||||
adapter 使用。
|
||||
- 本地且确定:页面只依赖本地服务,使用稳定 marker 和按 run 隔离的事件。
|
||||
- 可扩展:manifest、fixture 和 workflow 都从目录自动发现。
|
||||
- 可复现:生成式页面的所有变化由 seed 决定,报告会保留 seed。
|
||||
- 如实验证:页面事件、回答文本和 adapter 外部证据分别统计;拿不到的证据标为
|
||||
`unverified`,不会冒充通过。
|
||||
- 隐私安全:用户 badcase 必须先缩减为合成 DOM 和匿名数据,再进入仓库。
|
||||
|
||||
当前有 6 个稳定的 `core` case 和 1 个 seed 驱动的 `matrix` case。28 表示浏览器操作能力
|
||||
清单,并不是 case 数量;同一种操作可以在不同页面结构、时序和浏览器状态下由多个 case
|
||||
重复覆盖。
|
||||
|
||||
## 快速使用
|
||||
|
||||
不启动浏览器的校验:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser validate
|
||||
pnpm eval:browser list
|
||||
pnpm eval:browser coverage
|
||||
pnpm eval:browser:test
|
||||
```
|
||||
|
||||
使用仓库当前代码编译 `bsk`,再跑真实 CLI、daemon、扩展、小窗和页面链路:
|
||||
|
||||
```sh
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
每个 smoke case 都会创建独立 session,并在 `finally` 中关闭。JSON 报告和截图输出到已被
|
||||
Git 忽略的 `evals/browser/results/`。
|
||||
|
||||
也可以写成 `pnpm run eval:browser -- coverage`;CLI 已兼容 pnpm 传入的前导 `--`。
|
||||
|
||||
## Suite 与筛选
|
||||
|
||||
| Suite | 用途 | 默认执行 |
|
||||
| --- | --- | --- |
|
||||
| `core` | 日常 smoke 和 CI 使用的小而稳定的验收集 | 是 |
|
||||
| `matrix` | seed 驱动的 DOM、时序、布局组合 | 显式选择 |
|
||||
| `regression` | 从用户 badcase 缩减并与 issue/修复关联的回归 | 显式选择 |
|
||||
| `stress` | 更高次数或更慢的边界测试 | 显式选择 |
|
||||
| `manual` | 必须依赖真人或真实用户标签页的能力 | 显式选择 |
|
||||
|
||||
`smoke` 和 `run-agent` 不带筛选时只跑 `core`,所以以后新增 stress/badcase 不会突然拖慢
|
||||
已有 CI。`list`、`coverage`、`validate` 默认检查全部语料。
|
||||
|
||||
```sh
|
||||
pnpm eval:browser list --suite core
|
||||
pnpm eval:browser list --tag form
|
||||
pnpm eval:browser smoke --case form-controls
|
||||
pnpm eval:browser smoke --case form-controls,generated-form
|
||||
pnpm eval:browser smoke --case all
|
||||
```
|
||||
|
||||
不同维度的筛选会叠加;逗号分隔的 suite/case 是“或”,多个 tag 必须全部命中。旧参数
|
||||
`--task` 仍作为 `--case` 的兼容别名。
|
||||
|
||||
## 自动发现结构
|
||||
|
||||
```text
|
||||
evals/browser/
|
||||
cases/
|
||||
core/*.case.json
|
||||
matrix/*.case.json
|
||||
regression/<case-id>/
|
||||
<case-id>.case.json
|
||||
<case-id>.fixture.mjs
|
||||
README.md
|
||||
fixtures/pages/*.fixture.mjs
|
||||
schemas/case.schema.json
|
||||
lib/
|
||||
tests/
|
||||
```
|
||||
|
||||
- `cases/**/*.case.json` 都会作为 case manifest 加载。
|
||||
- `fixtures/**/*.fixture.mjs` 和 `cases/**/*.fixture.mjs` 都会注册为页面 fixture。
|
||||
- 顺序按 suite、可选的数字 `order`、id 排列。
|
||||
- `validate` 会阻止重复 case id、重复路由、缺失 fixture、未知操作/action、无效 workflow
|
||||
变量,以及没有任何步骤产出的 adapter evidence。
|
||||
|
||||
一个 manifest 同时描述:中英文 prompt、能力 coverage、页面事件/回答/adapter 三类断言,以及
|
||||
`bsk` 直连 smoke 的声明式步骤。prompt 可用 `{url}`、`{baseUrl}`、`{runId}`、`{seed}`;
|
||||
workflow 可以用 `saveAs` 保存 JSON 结果,并用 `{child.tab_id}` 之类的变量引用后续步骤。
|
||||
完整写法可直接参考 `cases/core/` 和 `cases/matrix/`。
|
||||
|
||||
## 新增普通 case
|
||||
|
||||
1. 新增导出 `{ id, routes, render(context) }` 的 `*.fixture.mjs`,可放共享 fixture 目录,也可
|
||||
和 regression case 放在一起。
|
||||
2. 新增 `*.case.json`,写好 prompt、operation coverage、断言和 smoke steps。
|
||||
3. 页面使用共享的 `browserEval.send(...)` 上报按 run 隔离的事件。
|
||||
4. 执行 `pnpm eval:browser validate` 和 `pnpm eval:browser:test`。
|
||||
5. 用 `smoke --case <id>` 验证真实本地编译链路。
|
||||
|
||||
建议一个 case 只表达一种行为或故障机制,并使用稳定的合成 marker,避免断言偶然文案。
|
||||
|
||||
## 把用户 badcase 固化成回归
|
||||
|
||||
先生成一个自包含模板:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser scaffold reported-timeout \
|
||||
--title "Navigation settles after a late frame" \
|
||||
--source "issue-123"
|
||||
```
|
||||
|
||||
它会在 `cases/regression/reported-timeout/` 下生成 manifest、fixture 和复现说明。随后只保留
|
||||
真正触发问题的 DOM 结构、时序、跳转、frame 或浏览器状态,并遵守:
|
||||
|
||||
- 所有姓名、账号、网址、token、截图和文案都替换为合成值。
|
||||
- 不提交 HAR、cookie、凭据、生产 HTML 或私有素材。
|
||||
- 记录 issue/PR 来源、相关 seed、期望行为和后续修复 PR。
|
||||
- 条件允许时,证明新 case 在旧版本失败、修复版本通过。
|
||||
|
||||
## Seed 矩阵
|
||||
|
||||
`generated-form` 会确定性改变 label 关联方式、DOM 嵌套、hydration 延迟、干扰控件、字段
|
||||
顺序和 element id,但任务和 oracle 保持不变:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser generate --seed user-badcase-42
|
||||
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
|
||||
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
|
||||
pnpm eval:browser smoke --case generated-form --seed 4,7,14
|
||||
```
|
||||
|
||||
逗号分隔或重复的 seed 会生成相互独立的结果。任何失败都能按报告中的规范化 seed 精确重放;
|
||||
若它代表独立 bug,再提升为命名 regression case。
|
||||
|
||||
## 测任意 Agent 与比较 28/6 工具
|
||||
|
||||
把 `agents.example.json` 复制为已被 Git 忽略的 `agents.local.json`,分别配置两个隔离的 Agent
|
||||
profile/build。参数以数组直接传给可执行程序,不经过 shell。
|
||||
|
||||
```sh
|
||||
pnpm eval:browser run-agent \
|
||||
--config evals/browser/agents.local.json \
|
||||
--agent dsh-granular,dsh-domain \
|
||||
--suite core \
|
||||
--repeat 10
|
||||
```
|
||||
|
||||
浏览器任务应串行运行,避免 session 竞争;更大实验中使用干净 profile 并轮换 adapter 顺序,
|
||||
降低缓存偏差。
|
||||
|
||||
## 如何看结果
|
||||
|
||||
```sh
|
||||
pnpm eval:browser summarize evals/browser/results/*.json
|
||||
```
|
||||
|
||||
汇总按 adapter/variant 展示通过率、完全验证率、进程失败、错误数、平均工具调用数和耗时。
|
||||
原始 JSON 会保留 case、suite、tag、seed、prompt、命令输出、页面事件数和每条 oracle 结果。
|
||||
|
||||
健康运行应满足:没有 timeout/非零退出,direct smoke 没有 `executionError`,验证状态为
|
||||
`passed`(无法暴露外部证据的 Agent adapter 可明确为 `passed-with-unverified`),session cleanup
|
||||
通过,并且错误数、调用数和耗时没有相对基线异常上涨。
|
||||
|
||||
## 开发基准线
|
||||
|
||||
后续开发统一按下面的门禁执行:
|
||||
|
||||
| 改动范围 | 必跑检查 |
|
||||
| --- | --- |
|
||||
| 任意 case、fixture、oracle 或 harness 改动 | `pnpm eval:browser:check` |
|
||||
| CLI、daemon、扩展、DSH 插件或浏览器操作改动 | 上述检查、`cargo build -p bsk`、真实浏览器 `smoke --suite core` |
|
||||
| DOM 发现、表单交互、时序或等待逻辑改动 | 上述检查,再跑 `smoke --suite matrix --seed 4,7,14` |
|
||||
| 修复已有明确复现的 bug | 新增 `regression` case,并在修复版本单独运行该 case |
|
||||
|
||||
提交这套 harness 前,完整验收命令为:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser:check
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
所有命令必须以 0 退出;smoke 汇总必须达到 100% pass、100% fully verified、0 execution
|
||||
failure/error,结束后 `bsk session list --json` 必须为 `[]`。不能为了让回归通过而削弱 oracle
|
||||
或替换稳定 marker;如果产品行为确实改变,case 与预期基线必须在同一个受审查改动中更新。
|
||||
|
||||
直连 smoke 自动覆盖 28 项中的 25 项。`tabs.borrow`、`tabs.return`、`assist.request-help` 需要
|
||||
真实用户标签页或真人交互,因此保留在 manual lane,避免默认测试具有破坏性或不确定性。
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"agents": {
|
||||
"dsh-local": {
|
||||
"variant": "granular-28",
|
||||
"command": "dsh",
|
||||
"args": ["--profile", "headless", "{prompt}"],
|
||||
"cwd": "../..",
|
||||
"timeoutMs": 300000,
|
||||
"metadata": {
|
||||
"toolCount": 28
|
||||
},
|
||||
"metrics": {
|
||||
"toolCallPattern": "browser[_-][a-z_-]+"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# Browser evaluation cases
|
||||
|
||||
Cases are discovered recursively from `*.case.json`; colocated `*.fixture.mjs` files are discovered
|
||||
as fixtures. Use `core` for small deterministic acceptance checks, `matrix` for seeded variations,
|
||||
and `regression/<case-id>/` for privacy-safe minimized user badcases.
|
||||
|
||||
Run `pnpm eval:browser validate` after every case change. See the parent
|
||||
[`README.md`](../README.md) for the manifest contract, workflow actions, scaffolding, and acceptance
|
||||
criteria.
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "diagnostics",
|
||||
"title": "Console and network inspection",
|
||||
"order": 50,
|
||||
"suite": "core",
|
||||
"tags": ["console", "network", "diagnostics"],
|
||||
"fixture": { "startPath": "/diagnostics" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Use browser diagnostics to inspect the page console and recent network activity. Report the console marker and the token returned by the ping request, then close the browser session.",
|
||||
"zh-CN": "打开 {url}。使用浏览器诊断能力检查页面 console 和最近的 network 活动,报告 console marker 以及 ping 请求返回的 token,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"inspect.console",
|
||||
"inspect.network"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "diagnostic ping completed",
|
||||
"type": "network.ping",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported console marker", "includes": "CONSOLE-61" },
|
||||
{ "label": "reported network token", "includes": "NETWORK-73" }
|
||||
],
|
||||
"adapter": [
|
||||
{ "label": "console inspection was executed", "key": "consoleRead" },
|
||||
{ "label": "network inspection was executed", "key": "networkRead" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "wait", "duration": "300ms" },
|
||||
{ "action": "console", "evidence": "consoleRead" },
|
||||
{ "action": "network", "evidence": "networkRead" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "form-controls",
|
||||
"title": "Form, select, keyboard, and screenshot",
|
||||
"order": 20,
|
||||
"suite": "core",
|
||||
"tags": ["form", "keyboard", "screenshot", "select"],
|
||||
"fixture": { "startPath": "/form" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Fill Text input with \"agent-parity\" and Textarea with \"BrowserSkill works\". Select the visible option \"Two\" from Dropdown. Capture a screenshot of the form, focus Submit, and press Enter to submit. Confirm that the result says \"Received!\", then close the browser session.",
|
||||
"zh-CN": "打开 {url}。把 Text input 填成“agent-parity”,Textarea 填成“BrowserSkill works”,Dropdown 选择可见选项“Two”。截取表单截图,然后聚焦 Submit 并按 Enter 提交。确认结果页出现“Received!”,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.fill",
|
||||
"interact.select",
|
||||
"interact.press",
|
||||
"inspect.screenshot"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "Enter was pressed on Submit",
|
||||
"type": "form.enter_pressed",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "form values were submitted",
|
||||
"type": "form.submitted",
|
||||
"where": {
|
||||
"data.text": "agent-parity",
|
||||
"data.notes": "BrowserSkill works",
|
||||
"data.choice": "two"
|
||||
},
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "result page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/result" },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "confirmed result", "includes": "Received!" }],
|
||||
"adapter": [
|
||||
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{
|
||||
"action": "fill",
|
||||
"selector": "#text-input",
|
||||
"value": "agent-parity"
|
||||
},
|
||||
{
|
||||
"action": "fill",
|
||||
"selector": "#notes",
|
||||
"value": "BrowserSkill works"
|
||||
},
|
||||
{ "action": "select", "selector": "#choice", "values": ["two"] },
|
||||
{
|
||||
"action": "screenshot",
|
||||
"artifactName": "form.png",
|
||||
"evidence": "screenshotCreated"
|
||||
},
|
||||
{ "action": "press", "key": "Enter", "selector": "#submit" },
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "page.shown",
|
||||
"where": { "path": "/result" }
|
||||
},
|
||||
{ "action": "wait", "duration": "200ms" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "hover-inspect",
|
||||
"title": "Hover, local HTML, and screenshot",
|
||||
"order": 30,
|
||||
"suite": "core",
|
||||
"tags": ["hover", "html", "screenshot"],
|
||||
"fixture": { "startPath": "/hover-inspect" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Hover over Account, then choose Profile beta from the revealed menu. Inspect the local HTML around the Inspection card and capture a screenshot. Report both marker codes you find, then close the browser session.",
|
||||
"zh-CN": "打开 {url}。把鼠标悬停在“Account”上,然后点击展开菜单中的“Profile beta”。检查 Inspection card 附近的局部 HTML,并截取页面截图。报告找到的两个标记代码,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.hover",
|
||||
"interact.click",
|
||||
"inspect.html",
|
||||
"inspect.screenshot"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "Account menu was hovered",
|
||||
"type": "hover.menu_opened",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "Profile beta was selected",
|
||||
"type": "hover.profile_clicked",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported hover marker", "includes": "HOVER-17" },
|
||||
{ "label": "reported HTML marker", "includes": "INSPECT-27" }
|
||||
],
|
||||
"adapter": [
|
||||
{ "label": "HTML inspection was executed", "key": "htmlInspected" },
|
||||
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "hover", "selector": "#account-menu" },
|
||||
{ "action": "click", "selector": "#profile-link" },
|
||||
{ "action": "html", "evidence": "htmlInspected" },
|
||||
{
|
||||
"action": "screenshot",
|
||||
"artifactName": "hover-inspect.png",
|
||||
"evidence": "screenshotCreated"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "mobile-emulation",
|
||||
"title": "Mobile emulation",
|
||||
"order": 60,
|
||||
"suite": "core",
|
||||
"tags": ["emulation", "mobile", "viewport"],
|
||||
"fixture": { "startPath": "/responsive" },
|
||||
"prompts": {
|
||||
"en": "Open {url}, emulate an iPhone 14, and report the viewport marker shown on the page. Close the browser session when finished.",
|
||||
"zh-CN": "打开 {url},模拟 iPhone 14,并报告页面显示的 viewport marker。完成后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": ["session.start", "session.stop", "page.navigate", "assist.emulate"],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "iPhone-sized viewport was observed",
|
||||
"type": "viewport.sample",
|
||||
"where": { "data.width": 390, "data.height": 844 },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported viewport marker", "includes": "VIEWPORT-84" }],
|
||||
"adapter": [
|
||||
{ "label": "device emulation was executed", "key": "emulatedDevice" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "resize", "width": 900, "height": 700 },
|
||||
{
|
||||
"action": "emulate",
|
||||
"device": "iphone-14",
|
||||
"evidence": "emulatedDevice"
|
||||
},
|
||||
{ "action": "wait", "duration": "300ms" },
|
||||
{ "action": "html" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "navigation-history",
|
||||
"title": "Navigation, history, and reload",
|
||||
"order": 10,
|
||||
"suite": "core",
|
||||
"tags": ["navigation", "history", "reload"],
|
||||
"fixture": { "startPath": "/navigation/start" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Read the main heading, follow the Details link, and report the code shown there. Then go back, go forward, reload the page, and close the browser session. Use only the browser capability for page operations.",
|
||||
"zh-CN": "打开 {url}。读取页面主标题,点击“详情”,并报告详情页上的代码。然后依次后退、前进、刷新页面,最后关闭浏览器会话。页面操作只使用浏览器能力。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"page.back",
|
||||
"page.forward",
|
||||
"page.reload",
|
||||
"interact.click"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "details link was clicked",
|
||||
"type": "navigation.detail.clicked",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "back returned to the start page",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/navigation/start" },
|
||||
"minCount": 2
|
||||
},
|
||||
{
|
||||
"label": "forward returned to the detail page",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/navigation/detail" },
|
||||
"minCount": 2
|
||||
},
|
||||
{
|
||||
"label": "detail page was reloaded",
|
||||
"type": "page.shown",
|
||||
"where": {
|
||||
"data.path": "/navigation/detail",
|
||||
"data.navigationType": "reload"
|
||||
},
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported detail code", "includes": "NAV-42" }],
|
||||
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "observe" },
|
||||
{ "action": "snapshot" },
|
||||
{ "action": "click", "selector": "#details-link" },
|
||||
{ "action": "html" },
|
||||
{ "action": "back" },
|
||||
{ "action": "forward" },
|
||||
{ "action": "reload" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "tabs",
|
||||
"title": "Create, select, and close a tab",
|
||||
"order": 40,
|
||||
"suite": "core",
|
||||
"tags": ["tabs", "ownership", "lifecycle"],
|
||||
"fixture": { "startPath": "/tabs" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Create a new browser tab at {baseUrl}/tabs/child?run={runId}, read its token, click \"Mark child visited\", close the child tab, return to the original tab, and close the browser session.",
|
||||
"zh-CN": "打开 {url}。新建一个浏览器标签页并访问 {baseUrl}/tabs/child?run={runId},读取其中的 token,点击“Mark child visited”,关闭子标签页,回到原标签页,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.click",
|
||||
"tabs.create",
|
||||
"tabs.select",
|
||||
"tabs.close"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "child tab page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/tabs/child" },
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "child tab action ran",
|
||||
"type": "tabs.child_marked",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported child token", "includes": "TAB-29" }],
|
||||
"adapter": [
|
||||
{ "label": "child tab was closed", "key": "childTabClosed" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate", "saveAs": "original" },
|
||||
{ "action": "tab-list" },
|
||||
{
|
||||
"action": "tab-create",
|
||||
"url": "{baseUrl}/tabs/child?run={runId}",
|
||||
"saveAs": "child"
|
||||
},
|
||||
{
|
||||
"action": "navigate",
|
||||
"url": "{baseUrl}/tabs/child?run={runId}",
|
||||
"tabId": "{child.tab_id}"
|
||||
},
|
||||
{
|
||||
"action": "click",
|
||||
"selector": "#mark-child",
|
||||
"tabId": "{child.tab_id}"
|
||||
},
|
||||
{ "action": "html", "tabId": "{child.tab_id}" },
|
||||
{
|
||||
"action": "tab-close",
|
||||
"tabId": "{child.tab_id}",
|
||||
"evidence": "childTabClosed"
|
||||
},
|
||||
{ "action": "tab-select", "tabId": "{original.tab_id}" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# Seeded matrix cases
|
||||
|
||||
Matrix cases vary page structure or timing deterministically from a recorded seed. The task,
|
||||
assertions, and success marker remain stable so a failed seed can be replayed exactly. A seed that
|
||||
exposes a distinct product bug should be minimized and promoted to a named regression case.
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "generated-form",
|
||||
"title": "Seeded form structure and hydration variants",
|
||||
"order": 10,
|
||||
"suite": "matrix",
|
||||
"tags": ["dynamic-dom", "form", "hydration", "seeded"],
|
||||
"seed": 20260827,
|
||||
"fixture": { "startPath": "/matrix/generated-form" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Wait for the seeded form to finish hydrating. Fill Text input with \"seeded-agent\", select the visible option \"Two\", submit with Enter, and report the result marker and seed. Close the browser session.",
|
||||
"zh-CN": "打开 {url}。等待 seeded form 完成 hydration,把 Text input 填成“seeded-agent”,选择可见选项“Two”,用 Enter 提交,并报告结果 marker 和 seed。最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"page.wait",
|
||||
"inspect.observe",
|
||||
"interact.fill",
|
||||
"interact.select",
|
||||
"interact.press"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{ "label": "generated form hydrated", "type": "matrix.ready", "minCount": 1 },
|
||||
{
|
||||
"label": "Enter was pressed on generated form",
|
||||
"type": "matrix.enter_pressed",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "generated values were submitted",
|
||||
"type": "matrix.submitted",
|
||||
"where": { "data.text": "seeded-agent", "data.choice": "two" },
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "matrix result page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/matrix/generated-result" },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported matrix result", "includes": "MATRIX-OK" },
|
||||
{ "label": "reported selected seed", "includes": "SEED-" }
|
||||
],
|
||||
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "matrix.ready",
|
||||
"timeoutMs": 5000
|
||||
},
|
||||
{ "action": "observe" },
|
||||
{ "action": "fill", "selector": "[name=text]", "value": "seeded-agent" },
|
||||
{ "action": "select", "selector": "[name=choice]", "values": ["two"] },
|
||||
{
|
||||
"action": "press",
|
||||
"key": "Enter",
|
||||
"selector": "button[type=submit]"
|
||||
},
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "page.shown",
|
||||
"where": { "path": "/matrix/generated-result" }
|
||||
},
|
||||
{ "action": "wait", "duration": "200ms" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
# Regression cases
|
||||
|
||||
Each user-reported badcase should live in its own directory with a manifest, synthetic fixture, and
|
||||
README describing the symptom, minimal reproduction, expected behavior, source reference, and fix.
|
||||
Never commit credentials, cookies, HAR files, production HTML, private screenshots, or proprietary
|
||||
assets.
|
||||
|
||||
Create a starting point with:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser scaffold <case-id> --title "..." --source "issue-or-pr"
|
||||
```
|
||||
@@ -0,0 +1,467 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import { loadAgentConfig, runCommandTask } from "./lib/agent-runner.mjs";
|
||||
import { runBskSmokeTask } from "./lib/bsk-runner.mjs";
|
||||
import { buildOperationCoverage } from "./lib/coverage.mjs";
|
||||
import { loadFixtureRegistry } from "./lib/fixture-registry.mjs";
|
||||
import { verifyTask } from "./lib/oracle.mjs";
|
||||
import { resultsDirectory } from "./lib/paths.mjs";
|
||||
import { repositorySummary, validateRepositoryCases } from "./lib/repository-validation.mjs";
|
||||
import { scaffoldCase } from "./lib/scaffold-case.mjs";
|
||||
import { describeSeed, normalizeSeed } from "./lib/seeded-generator.mjs";
|
||||
import { createEvalServer } from "./lib/server.mjs";
|
||||
import { printSummary, summarizeReports } from "./lib/summary.mjs";
|
||||
import { allTasks, getTask, listTasks, renderPrompt } from "./lib/tasks.mjs";
|
||||
|
||||
const booleanOptions = new Set(["help", "json"]);
|
||||
|
||||
function parseArguments(argv) {
|
||||
const normalized = argv[0] === "--" ? argv.slice(1) : argv;
|
||||
let [command, ...rest] = normalized;
|
||||
const options = new Map();
|
||||
const positionals = [];
|
||||
if (command === "--help" || command === "-h") {
|
||||
command = undefined;
|
||||
options.set("help", [true]);
|
||||
}
|
||||
for (let index = 0; index < rest.length; index += 1) {
|
||||
const argument = rest[index];
|
||||
if (!argument.startsWith("--")) {
|
||||
positionals.push(argument);
|
||||
continue;
|
||||
}
|
||||
const equals = argument.indexOf("=");
|
||||
const name = argument.slice(2, equals === -1 ? undefined : equals);
|
||||
let value = equals === -1 ? undefined : argument.slice(equals + 1);
|
||||
if (value === undefined && !booleanOptions.has(name)) {
|
||||
const candidate = rest[index + 1];
|
||||
if (candidate === undefined || candidate.startsWith("--")) {
|
||||
throw new Error(`--${name} requires a value`);
|
||||
}
|
||||
value = candidate;
|
||||
index += 1;
|
||||
}
|
||||
const values = options.get(name) ?? [];
|
||||
values.push(value ?? true);
|
||||
options.set(name, values);
|
||||
}
|
||||
return { command, options, positionals };
|
||||
}
|
||||
|
||||
function option(parsed, name, fallback) {
|
||||
return parsed.options.get(name)?.at(-1) ?? fallback;
|
||||
}
|
||||
|
||||
function optionList(parsed, name) {
|
||||
return (parsed.options.get(name) ?? [])
|
||||
.flatMap((value) => String(value).split(","))
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function numericOption(parsed, name, fallback) {
|
||||
const value = Number(option(parsed, name, fallback));
|
||||
if (!Number.isFinite(value) || value < 0) throw new Error(`--${name} must be non-negative`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function seedOptions(parsed, fallback) {
|
||||
const requested = optionList(parsed, "seed");
|
||||
return requested.length > 0 ? unique(requested.map(normalizeSeed)) : [fallback];
|
||||
}
|
||||
|
||||
function seedOption(parsed, fallback) {
|
||||
const seeds = seedOptions(parsed, fallback);
|
||||
if (seeds.length > 1) throw new Error("this command accepts only one --seed value");
|
||||
return seeds[0];
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function selectCases(cases, parsed, { defaultSuite } = {}) {
|
||||
const requestedIds = unique([...optionList(parsed, "case"), ...optionList(parsed, "task")]);
|
||||
const requestedSuites = unique(optionList(parsed, "suite"));
|
||||
const requestedTags = unique(optionList(parsed, "tag"));
|
||||
const hasSelector =
|
||||
requestedIds.length > 0 || requestedSuites.length > 0 || requestedTags.length > 0;
|
||||
|
||||
if (requestedIds.includes("all") && requestedIds.length > 1) {
|
||||
throw new Error("--case all cannot be combined with named cases");
|
||||
}
|
||||
if (requestedSuites.includes("all") && requestedSuites.length > 1) {
|
||||
throw new Error("--suite all cannot be combined with named suites");
|
||||
}
|
||||
|
||||
const caseIds = requestedIds.includes("all") ? [] : requestedIds;
|
||||
const suites = requestedSuites.includes("all")
|
||||
? []
|
||||
: requestedSuites.length > 0
|
||||
? requestedSuites
|
||||
: !hasSelector && defaultSuite
|
||||
? [defaultSuite]
|
||||
: [];
|
||||
|
||||
const knownIds = new Set(cases.map(({ id }) => id));
|
||||
const missingIds = caseIds.filter((id) => !knownIds.has(id));
|
||||
if (missingIds.length > 0) throw new Error(`unknown case(s): ${missingIds.join(", ")}`);
|
||||
const knownSuites = new Set(cases.map(({ suite }) => suite));
|
||||
const missingSuites = suites.filter((suite) => !knownSuites.has(suite));
|
||||
if (missingSuites.length > 0) throw new Error(`unknown suite(s): ${missingSuites.join(", ")}`);
|
||||
const knownTags = new Set(cases.flatMap(({ tags }) => tags));
|
||||
const missingTags = requestedTags.filter((tag) => !knownTags.has(tag));
|
||||
if (missingTags.length > 0) throw new Error(`unknown tag(s): ${missingTags.join(", ")}`);
|
||||
|
||||
const selected = cases.filter(
|
||||
(testCase) =>
|
||||
(caseIds.length === 0 || caseIds.includes(testCase.id)) &&
|
||||
(suites.length === 0 || suites.includes(testCase.suite)) &&
|
||||
requestedTags.every((tag) => testCase.tags.includes(tag)),
|
||||
);
|
||||
if (selected.length === 0) throw new Error("case selectors matched no cases");
|
||||
return selected;
|
||||
}
|
||||
|
||||
function reportFilename(mode) {
|
||||
return `${new Date().toISOString().replaceAll(/[:.]/g, "-")}-${mode}.json`;
|
||||
}
|
||||
|
||||
async function saveReport(report, outputDirectory) {
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const path = join(outputDirectory, reportFilename(report.mode));
|
||||
await writeFile(path, `${JSON.stringify(report, null, 2)}\n`);
|
||||
return path;
|
||||
}
|
||||
|
||||
async function loadValidatedCorpus() {
|
||||
const [cases, fixtureRegistry] = await Promise.all([allTasks(), loadFixtureRegistry()]);
|
||||
const errors = validateRepositoryCases(cases, fixtureRegistry);
|
||||
if (errors.length > 0) throw new Error(`browser eval validation failed:\n${errors.join("\n")}`);
|
||||
return { cases, fixtureRegistry };
|
||||
}
|
||||
|
||||
function selectionMetadata(cases) {
|
||||
return {
|
||||
cases: cases.map(({ id }) => id),
|
||||
suites: unique(cases.map(({ suite }) => suite)),
|
||||
tags: unique(cases.flatMap(({ tags }) => tags)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
function usage() {
|
||||
console.log(`BrowserSkill local browser evaluation
|
||||
|
||||
Usage:
|
||||
node evals/browser/cli.mjs validate [--json]
|
||||
node evals/browser/cli.mjs list [--suite NAME] [--tag NAME] [--case ID] [--json]
|
||||
node evals/browser/cli.mjs coverage [--suite NAME] [--tag NAME] [--case ID] [--json]
|
||||
node evals/browser/cli.mjs serve [--host 127.0.0.1] [--port 4173]
|
||||
node evals/browser/cli.mjs prompt <case> [--base-url URL] [--run-id ID] [--seed VALUE]
|
||||
node evals/browser/cli.mjs verify <case> --base-url URL --run-id ID [--response TEXT]
|
||||
node evals/browser/cli.mjs smoke [--bsk bsk] [--suite core] [--case ID] [--seed VALUE]
|
||||
node evals/browser/cli.mjs run-agent --config FILE --agent NAME [--suite core] [--case ID]
|
||||
node evals/browser/cli.mjs generate --seed VALUE [--json]
|
||||
node evals/browser/cli.mjs scaffold <case-id> [--suite regression] [--title TEXT]
|
||||
node evals/browser/cli.mjs summarize <report.json> [more reports...]
|
||||
|
||||
Selection:
|
||||
--case ID[,ID] Select case ids; --task is a backward-compatible alias
|
||||
--suite NAME[,NAME] Select suites
|
||||
--tag NAME[,NAME] Require every selected tag
|
||||
--case all Explicitly include every suite (run commands default to core)
|
||||
--seed VALUE[,VALUE] Run one or more reproducible matrix seeds
|
||||
|
||||
run-agent options:
|
||||
--agent NAME[,NAME] One or more entries from the config file
|
||||
--variant NAME Override the report label (does not switch plugin code)
|
||||
--locale en|zh-CN Prompt language (default zh-CN)
|
||||
--repeat N Repeat every selected case (default 1)
|
||||
--timeout MS Per-agent timeout (default 300000)
|
||||
--out DIRECTORY Report/artifact directory
|
||||
`);
|
||||
}
|
||||
|
||||
async function commandValidate(parsed) {
|
||||
const { cases, fixtureRegistry } = await loadValidatedCorpus();
|
||||
const summary = { ok: true, ...repositorySummary(cases, fixtureRegistry) };
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(summary, null, 2));
|
||||
else {
|
||||
console.log(`Validated ${summary.cases} cases and ${summary.fixtureRoutes} fixture routes.`);
|
||||
console.log(
|
||||
`Suites: ${Object.entries(summary.suites)
|
||||
.map(([suite, count]) => `${suite}=${count}`)
|
||||
.join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function commandList(parsed) {
|
||||
const cases = selectCases(await allTasks(), parsed);
|
||||
const tasks = listTasks(cases);
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(tasks, null, 2));
|
||||
else {
|
||||
for (const task of tasks) {
|
||||
console.log(`${task.id}\t${task.suite}\t${task.tags.join(",")}\t${task.title}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function commandCoverage(parsed) {
|
||||
const cases = selectCases(await allTasks(), parsed);
|
||||
const coverage = buildOperationCoverage(cases);
|
||||
if (option(parsed, "json", false)) {
|
||||
console.log(JSON.stringify(coverage, null, 2));
|
||||
} else {
|
||||
console.table(
|
||||
coverage.map(({ operation, agentCases, smokeCases, directSmoke, note }) => ({
|
||||
operation,
|
||||
agent_cases: agentCases.join(", ") || "-",
|
||||
smoke_cases: smokeCases.join(", ") || "-",
|
||||
direct_smoke: directSmoke ? "yes" : "manual",
|
||||
note: note ?? "",
|
||||
})),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function commandServe(parsed) {
|
||||
const { fixtureRegistry } = await loadValidatedCorpus();
|
||||
const server = createEvalServer({
|
||||
host: option(parsed, "host", "127.0.0.1"),
|
||||
port: numericOption(parsed, "port", 4173),
|
||||
fixtureRegistry,
|
||||
});
|
||||
const info = await server.start();
|
||||
console.log(`Browser eval server listening at ${info.baseUrl}`);
|
||||
console.log("Press Ctrl-C to stop.");
|
||||
await new Promise((resolveSignal) => {
|
||||
process.once("SIGINT", resolveSignal);
|
||||
process.once("SIGTERM", resolveSignal);
|
||||
});
|
||||
await server.stop();
|
||||
}
|
||||
|
||||
async function commandPrompt(parsed) {
|
||||
const cases = await allTasks();
|
||||
const task = getTask(parsed.positionals[0], cases);
|
||||
const prompt = renderPrompt(task, {
|
||||
baseUrl: option(parsed, "base-url", "http://127.0.0.1:4173"),
|
||||
runId: option(parsed, "run-id", randomUUID()),
|
||||
locale: option(parsed, "locale", "zh-CN"),
|
||||
seed: seedOption(parsed, task.seed),
|
||||
});
|
||||
console.log(prompt);
|
||||
}
|
||||
|
||||
async function commandVerify(parsed) {
|
||||
const task = getTask(parsed.positionals[0], await allTasks());
|
||||
const baseUrl = option(parsed, "base-url");
|
||||
const runId = option(parsed, "run-id");
|
||||
if (!baseUrl || !runId) throw new Error("verify requires --base-url and --run-id");
|
||||
let responseText = option(parsed, "response", "");
|
||||
const responseFile = option(parsed, "response-file");
|
||||
if (responseFile) responseText = await readFile(resolve(responseFile), "utf8");
|
||||
const stateResponse = await fetch(`${baseUrl}/api/runs/${encodeURIComponent(runId)}`);
|
||||
if (!stateResponse.ok) throw new Error(`oracle request failed with ${stateResponse.status}`);
|
||||
const state = await stateResponse.json();
|
||||
const verification = verifyTask(task, { events: state.events, responseText });
|
||||
console.log(JSON.stringify(verification, null, 2));
|
||||
if (verification.status === "failed") process.exitCode = 1;
|
||||
}
|
||||
|
||||
async function commandRunAgent(parsed) {
|
||||
const configPath = option(parsed, "config");
|
||||
const agentNames = optionList(parsed, "agent");
|
||||
if (!configPath || agentNames.length === 0) {
|
||||
throw new Error("run-agent requires --config and at least one --agent");
|
||||
}
|
||||
const { cases, fixtureRegistry } = await loadValidatedCorpus();
|
||||
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
|
||||
const repeat = numericOption(parsed, "repeat", 1);
|
||||
if (!Number.isInteger(repeat) || repeat < 1) throw new Error("--repeat must be at least 1");
|
||||
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
|
||||
const server = createEvalServer({ fixtureRegistry });
|
||||
const serverInfo = await server.start();
|
||||
const results = [];
|
||||
const startedAt = new Date().toISOString();
|
||||
try {
|
||||
for (const agentName of agentNames) {
|
||||
const { adapter, configDirectory } = await loadAgentConfig(configPath, agentName);
|
||||
for (let iteration = 1; iteration <= repeat; iteration += 1) {
|
||||
for (const task of selected) {
|
||||
for (const seed of seedOptions(parsed, task.seed)) {
|
||||
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
|
||||
console.log(`[run] ${agentName} ${task.id}${seedLabel} (${iteration}/${repeat})`);
|
||||
const result = await runCommandTask({
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
adapter,
|
||||
adapterName: agentName,
|
||||
configDirectory,
|
||||
iteration,
|
||||
locale: option(parsed, "locale", "zh-CN"),
|
||||
seed,
|
||||
timeoutMs: numericOption(parsed, "timeout", 300_000),
|
||||
variantOverride: option(parsed, "variant"),
|
||||
});
|
||||
results.push(result);
|
||||
console.log(` ${result.verification.status} in ${result.execution.durationMs}ms`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
mode: "agent",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
selection: selectionMetadata(selected),
|
||||
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
|
||||
results,
|
||||
};
|
||||
const path = await saveReport(report, outputDirectory);
|
||||
printSummary(summarizeReports([report]));
|
||||
console.log(`Report: ${path}`);
|
||||
if (
|
||||
results.some(
|
||||
(result) =>
|
||||
result.verification.status === "failed" ||
|
||||
result.execution.timedOut ||
|
||||
result.execution.exitCode !== 0,
|
||||
)
|
||||
) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function commandSmoke(parsed) {
|
||||
const { cases, fixtureRegistry } = await loadValidatedCorpus();
|
||||
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
|
||||
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
|
||||
const server = createEvalServer({ fixtureRegistry });
|
||||
const serverInfo = await server.start();
|
||||
const results = [];
|
||||
const startedAt = new Date().toISOString();
|
||||
try {
|
||||
for (const task of selected) {
|
||||
for (const seed of seedOptions(parsed, task.seed)) {
|
||||
const runId = `bsk-${task.id}-${randomUUID().slice(0, 8)}`;
|
||||
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
|
||||
console.log(`[smoke] ${task.id}${seedLabel}`);
|
||||
const result = await runBskSmokeTask({
|
||||
bskCommand: option(parsed, "bsk", "bsk"),
|
||||
outputDirectory,
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
runId,
|
||||
seed,
|
||||
timeoutMs: numericOption(parsed, "timeout", 60_000),
|
||||
});
|
||||
results.push(result);
|
||||
console.log(
|
||||
` ${result.verification.status}${result.executionError ? `: ${result.executionError}` : ""}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
mode: "bsk-smoke",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
selection: selectionMetadata(selected),
|
||||
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
|
||||
results,
|
||||
};
|
||||
const path = await saveReport(report, outputDirectory);
|
||||
printSummary(summarizeReports([report]));
|
||||
console.log(`Report: ${path}`);
|
||||
if (results.some((result) => result.executionError || result.verification.status === "failed")) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function commandGenerate(parsed) {
|
||||
if (!parsed.options.has("seed")) throw new Error("generate requires --seed");
|
||||
const dimensions = unique(optionList(parsed, "seed").map(normalizeSeed)).map(describeSeed);
|
||||
const output = dimensions.length === 1 ? dimensions[0] : dimensions;
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(output, null, 2));
|
||||
else console.table(dimensions);
|
||||
}
|
||||
|
||||
async function commandScaffold(parsed) {
|
||||
const id = parsed.positionals[0];
|
||||
if (!id) throw new Error("scaffold requires a case id");
|
||||
const result = await scaffoldCase({
|
||||
id,
|
||||
title: option(parsed, "title", id),
|
||||
suite: option(parsed, "suite", "regression"),
|
||||
source: option(parsed, "source"),
|
||||
});
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(result, null, 2));
|
||||
else {
|
||||
console.log(`Created ${result.directory}`);
|
||||
console.log(
|
||||
"Edit the fixture, prompt, workflow, and assertions, then run eval:browser validate.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function commandSummarize(parsed) {
|
||||
const paths = [...parsed.positionals, ...optionList(parsed, "input")];
|
||||
if (paths.length === 0) throw new Error("summarize requires at least one report path");
|
||||
const reports = await Promise.all(
|
||||
paths.map(async (path) => JSON.parse(await readFile(resolve(path), "utf8"))),
|
||||
);
|
||||
const rows = summarizeReports(reports);
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(rows, null, 2));
|
||||
else printSummary(rows);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const parsed = parseArguments(process.argv.slice(2));
|
||||
if (!parsed.command || option(parsed, "help", false)) return usage();
|
||||
switch (parsed.command) {
|
||||
case "validate":
|
||||
return commandValidate(parsed);
|
||||
case "list":
|
||||
return commandList(parsed);
|
||||
case "coverage":
|
||||
return commandCoverage(parsed);
|
||||
case "serve":
|
||||
return commandServe(parsed);
|
||||
case "prompt":
|
||||
return commandPrompt(parsed);
|
||||
case "verify":
|
||||
return commandVerify(parsed);
|
||||
case "run-agent":
|
||||
return commandRunAgent(parsed);
|
||||
case "smoke":
|
||||
return commandSmoke(parsed);
|
||||
case "generate":
|
||||
return commandGenerate(parsed);
|
||||
case "scaffold":
|
||||
return commandScaffold(parsed);
|
||||
case "summarize":
|
||||
return commandSummarize(parsed);
|
||||
default:
|
||||
throw new Error(`unknown command ${JSON.stringify(parsed.command)}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`browser eval: ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "diagnostics",
|
||||
routes: ["/diagnostics"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Diagnostics Lab",
|
||||
body: `<section class="card"><h1>Diagnostics Lab</h1><p>Inspect console and network buffers.</p><p id="network-status" class="muted">Ping pending</p></section>`,
|
||||
script: `
|
||||
console.log("Browser eval console marker: CONSOLE-61");
|
||||
fetch("/api/ping?run=" + encodeURIComponent(browserEval.runId))
|
||||
.then((response) => response.json())
|
||||
.then(({ token }) => { document.querySelector("#network-status").textContent = "Ping completed: " + token; });
|
||||
`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,25 @@
|
||||
import { escapeHtml, page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "form",
|
||||
routes: ["/form", "/result"],
|
||||
render({ pathname, runId, query, record }) {
|
||||
if (pathname === "/form") {
|
||||
return page({
|
||||
title: "Browser Eval Form",
|
||||
body: `<section class="card"><h1>Web form</h1><form action="/result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><label>Text input<input id="text-input" name="text" autocomplete="off"></label><label>Textarea<textarea id="notes" name="notes"></textarea></label><label>Dropdown<select id="choice" name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label><button id="submit" type="submit">Submit</button></form></section>`,
|
||||
script: `document.querySelector("#submit").addEventListener("keydown", (event) => { if (event.key === "Enter") browserEval.send("form.enter_pressed"); });`,
|
||||
});
|
||||
}
|
||||
const values = {
|
||||
text: query.get("text") ?? "",
|
||||
notes: query.get("notes") ?? "",
|
||||
choice: query.get("choice") ?? "",
|
||||
};
|
||||
record("form.submitted", values, { path: pathname });
|
||||
return page({
|
||||
title: "Form Result",
|
||||
body: `<section class="card"><h1>Received!</h1><p id="submitted-values" class="marker">${escapeHtml(values.text)} | ${escapeHtml(values.notes)} | ${escapeHtml(values.choice)}</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
import { escapeHtml, page } from "../../lib/fixtures.mjs";
|
||||
import { describeSeed } from "../../lib/seeded-generator.mjs";
|
||||
|
||||
function nest(content, depth) {
|
||||
let nested = content;
|
||||
for (let index = 0; index < depth; index += 1) {
|
||||
nested = `<div data-depth="${index + 1}">${nested}</div>`;
|
||||
}
|
||||
return nested;
|
||||
}
|
||||
|
||||
function textField(dimensions) {
|
||||
const id = `matrix-text-${dimensions.idSuffix}`;
|
||||
const input = `<input id="${id}" name="text" autocomplete="off">`;
|
||||
if (dimensions.labelMode === "wrapped") return `<label>Text input${input}</label>`;
|
||||
if (dimensions.labelMode === "for") return `<label for="${id}">Text input</label>${input}`;
|
||||
return `<span id="${id}-label">Text input</span><input id="${id}" name="text" aria-labelledby="${id}-label" autocomplete="off">`;
|
||||
}
|
||||
|
||||
function formMarkup(dimensions, runId) {
|
||||
const text = textField(dimensions);
|
||||
const select = `<label>Dropdown<select name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label>`;
|
||||
const fields = dimensions.fieldOrder === "text-first" ? `${text}${select}` : `${select}${text}`;
|
||||
const decoy =
|
||||
dimensions.decoy === "disabled-input"
|
||||
? `<label>Text input<input disabled value="decoy"></label>`
|
||||
: dimensions.decoy === "duplicate-text"
|
||||
? `<p class="muted">Text input is intentionally repeated in helper copy.</p>`
|
||||
: "";
|
||||
return nest(
|
||||
`<form action="/matrix/generated-result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><input type="hidden" name="seed" value="${dimensions.seed}">${decoy}${fields}<button type="submit">Submit matrix form</button></form>`,
|
||||
dimensions.nestingDepth,
|
||||
);
|
||||
}
|
||||
|
||||
export default {
|
||||
id: "generated-form",
|
||||
routes: ["/matrix/generated-form", "/matrix/generated-result"],
|
||||
render({ pathname, runId, query, record }) {
|
||||
const dimensions = describeSeed(query.get("seed") ?? 1);
|
||||
if (pathname === "/matrix/generated-form") {
|
||||
const markup = JSON.stringify(formMarkup(dimensions, runId));
|
||||
const dimensionsJson = JSON.stringify(dimensions);
|
||||
return page({
|
||||
title: "Seeded Matrix Form",
|
||||
body: `<section class="card"><h1>Seeded Matrix Form</h1><p class="marker">SEED-${dimensions.seed}</p><div id="matrix-root">Hydrating…</div></section>`,
|
||||
script: `
|
||||
const hydrate = () => {
|
||||
const root = document.querySelector("#matrix-root");
|
||||
root.innerHTML = ${markup};
|
||||
root.querySelector("button[type=submit]").addEventListener("keydown", (event) => {
|
||||
if (event.key === "Enter") browserEval.send("matrix.enter_pressed");
|
||||
});
|
||||
browserEval.send("matrix.ready", ${dimensionsJson});
|
||||
};
|
||||
setTimeout(hydrate, ${dimensions.hydrationDelayMs});
|
||||
`,
|
||||
});
|
||||
}
|
||||
const values = {
|
||||
seed: dimensions.seed,
|
||||
text: query.get("text") ?? "",
|
||||
choice: query.get("choice") ?? "",
|
||||
};
|
||||
record("matrix.submitted", values, { path: pathname });
|
||||
return page({
|
||||
title: "Matrix Result",
|
||||
body: `<section class="card"><h1>Matrix Result</h1><p class="marker">MATRIX-OK</p><p>${escapeHtml(values.text)} | ${escapeHtml(values.choice)} | seed ${values.seed}</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "hover-inspect",
|
||||
routes: ["/hover-inspect"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Hover and Inspect",
|
||||
body: `<section class="card"><h1>Hover and Inspect</h1><div id="account-menu" tabindex="0"><button id="account-button" type="button">Account</button><div id="hover-panel"><button id="profile-link" type="button">Profile beta</button></div></div><p id="hover-result" class="marker" hidden>HOVER-17</p><article id="inspect-card" class="card" data-build="2026.08" data-marker="INSPECT-27"><h2>Inspection card</h2><p>Local HTML marker: <code>INSPECT-27</code></p></article></section>`,
|
||||
script: `
|
||||
const menu = document.querySelector("#account-menu");
|
||||
menu.addEventListener("mouseenter", () => { menu.classList.add("open"); browserEval.send("hover.menu_opened"); });
|
||||
document.querySelector("#profile-link").addEventListener("click", () => { document.querySelector("#hover-result").hidden = false; browserEval.send("hover.profile_clicked"); });
|
||||
`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "fixture-index",
|
||||
routes: ["/"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Browser Eval Fixtures",
|
||||
body: `<section class="card"><h1>Browser Eval Fixtures</h1><p class="muted">Deterministic local pages for agent-neutral browser evaluation.</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
import { page, withRun } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "navigation",
|
||||
routes: ["/navigation/start", "/navigation/detail"],
|
||||
render({ pathname, runId }) {
|
||||
if (pathname === "/navigation/start") {
|
||||
return page({
|
||||
title: "Navigation Start",
|
||||
body: `<section class="card"><h1>Navigation Start</h1><p>Use the link to open the deterministic detail page.</p><a id="details-link" class="button" href="${withRun("/navigation/detail", runId)}">Details</a></section>`,
|
||||
script: `document.querySelector("#details-link").addEventListener("click", () => browserEval.send("navigation.detail.clicked"));`,
|
||||
});
|
||||
}
|
||||
return page({
|
||||
title: "Navigation Detail",
|
||||
body: `<section class="card"><h1>Navigation Detail</h1><p>Stable detail code:</p><p id="navigation-code" class="marker">NAV-42</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "responsive",
|
||||
routes: ["/responsive"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Responsive Lab",
|
||||
body: `<section class="card"><h1>Responsive Lab</h1><p id="viewport-marker" class="marker">VIEWPORT-84</p><p id="viewport-value"></p></section>`,
|
||||
script: `
|
||||
const sample = () => {
|
||||
const data = { width: innerWidth, height: innerHeight, touchPoints: navigator.maxTouchPoints, userAgent: navigator.userAgent };
|
||||
document.querySelector("#viewport-value").textContent = data.width + "x" + data.height;
|
||||
browserEval.send("viewport.sample", data);
|
||||
};
|
||||
window.addEventListener("resize", sample);
|
||||
sample();
|
||||
`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
import { escapeHtml, page, withRun } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "tabs",
|
||||
routes: ["/tabs", "/tabs/child"],
|
||||
render({ pathname, runId }) {
|
||||
if (pathname === "/tabs") {
|
||||
return page({
|
||||
title: "Tab Lab",
|
||||
body: `<section class="card"><h1>Tab Lab</h1><p>Keep this original tab open while visiting the child page in a new tab.</p><p class="muted">Child URL: <code>${escapeHtml(withRun("/tabs/child", runId))}</code></p></section>`,
|
||||
});
|
||||
}
|
||||
return page({
|
||||
title: "Child Tab",
|
||||
body: `<section class="card"><h1>Child Tab</h1><p id="tab-token" class="marker">TAB-29</p><button id="mark-child" type="button">Mark child visited</button><p id="child-status" hidden>Marked</p></section>`,
|
||||
script: `document.querySelector("#mark-child").addEventListener("click", () => { document.querySelector("#child-status").hidden = false; browserEval.send("tabs.child_marked"); });`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import { verifyTask } from "./oracle.mjs";
|
||||
import { countPattern, runProcess, substituteArgs } from "./process.mjs";
|
||||
import { renderPrompt } from "./tasks.mjs";
|
||||
|
||||
function safeRunPart(value) {
|
||||
return value.replaceAll(/[^A-Za-z0-9._-]/g, "-").slice(0, 48);
|
||||
}
|
||||
|
||||
export async function loadAgentConfig(configPath, agentName) {
|
||||
const absolute = resolve(configPath);
|
||||
const config = JSON.parse(await readFile(absolute, "utf8"));
|
||||
const adapter = config.agents?.[agentName];
|
||||
if (!adapter) {
|
||||
const names = Object.keys(config.agents ?? {}).join(", ");
|
||||
throw new Error(`unknown agent ${JSON.stringify(agentName)}; configured agents: ${names}`);
|
||||
}
|
||||
if (typeof adapter.command !== "string" || !Array.isArray(adapter.args)) {
|
||||
throw new Error(`agent ${JSON.stringify(agentName)} must define command and args`);
|
||||
}
|
||||
return { adapter, configDirectory: dirname(absolute) };
|
||||
}
|
||||
|
||||
export async function runCommandTask({
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
adapter,
|
||||
adapterName,
|
||||
configDirectory,
|
||||
iteration,
|
||||
locale,
|
||||
seed = task.seed,
|
||||
timeoutMs,
|
||||
variantOverride,
|
||||
}) {
|
||||
const variant = variantOverride ?? adapter.variant ?? "unspecified";
|
||||
const runId = [
|
||||
safeRunPart(adapterName),
|
||||
safeRunPart(variant),
|
||||
safeRunPart(task.id),
|
||||
iteration,
|
||||
randomUUID().slice(0, 8),
|
||||
].join("-");
|
||||
server.reset(runId);
|
||||
const prompt = renderPrompt(task, { baseUrl: serverInfo.baseUrl, runId, locale, seed });
|
||||
const values = {
|
||||
prompt,
|
||||
caseId: task.id,
|
||||
taskId: task.id,
|
||||
runId,
|
||||
baseUrl: serverInfo.baseUrl,
|
||||
seed: seed ?? "",
|
||||
variant,
|
||||
};
|
||||
const args = substituteArgs(adapter.args, values);
|
||||
const execution = await runProcess(adapter.command, args, {
|
||||
cwd: adapter.cwd ? resolve(configDirectory, adapter.cwd) : configDirectory,
|
||||
env: adapter.env,
|
||||
timeoutMs: adapter.timeoutMs ?? timeoutMs,
|
||||
});
|
||||
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
|
||||
const responseText = execution.stdout;
|
||||
const combinedOutput = `${execution.stdout}\n${execution.stderr}`;
|
||||
const verification = verifyTask(task, {
|
||||
events: server.snapshot(runId).events,
|
||||
responseText,
|
||||
});
|
||||
const defaultErrorPattern = "(?:\\bError:|timed out|unfinished command|session already has)";
|
||||
return {
|
||||
runId,
|
||||
caseId: task.id,
|
||||
taskId: task.id,
|
||||
suite: task.suite,
|
||||
tags: task.tags,
|
||||
seed,
|
||||
iteration,
|
||||
adapter: adapterName,
|
||||
variant,
|
||||
adapterMetadata: adapter.metadata ?? {},
|
||||
locale,
|
||||
prompt,
|
||||
execution,
|
||||
metrics: {
|
||||
errorCount: countPattern(
|
||||
combinedOutput,
|
||||
adapter.metrics?.errorPattern ?? defaultErrorPattern,
|
||||
),
|
||||
toolCallCount: countPattern(combinedOutput, adapter.metrics?.toolCallPattern),
|
||||
responseChars: responseText.length,
|
||||
},
|
||||
eventCount: server.snapshot(runId).events.length,
|
||||
verification,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
import { mkdir, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { verifyTask } from "./oracle.mjs";
|
||||
import { runProcess } from "./process.mjs";
|
||||
import { taskUrl } from "./tasks.mjs";
|
||||
|
||||
function parseJsonOutput(step) {
|
||||
try {
|
||||
return JSON.parse(step.stdout);
|
||||
} catch (error) {
|
||||
throw new Error(`could not parse bsk JSON output: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function renderStepError(step) {
|
||||
const raw = step.stderr.trim() || step.stdout.trim();
|
||||
if (!raw) return `exit ${step.exitCode}`;
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed.hint ? `${parsed.message} (hint: ${parsed.hint})` : parsed.message;
|
||||
} catch {
|
||||
return raw;
|
||||
}
|
||||
}
|
||||
|
||||
async function hasNonemptyFile(path) {
|
||||
try {
|
||||
return (await stat(path)).size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readPath(value, path) {
|
||||
return path.split(".").reduce((current, part) => current?.[part], value);
|
||||
}
|
||||
|
||||
function resolveValue(value, variables) {
|
||||
if (typeof value !== "string") return value;
|
||||
const exact = value.match(/^\{([^{}]+)}$/);
|
||||
if (exact) {
|
||||
const resolved = readPath(variables, exact[1]);
|
||||
if (resolved === undefined) throw new Error(`unknown workflow variable ${exact[1]}`);
|
||||
return resolved;
|
||||
}
|
||||
return value.replaceAll(/\{([^{}]+)\}/g, (_match, path) => {
|
||||
const resolved = readPath(variables, path);
|
||||
if (resolved === undefined) throw new Error(`unknown workflow variable ${path}`);
|
||||
return String(resolved);
|
||||
});
|
||||
}
|
||||
|
||||
function targetArgs(step, variables) {
|
||||
if (step.selector) return ["--selector", String(resolveValue(step.selector, variables))];
|
||||
if (step.ref) return ["--ref", String(resolveValue(step.ref, variables))];
|
||||
return [];
|
||||
}
|
||||
|
||||
function tabArgs(step, variables) {
|
||||
return step.tabId === undefined ? [] : ["--tab-id", String(resolveValue(step.tabId, variables))];
|
||||
}
|
||||
|
||||
function matchesWhere(value, where = {}) {
|
||||
return Object.entries(where).every(([path, expected]) => readPath(value, path) === expected);
|
||||
}
|
||||
|
||||
export async function runBskSmokeTask({
|
||||
bskCommand,
|
||||
outputDirectory,
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
runId,
|
||||
seed = task.seed,
|
||||
timeoutMs = 60_000,
|
||||
}) {
|
||||
const steps = [];
|
||||
const evidence = {};
|
||||
const variables = {
|
||||
baseUrl: serverInfo.baseUrl,
|
||||
runId,
|
||||
seed: seed ?? "",
|
||||
url: taskUrl(task, serverInfo.baseUrl, runId, seed),
|
||||
};
|
||||
let sessionId;
|
||||
let executionError;
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
|
||||
async function bsk(args, { timeout = timeoutMs } = {}) {
|
||||
const execution = await runProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout });
|
||||
steps.push(execution);
|
||||
if (execution.exitCode !== 0 || execution.timedOut || execution.error) {
|
||||
throw new Error(
|
||||
`bsk ${args.join(" ")} failed: ${execution.error ?? renderStepError(execution)}`,
|
||||
);
|
||||
}
|
||||
return parseJsonOutput(execution);
|
||||
}
|
||||
|
||||
async function waitForSiteEvent(step) {
|
||||
const timeout = step.timeoutMs ?? 3_000;
|
||||
const deadline = Date.now() + timeout;
|
||||
while (Date.now() < deadline) {
|
||||
if (
|
||||
server
|
||||
.snapshot(runId)
|
||||
.events.some(
|
||||
(event) => event.type === step.type && matchesWhere(event.data ?? {}, step.where ?? {}),
|
||||
)
|
||||
) {
|
||||
return { observed: true };
|
||||
}
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 50));
|
||||
}
|
||||
throw new Error(`local fixture did not report ${step.type} within ${timeout}ms`);
|
||||
}
|
||||
|
||||
async function executeWorkflowStep(step) {
|
||||
const session = ["--session", sessionId];
|
||||
let result;
|
||||
switch (step.action) {
|
||||
case "navigate":
|
||||
result = await bsk([
|
||||
"navigate",
|
||||
String(resolveValue(step.url ?? "{url}", variables)),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "back":
|
||||
case "forward":
|
||||
result = await bsk([`navigate-${step.action}`, ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
case "reload":
|
||||
result = await bsk(["reload", ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
case "wait":
|
||||
result = await bsk(["wait-ms", String(resolveValue(step.duration, variables))]);
|
||||
break;
|
||||
case "wait-site-event":
|
||||
result = await waitForSiteEvent(step);
|
||||
break;
|
||||
case "observe":
|
||||
case "snapshot":
|
||||
case "html": {
|
||||
const command = step.action === "html" ? "get-html" : step.action;
|
||||
result = await bsk([command, ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
}
|
||||
case "screenshot": {
|
||||
const name = resolveValue(step.artifactName ?? "screenshot.png", variables);
|
||||
const path = join(outputDirectory, `${runId}-${name}`);
|
||||
result = await bsk(["screenshot", ...session, ...tabArgs(step, variables), "--out", path]);
|
||||
if (step.evidence) evidence[step.evidence] = await hasNonemptyFile(path);
|
||||
break;
|
||||
}
|
||||
case "console":
|
||||
case "network":
|
||||
result = await bsk([step.action, ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
case "click":
|
||||
case "hover":
|
||||
result = await bsk([
|
||||
step.action,
|
||||
...targetArgs(step, variables),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "fill":
|
||||
result = await bsk([
|
||||
"fill",
|
||||
...targetArgs(step, variables),
|
||||
"--value",
|
||||
String(resolveValue(step.value, variables)),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "select":
|
||||
result = await bsk([
|
||||
"select",
|
||||
...targetArgs(step, variables),
|
||||
...step.values.flatMap((value) => ["--value", String(resolveValue(value, variables))]),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "press":
|
||||
result = await bsk([
|
||||
"press",
|
||||
String(resolveValue(step.key, variables)),
|
||||
...targetArgs(step, variables),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "tab-list":
|
||||
result = await bsk(["tab", "list", ...session]);
|
||||
break;
|
||||
case "tab-create":
|
||||
result = await bsk([
|
||||
"tab",
|
||||
"create",
|
||||
...session,
|
||||
...(step.url ? ["--url", String(resolveValue(step.url, variables))] : []),
|
||||
]);
|
||||
break;
|
||||
case "tab-select":
|
||||
case "tab-close":
|
||||
result = await bsk([
|
||||
"tab",
|
||||
step.action.slice(4),
|
||||
String(resolveValue(step.tabId, variables)),
|
||||
...session,
|
||||
]);
|
||||
break;
|
||||
case "borrow":
|
||||
case "return":
|
||||
result = await bsk([
|
||||
"tab",
|
||||
step.action,
|
||||
String(resolveValue(step.tabId, variables)),
|
||||
...session,
|
||||
]);
|
||||
break;
|
||||
case "resize":
|
||||
result = await bsk([
|
||||
"window",
|
||||
"resize",
|
||||
...session,
|
||||
"--width",
|
||||
String(step.width),
|
||||
"--height",
|
||||
String(step.height),
|
||||
]);
|
||||
break;
|
||||
case "emulate":
|
||||
result = await bsk([
|
||||
"emulate",
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
"--device",
|
||||
String(resolveValue(step.device, variables)),
|
||||
]);
|
||||
break;
|
||||
case "request-help":
|
||||
result = await bsk([
|
||||
"request-help",
|
||||
...session,
|
||||
"--prompt",
|
||||
String(resolveValue(step.prompt, variables)),
|
||||
...(step.title ? ["--title", String(resolveValue(step.title, variables))] : []),
|
||||
...(step.timeout ? ["--timeout", String(step.timeout)] : []),
|
||||
]);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`unsupported smoke workflow action ${step.action}`);
|
||||
}
|
||||
if (step.saveAs) variables[step.saveAs] = result;
|
||||
if (step.evidence && step.action !== "screenshot") evidence[step.evidence] = true;
|
||||
}
|
||||
|
||||
server.reset(runId);
|
||||
try {
|
||||
if (task.smokeSteps.length === 0) throw new Error(`case ${task.id} has no smoke workflow`);
|
||||
const started = await bsk(["session", "start", "--no-focus"], { timeout: 90_000 });
|
||||
sessionId = started.session_id;
|
||||
variables.sessionId = sessionId;
|
||||
await bsk(["session", "list"]);
|
||||
for (const step of task.smokeSteps) await executeWorkflowStep(step);
|
||||
} catch (error) {
|
||||
executionError = error instanceof Error ? error.message : String(error);
|
||||
} finally {
|
||||
if (sessionId) {
|
||||
try {
|
||||
const stopped = await bsk(["session", "stop", sessionId], { timeout: 90_000 });
|
||||
evidence.sessionStopped =
|
||||
stopped.stopped?.includes(sessionId) === true && (stopped.failed?.length ?? 0) === 0;
|
||||
if (!evidence.sessionStopped) {
|
||||
executionError ??= `bsk did not confirm that session ${sessionId} stopped`;
|
||||
}
|
||||
} catch (error) {
|
||||
evidence.sessionStopped = false;
|
||||
executionError ??= error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 350));
|
||||
const responseText = steps.map(({ stdout }) => stdout).join("\n");
|
||||
const events = server.snapshot(runId).events;
|
||||
return {
|
||||
runId,
|
||||
caseId: task.id,
|
||||
taskId: task.id,
|
||||
suite: task.suite,
|
||||
tags: task.tags,
|
||||
seed,
|
||||
adapter: "bsk-direct",
|
||||
variant: "cli-direct",
|
||||
executionError,
|
||||
steps,
|
||||
evidence,
|
||||
eventCount: events.length,
|
||||
verification: verifyTask(task, { events, responseText, adapterEvidence: evidence }),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import { discoverFiles } from "./discovery.mjs";
|
||||
import { OPERATION_NAMES, WORKFLOW_ACTIONS } from "./operations.mjs";
|
||||
import { casesDirectory as defaultCasesDirectory } from "./paths.mjs";
|
||||
|
||||
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
const EVIDENCE_PATTERN = /^[A-Za-z][A-Za-z0-9]*$/;
|
||||
const TOP_LEVEL_KEYS = new Set([
|
||||
"$schema",
|
||||
"schemaVersion",
|
||||
"id",
|
||||
"title",
|
||||
"order",
|
||||
"suite",
|
||||
"tags",
|
||||
"seed",
|
||||
"source",
|
||||
"fixture",
|
||||
"prompts",
|
||||
"coverage",
|
||||
"assertions",
|
||||
"smoke",
|
||||
]);
|
||||
|
||||
function isObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function checkString(errors, value, path) {
|
||||
if (typeof value !== "string" || value.trim() === "") errors.push(`${path} must be a string`);
|
||||
}
|
||||
|
||||
function validateAssertion(errors, assertion, path, kind) {
|
||||
if (!isObject(assertion)) {
|
||||
errors.push(`${path} must be an object`);
|
||||
return;
|
||||
}
|
||||
checkString(errors, assertion.label, `${path}.label`);
|
||||
if (kind === "site") {
|
||||
checkString(errors, assertion.type, `${path}.type`);
|
||||
if (!Number.isInteger(assertion.minCount) || assertion.minCount < 1) {
|
||||
errors.push(`${path}.minCount must be a positive integer`);
|
||||
}
|
||||
if (assertion.where !== undefined && !isObject(assertion.where)) {
|
||||
errors.push(`${path}.where must be an object`);
|
||||
}
|
||||
} else if (kind === "response") {
|
||||
checkString(errors, assertion.includes, `${path}.includes`);
|
||||
} else {
|
||||
checkString(errors, assertion.key, `${path}.key`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateWorkflowStep(errors, step, path) {
|
||||
if (!isObject(step)) {
|
||||
errors.push(`${path} must be an object`);
|
||||
return;
|
||||
}
|
||||
if (!WORKFLOW_ACTIONS.has(step.action)) {
|
||||
errors.push(`${path}.action is unsupported: ${JSON.stringify(step.action)}`);
|
||||
}
|
||||
if (step.saveAs !== undefined && !ID_PATTERN.test(step.saveAs)) {
|
||||
errors.push(`${path}.saveAs must use lowercase kebab-case`);
|
||||
}
|
||||
if (step.evidence !== undefined && !EVIDENCE_PATTERN.test(step.evidence)) {
|
||||
errors.push(`${path}.evidence must be an identifier`);
|
||||
}
|
||||
if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) {
|
||||
errors.push(`${path} requires selector or ref`);
|
||||
}
|
||||
if (step.action === "fill") checkString(errors, step.value, `${path}.value`);
|
||||
if (step.action === "select") {
|
||||
if (!Array.isArray(step.values) || step.values.length === 0) {
|
||||
errors.push(`${path}.values must be a non-empty array`);
|
||||
} else if (step.values.some((value) => typeof value !== "string")) {
|
||||
errors.push(`${path}.values must contain only strings`);
|
||||
}
|
||||
}
|
||||
if (step.action === "press") checkString(errors, step.key, `${path}.key`);
|
||||
if (step.action === "wait") checkString(errors, step.duration, `${path}.duration`);
|
||||
if (step.action === "wait-site-event") checkString(errors, step.type, `${path}.type`);
|
||||
if (["tab-select", "tab-close", "borrow", "return"].includes(step.action)) {
|
||||
checkString(errors, step.tabId, `${path}.tabId`);
|
||||
}
|
||||
if (step.action === "resize") {
|
||||
if (
|
||||
!Number.isInteger(step.width) ||
|
||||
step.width < 1 ||
|
||||
!Number.isInteger(step.height) ||
|
||||
step.height < 1
|
||||
) {
|
||||
errors.push(`${path}.width and height must be positive integers`);
|
||||
}
|
||||
}
|
||||
if (step.action === "emulate") checkString(errors, step.device, `${path}.device`);
|
||||
if (step.action === "request-help") checkString(errors, step.prompt, `${path}.prompt`);
|
||||
if (step.timeoutMs !== undefined && (!Number.isInteger(step.timeoutMs) || step.timeoutMs < 1)) {
|
||||
errors.push(`${path}.timeoutMs must be a positive integer`);
|
||||
}
|
||||
}
|
||||
|
||||
export function validateCaseManifest(manifest, { source = "case manifest" } = {}) {
|
||||
const errors = [];
|
||||
if (!isObject(manifest)) return [`${source} must contain a JSON object`];
|
||||
for (const key of Object.keys(manifest)) {
|
||||
if (!TOP_LEVEL_KEYS.has(key)) errors.push(`unknown top-level property ${key}`);
|
||||
}
|
||||
if (manifest.schemaVersion !== 1) errors.push("schemaVersion must equal 1");
|
||||
if (!ID_PATTERN.test(manifest.id ?? "")) errors.push("id must use lowercase kebab-case");
|
||||
checkString(errors, manifest.title, "title");
|
||||
if (manifest.order !== undefined && (!Number.isInteger(manifest.order) || manifest.order < 0)) {
|
||||
errors.push("order must be a non-negative integer");
|
||||
}
|
||||
if (!ID_PATTERN.test(manifest.suite ?? "")) errors.push("suite must use lowercase kebab-case");
|
||||
if (!Array.isArray(manifest.tags) || manifest.tags.some((tag) => !ID_PATTERN.test(tag))) {
|
||||
errors.push("tags must be an array of lowercase kebab-case strings");
|
||||
} else if (new Set(manifest.tags).size !== manifest.tags.length) {
|
||||
errors.push("tags must not contain duplicates");
|
||||
}
|
||||
if (!isObject(manifest.fixture)) errors.push("fixture must be an object");
|
||||
else if (
|
||||
typeof manifest.fixture.startPath !== "string" ||
|
||||
!manifest.fixture.startPath.startsWith("/")
|
||||
) {
|
||||
errors.push("fixture.startPath must be an absolute URL path");
|
||||
}
|
||||
if (!isObject(manifest.prompts)) errors.push("prompts must be an object");
|
||||
else {
|
||||
checkString(errors, manifest.prompts.en, "prompts.en");
|
||||
checkString(errors, manifest.prompts["zh-CN"], "prompts.zh-CN");
|
||||
}
|
||||
if (!Array.isArray(manifest.coverage)) errors.push("coverage must be an array");
|
||||
else {
|
||||
for (const operation of manifest.coverage) {
|
||||
if (!OPERATION_NAMES.has(operation))
|
||||
errors.push(`coverage contains unknown operation ${operation}`);
|
||||
}
|
||||
if (new Set(manifest.coverage).size !== manifest.coverage.length) {
|
||||
errors.push("coverage must not contain duplicates");
|
||||
}
|
||||
}
|
||||
if (!isObject(manifest.assertions)) errors.push("assertions must be an object");
|
||||
else {
|
||||
for (const kind of ["site", "response", "adapter"]) {
|
||||
if (!Array.isArray(manifest.assertions[kind])) {
|
||||
errors.push(`assertions.${kind} must be an array`);
|
||||
} else {
|
||||
manifest.assertions[kind].forEach((assertion, index) =>
|
||||
validateAssertion(errors, assertion, `assertions.${kind}[${index}]`, kind),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!isObject(manifest.smoke) || !Array.isArray(manifest.smoke.steps)) {
|
||||
errors.push("smoke.steps must be an array");
|
||||
} else {
|
||||
manifest.smoke.steps.forEach((step, index) =>
|
||||
validateWorkflowStep(errors, step, `smoke.steps[${index}]`),
|
||||
);
|
||||
}
|
||||
if (manifest.seed !== undefined && !Number.isSafeInteger(manifest.seed)) {
|
||||
errors.push("seed must be a safe integer");
|
||||
}
|
||||
if (manifest.source !== undefined && !isObject(manifest.source)) {
|
||||
errors.push("source must be an object");
|
||||
}
|
||||
return errors.map((error) => `${source}: ${error}`);
|
||||
}
|
||||
|
||||
function normalizeCase(manifest, sourceFile) {
|
||||
return Object.freeze({
|
||||
...manifest,
|
||||
startPath: manifest.fixture.startPath,
|
||||
siteAssertions: manifest.assertions.site,
|
||||
responseAssertions: manifest.assertions.response,
|
||||
adapterAssertions: manifest.assertions.adapter,
|
||||
smokeSteps: manifest.smoke.steps,
|
||||
sourceFile,
|
||||
});
|
||||
}
|
||||
|
||||
export async function loadCaseManifests({ casesDirectory = defaultCasesDirectory } = {}) {
|
||||
const files = await discoverFiles(casesDirectory, (path) => path.endsWith(".case.json"));
|
||||
const cases = [];
|
||||
const ids = new Map();
|
||||
const errors = [];
|
||||
for (const file of files) {
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(file, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${file}: invalid JSON: ${error.message}`);
|
||||
continue;
|
||||
}
|
||||
const manifestErrors = validateCaseManifest(manifest, { source: file });
|
||||
errors.push(...manifestErrors);
|
||||
if (typeof manifest?.id === "string") {
|
||||
if (ids.has(manifest.id)) {
|
||||
errors.push(`${file}: duplicate id also declared by ${ids.get(manifest.id)}`);
|
||||
} else {
|
||||
ids.set(manifest.id, file);
|
||||
}
|
||||
}
|
||||
if (manifestErrors.length === 0) cases.push(normalizeCase(manifest, file));
|
||||
}
|
||||
if (files.length === 0) errors.push(`${casesDirectory}: no .case.json files found`);
|
||||
if (errors.length > 0) throw new Error(`case validation failed:\n${errors.join("\n")}`);
|
||||
cases.sort(
|
||||
(left, right) =>
|
||||
left.suite.localeCompare(right.suite) ||
|
||||
(left.order ?? Number.MAX_SAFE_INTEGER) - (right.order ?? Number.MAX_SAFE_INTEGER) ||
|
||||
left.id.localeCompare(right.id),
|
||||
);
|
||||
return Object.freeze(cases);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { OPERATION_CATALOG, WORKFLOW_ACTION_OPERATIONS } from "./operations.mjs";
|
||||
|
||||
export function buildOperationCoverage(cases) {
|
||||
return OPERATION_CATALOG.map(({ operation, manualNote }) => {
|
||||
const agentCases = cases
|
||||
.filter((testCase) => testCase.coverage.includes(operation))
|
||||
.map(({ id }) => id);
|
||||
let smokeCases;
|
||||
if (["session.start", "session.stop", "session.list"].includes(operation)) {
|
||||
smokeCases = cases.filter(({ smokeSteps }) => smokeSteps.length > 0).map(({ id }) => id);
|
||||
} else {
|
||||
smokeCases = cases
|
||||
.filter(({ smokeSteps }) =>
|
||||
smokeSteps.some((step) => WORKFLOW_ACTION_OPERATIONS[step.action] === operation),
|
||||
)
|
||||
.map(({ id }) => id);
|
||||
}
|
||||
return {
|
||||
operation,
|
||||
agentCases,
|
||||
smokeCases,
|
||||
directSmoke: smokeCases.length > 0,
|
||||
note: smokeCases.length === 0 && manualNote ? `manual lane: ${manualNote}` : undefined,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
export async function discoverFiles(root, predicate) {
|
||||
const files = [];
|
||||
|
||||
async function visit(directory) {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(directory, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
||||
const path = join(directory, entry.name);
|
||||
if (entry.isDirectory()) await visit(path);
|
||||
else if (entry.isFile() && predicate(path)) files.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
await visit(root);
|
||||
return files;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import { discoverFiles } from "./discovery.mjs";
|
||||
import { casesDirectory, fixturesDirectory } from "./paths.mjs";
|
||||
|
||||
export async function loadFixtureRegistry({ roots = [fixturesDirectory, casesDirectory] } = {}) {
|
||||
const files = (
|
||||
await Promise.all(
|
||||
roots.map((root) => discoverFiles(root, (path) => path.endsWith(".fixture.mjs"))),
|
||||
)
|
||||
).flat();
|
||||
const routes = new Map();
|
||||
const definitions = [];
|
||||
const errors = [];
|
||||
|
||||
for (const file of files.sort()) {
|
||||
let definition;
|
||||
try {
|
||||
definition = (await import(pathToFileURL(file).href)).default;
|
||||
} catch (error) {
|
||||
errors.push(`${file}: could not import fixture: ${error.message}`);
|
||||
continue;
|
||||
}
|
||||
if (!definition || typeof definition.id !== "string") {
|
||||
errors.push(`${file}: default export must define id`);
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(definition.routes) || definition.routes.length === 0) {
|
||||
errors.push(`${file}: default export must define non-empty routes`);
|
||||
continue;
|
||||
}
|
||||
if (typeof definition.render !== "function") {
|
||||
errors.push(`${file}: default export must define render(context)`);
|
||||
continue;
|
||||
}
|
||||
definitions.push({ ...definition, sourceFile: file });
|
||||
for (const route of definition.routes) {
|
||||
if (typeof route !== "string" || !route.startsWith("/")) {
|
||||
errors.push(`${file}: invalid route ${JSON.stringify(route)}`);
|
||||
} else if (routes.has(route)) {
|
||||
errors.push(
|
||||
`${file}: route ${route} is already registered by ${routes.get(route).sourceFile}`,
|
||||
);
|
||||
} else {
|
||||
routes.set(route, { definition, sourceFile: file });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) throw new Error(`fixture validation failed:\n${errors.join("\n")}`);
|
||||
return Object.freeze({
|
||||
definitions: Object.freeze(definitions),
|
||||
routes: Object.freeze([...routes.keys()].sort()),
|
||||
hasRoute(pathname) {
|
||||
return routes.has(pathname);
|
||||
},
|
||||
render(pathname, context) {
|
||||
const entry = routes.get(pathname);
|
||||
return entry ? entry.definition.render({ ...context, pathname }) : null;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function validateCaseFixtureLinks(cases, fixtureRegistry) {
|
||||
return cases
|
||||
.filter(({ startPath }) => !fixtureRegistry.hasRoute(startPath))
|
||||
.map(({ id, startPath }) => `case ${id} references unregistered fixture route ${startPath}`);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
const stylesheet = `
|
||||
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||
body { margin: 0; background: #f4f7fb; color: #182033; }
|
||||
main { width: min(760px, calc(100% - 40px)); margin: 48px auto; }
|
||||
.card { background: white; border: 1px solid #d9e1ef; border-radius: 14px; padding: 24px; box-shadow: 0 8px 28px #20304a12; }
|
||||
h1 { margin-top: 0; }
|
||||
a, button { font: inherit; }
|
||||
button, .button { display: inline-block; border: 0; border-radius: 8px; padding: 10px 16px; background: #3157d5; color: white; cursor: pointer; text-decoration: none; }
|
||||
label { display: block; margin: 16px 0; font-weight: 600; }
|
||||
input, textarea, select { display: block; box-sizing: border-box; width: 100%; margin-top: 6px; border: 1px solid #9caac2; border-radius: 7px; padding: 9px; font: inherit; }
|
||||
textarea { min-height: 90px; }
|
||||
.marker { border-left: 4px solid #3157d5; padding: 8px 12px; background: #eef2ff; font-family: ui-monospace, monospace; }
|
||||
.muted { color: #5c667a; }
|
||||
#account-menu { position: relative; display: inline-block; padding-bottom: 12px; }
|
||||
#hover-panel { display: none; position: absolute; z-index: 2; top: 34px; left: 0; width: 190px; padding: 10px; background: white; border: 1px solid #9caac2; border-radius: 8px; box-shadow: 0 8px 24px #20304a22; }
|
||||
#account-menu:hover #hover-panel, #account-menu.open #hover-panel { display: block; }
|
||||
#hover-panel button { width: 100%; }
|
||||
#inspect-card { margin-top: 80px; }
|
||||
code { font-family: ui-monospace, monospace; }
|
||||
`;
|
||||
|
||||
const clientScript = `
|
||||
(() => {
|
||||
const runId = new URLSearchParams(location.search).get("run");
|
||||
const navigation = performance.getEntriesByType("navigation")[0];
|
||||
const send = (type, data = {}) => {
|
||||
if (!runId) return Promise.resolve();
|
||||
return fetch("/api/events", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ runId, type, path: location.pathname, data }),
|
||||
keepalive: true,
|
||||
}).catch(() => undefined);
|
||||
};
|
||||
window.browserEval = { runId, send };
|
||||
window.addEventListener("pageshow", (event) => {
|
||||
send("page.shown", {
|
||||
path: location.pathname,
|
||||
persisted: event.persisted,
|
||||
navigationType: navigation?.type ?? "unknown",
|
||||
});
|
||||
});
|
||||
})();
|
||||
`;
|
||||
|
||||
export function escapeHtml(value) {
|
||||
return String(value)
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """);
|
||||
}
|
||||
|
||||
export function withRun(path, runId, extra = {}) {
|
||||
const url = new URL(path, "http://browser-eval.local");
|
||||
url.searchParams.set("run", runId);
|
||||
for (const [name, value] of Object.entries(extra)) {
|
||||
if (value !== undefined) url.searchParams.set(name, String(value));
|
||||
}
|
||||
return `${url.pathname}${url.search}`;
|
||||
}
|
||||
|
||||
export function page({ title, body, script = "" }) {
|
||||
return `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>${escapeHtml(title)}</title>
|
||||
<style>${stylesheet}</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>${body}</main>
|
||||
<script>${clientScript}</script>
|
||||
${script ? `<script>${script}</script>` : ""}
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
export const OPERATION_CATALOG = [
|
||||
{ operation: "session.start" },
|
||||
{ operation: "session.stop" },
|
||||
{ operation: "session.list" },
|
||||
{ operation: "page.navigate" },
|
||||
{ operation: "page.back" },
|
||||
{ operation: "page.forward" },
|
||||
{ operation: "page.reload" },
|
||||
{ operation: "page.wait" },
|
||||
{ operation: "inspect.observe" },
|
||||
{ operation: "inspect.snapshot" },
|
||||
{ operation: "inspect.html" },
|
||||
{ operation: "inspect.screenshot" },
|
||||
{ operation: "inspect.console" },
|
||||
{ operation: "inspect.network" },
|
||||
{ operation: "interact.click" },
|
||||
{ operation: "interact.hover" },
|
||||
{ operation: "interact.fill" },
|
||||
{ operation: "interact.select" },
|
||||
{ operation: "interact.press" },
|
||||
{ operation: "tabs.list" },
|
||||
{ operation: "tabs.create" },
|
||||
{ operation: "tabs.select" },
|
||||
{ operation: "tabs.close" },
|
||||
{
|
||||
operation: "tabs.borrow",
|
||||
manualNote: "requires an existing user tab and ownership confirmation",
|
||||
},
|
||||
{ operation: "tabs.return", manualNote: "paired with borrowing a real user tab" },
|
||||
{ operation: "assist.resize" },
|
||||
{ operation: "assist.emulate" },
|
||||
{ operation: "assist.request-help", manualNote: "intentionally waits for human interaction" },
|
||||
];
|
||||
|
||||
export const WORKFLOW_ACTION_OPERATIONS = {
|
||||
navigate: "page.navigate",
|
||||
back: "page.back",
|
||||
forward: "page.forward",
|
||||
reload: "page.reload",
|
||||
wait: "page.wait",
|
||||
observe: "inspect.observe",
|
||||
snapshot: "inspect.snapshot",
|
||||
html: "inspect.html",
|
||||
screenshot: "inspect.screenshot",
|
||||
console: "inspect.console",
|
||||
network: "inspect.network",
|
||||
click: "interact.click",
|
||||
hover: "interact.hover",
|
||||
fill: "interact.fill",
|
||||
select: "interact.select",
|
||||
press: "interact.press",
|
||||
"tab-list": "tabs.list",
|
||||
"tab-create": "tabs.create",
|
||||
"tab-select": "tabs.select",
|
||||
"tab-close": "tabs.close",
|
||||
borrow: "tabs.borrow",
|
||||
return: "tabs.return",
|
||||
resize: "assist.resize",
|
||||
emulate: "assist.emulate",
|
||||
"request-help": "assist.request-help",
|
||||
};
|
||||
|
||||
export const WORKFLOW_ACTIONS = new Set([
|
||||
...Object.keys(WORKFLOW_ACTION_OPERATIONS),
|
||||
"wait-site-event",
|
||||
]);
|
||||
|
||||
export const OPERATION_NAMES = new Set(OPERATION_CATALOG.map(({ operation }) => operation));
|
||||
@@ -0,0 +1,63 @@
|
||||
function readPath(value, path) {
|
||||
return path.split(".").reduce((current, part) => current?.[part], value);
|
||||
}
|
||||
|
||||
function eventMatches(event, assertion) {
|
||||
if (event.type !== assertion.type) return false;
|
||||
return Object.entries(assertion.where ?? {}).every(
|
||||
([path, expected]) => readPath(event, path) === expected,
|
||||
);
|
||||
}
|
||||
|
||||
export function verifyTask(task, { events = [], responseText = "", adapterEvidence = {} } = {}) {
|
||||
const site = task.siteAssertions.map((assertion) => {
|
||||
const actual = events.filter((event) => eventMatches(event, assertion)).length;
|
||||
return {
|
||||
kind: "site",
|
||||
label: assertion.label,
|
||||
status: actual >= assertion.minCount ? "passed" : "failed",
|
||||
expected: `at least ${assertion.minCount}`,
|
||||
actual,
|
||||
};
|
||||
});
|
||||
|
||||
const response = task.responseAssertions.map((assertion) => ({
|
||||
kind: "response",
|
||||
label: assertion.label,
|
||||
status: responseText.includes(assertion.includes) ? "passed" : "failed",
|
||||
expected: `response includes ${JSON.stringify(assertion.includes)}`,
|
||||
}));
|
||||
|
||||
const adapter = task.adapterAssertions.map((assertion) => {
|
||||
if (!Object.hasOwn(adapterEvidence, assertion.key)) {
|
||||
return {
|
||||
kind: "adapter",
|
||||
label: assertion.label,
|
||||
status: "unverified",
|
||||
expected: assertion.key,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "adapter",
|
||||
label: assertion.label,
|
||||
status: adapterEvidence[assertion.key] ? "passed" : "failed",
|
||||
expected: assertion.key,
|
||||
actual: adapterEvidence[assertion.key],
|
||||
};
|
||||
});
|
||||
|
||||
const checks = [...site, ...response, ...adapter];
|
||||
const failed = checks.filter(({ status }) => status === "failed").length;
|
||||
const unverified = checks.filter(({ status }) => status === "unverified").length;
|
||||
return {
|
||||
taskId: task.id,
|
||||
status: failed > 0 ? "failed" : unverified > 0 ? "passed-with-unverified" : "passed",
|
||||
summary: {
|
||||
passed: checks.length - failed - unverified,
|
||||
failed,
|
||||
unverified,
|
||||
total: checks.length,
|
||||
},
|
||||
checks,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const evalDirectory = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
export const casesDirectory = join(evalDirectory, "cases");
|
||||
export const fixturesDirectory = join(evalDirectory, "fixtures");
|
||||
export const schemasDirectory = join(evalDirectory, "schemas");
|
||||
export const resultsDirectory = join(evalDirectory, "results");
|
||||
@@ -0,0 +1,71 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const MAX_CAPTURE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
function appendCaptured(current, chunk) {
|
||||
if (current.length >= MAX_CAPTURE_BYTES) return current;
|
||||
return `${current}${chunk}`.slice(0, MAX_CAPTURE_BYTES);
|
||||
}
|
||||
|
||||
export function runProcess(command, args, { cwd, env, timeoutMs = 300_000 } = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
let settled = false;
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
env: { ...process.env, ...env },
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout = appendCaptured(stdout, chunk.toString());
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr = appendCaptured(stderr, chunk.toString());
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
|
||||
}, timeoutMs);
|
||||
|
||||
const finish = (result) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
resolve({
|
||||
command,
|
||||
args,
|
||||
stdout,
|
||||
stderr,
|
||||
timedOut,
|
||||
durationMs: Date.now() - startedAt,
|
||||
...result,
|
||||
});
|
||||
};
|
||||
|
||||
child.once("error", (error) => finish({ exitCode: null, signal: null, error: error.message }));
|
||||
child.once("close", (exitCode, signal) => finish({ exitCode, signal }));
|
||||
});
|
||||
}
|
||||
|
||||
export function substituteArgs(args, values) {
|
||||
return args.map((argument) => {
|
||||
let result = argument;
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
result = result.replaceAll(`{${key}}`, String(value));
|
||||
}
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
export function countPattern(text, pattern) {
|
||||
if (!pattern) return null;
|
||||
const matches = text.match(new RegExp(pattern, "g"));
|
||||
return matches?.length ?? 0;
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { relative, sep } from "node:path";
|
||||
|
||||
import { validateCaseFixtureLinks } from "./fixture-registry.mjs";
|
||||
import { casesDirectory } from "./paths.mjs";
|
||||
|
||||
function templateRoots(value) {
|
||||
if (typeof value === "string") {
|
||||
return [...value.matchAll(/\{([^{}]+)\}/g)].map((match) => match[1].split(".")[0]);
|
||||
}
|
||||
if (Array.isArray(value)) return value.flatMap(templateRoots);
|
||||
if (value && typeof value === "object") return Object.values(value).flatMap(templateRoots);
|
||||
return [];
|
||||
}
|
||||
|
||||
export function validateRepositoryCases(cases, fixtureRegistry) {
|
||||
const errors = [...validateCaseFixtureLinks(cases, fixtureRegistry)];
|
||||
for (const testCase of cases) {
|
||||
const relativePath = relative(casesDirectory, testCase.sourceFile);
|
||||
const suiteDirectory = relativePath.split(sep)[0];
|
||||
if (suiteDirectory !== testCase.suite) {
|
||||
errors.push(
|
||||
`case ${testCase.id} declares suite ${testCase.suite} but is stored under ${suiteDirectory}`,
|
||||
);
|
||||
}
|
||||
|
||||
const availableVariables = new Set(["baseUrl", "runId", "seed", "sessionId", "url"]);
|
||||
const evidence = new Set(["sessionStopped"]);
|
||||
for (const [index, step] of testCase.smokeSteps.entries()) {
|
||||
for (const root of templateRoots(step)) {
|
||||
if (!availableVariables.has(root)) {
|
||||
errors.push(
|
||||
`case ${testCase.id} smoke.steps[${index}] references unavailable variable ${root}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (step.saveAs) availableVariables.add(step.saveAs);
|
||||
if (step.evidence) evidence.add(step.evidence);
|
||||
}
|
||||
for (const assertion of testCase.adapterAssertions) {
|
||||
if (!evidence.has(assertion.key)) {
|
||||
errors.push(
|
||||
`case ${testCase.id} expects adapter evidence ${assertion.key} but no smoke step produces it`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function repositorySummary(cases, fixtureRegistry) {
|
||||
const suites = Object.fromEntries(
|
||||
[...new Set(cases.map(({ suite }) => suite))]
|
||||
.sort()
|
||||
.map((suite) => [suite, cases.filter((testCase) => testCase.suite === suite).length]),
|
||||
);
|
||||
const tags = [...new Set(cases.flatMap(({ tags }) => tags))].sort();
|
||||
return {
|
||||
cases: cases.length,
|
||||
suites,
|
||||
tags: tags.length,
|
||||
fixtureModules: fixtureRegistry.definitions.length,
|
||||
fixtureRoutes: fixtureRegistry.routes.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import { mkdir, realpath, stat, writeFile } from "node:fs/promises";
|
||||
import { join, relative, sep } from "node:path";
|
||||
|
||||
import { casesDirectory, evalDirectory } from "./paths.mjs";
|
||||
|
||||
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
|
||||
async function exists(path) {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function moduleSpecifier(fromDirectory, target) {
|
||||
const path = relative(await realpath(fromDirectory), await realpath(target))
|
||||
.split(sep)
|
||||
.join("/");
|
||||
return path.startsWith(".") ? path : `./${path}`;
|
||||
}
|
||||
|
||||
export async function scaffoldCase({
|
||||
id,
|
||||
title = id,
|
||||
suite = "regression",
|
||||
source,
|
||||
rootCasesDirectory = casesDirectory,
|
||||
}) {
|
||||
if (!ID_PATTERN.test(id ?? "")) throw new Error("case id must use lowercase kebab-case");
|
||||
if (!ID_PATTERN.test(suite ?? "")) throw new Error("suite must use lowercase kebab-case");
|
||||
const directory = join(rootCasesDirectory, suite, id);
|
||||
if (await exists(directory)) throw new Error(`case directory already exists: ${directory}`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
|
||||
const marker = `CASE-${id.toUpperCase().replaceAll("-", "_")}`;
|
||||
const startPath = `/cases/${id}`;
|
||||
const manifest = {
|
||||
$schema: "../../../schemas/case.schema.json",
|
||||
schemaVersion: 1,
|
||||
id,
|
||||
title,
|
||||
order: 10,
|
||||
suite,
|
||||
tags: ["badcase", "regression"],
|
||||
...(source ? { source: { type: "user-badcase", reference: source } } : {}),
|
||||
fixture: { startPath },
|
||||
prompts: {
|
||||
en: `Open {url}, complete the minimal regression scenario, report ${marker}, and close the browser session.`,
|
||||
"zh-CN": `打开 {url},完成最小回归场景,报告 ${marker},最后关闭浏览器会话。`,
|
||||
},
|
||||
coverage: ["session.start", "session.stop", "page.navigate", "inspect.observe"],
|
||||
assertions: {
|
||||
site: [
|
||||
{
|
||||
label: "regression fixture was displayed",
|
||||
type: "page.shown",
|
||||
where: { "data.path": startPath },
|
||||
minCount: 1,
|
||||
},
|
||||
],
|
||||
response: [{ label: "reported regression marker", includes: marker }],
|
||||
adapter: [{ label: "browser session was closed", key: "sessionStopped" }],
|
||||
},
|
||||
smoke: { steps: [{ action: "navigate" }, { action: "observe" }] },
|
||||
};
|
||||
|
||||
const fixtureHelper = await moduleSpecifier(
|
||||
directory,
|
||||
join(evalDirectory, "lib", "fixtures.mjs"),
|
||||
);
|
||||
const fixture = `import { escapeHtml, page } from ${JSON.stringify(fixtureHelper)};
|
||||
|
||||
const title = ${JSON.stringify(title)};
|
||||
|
||||
export default {
|
||||
id: ${JSON.stringify(id)},
|
||||
routes: [${JSON.stringify(startPath)}],
|
||||
render() {
|
||||
return page({
|
||||
title,
|
||||
body: \`<section class="card"><h1>\${escapeHtml(title)}</h1><p class="marker">${marker}</p><p>Replace this page with a privacy-safe minimal reproduction.</p></section>\`,
|
||||
});
|
||||
},
|
||||
};
|
||||
`;
|
||||
const readme = `# ${title}
|
||||
|
||||
## Original symptom
|
||||
|
||||
Describe what the user observed without including private data or proprietary page content.
|
||||
|
||||
## Minimal reproduction
|
||||
|
||||
Explain the smallest DOM, timing, or browser-state condition that reproduces the failure.
|
||||
|
||||
## Expected behavior
|
||||
|
||||
Describe the observable success condition and update the case assertions accordingly.
|
||||
|
||||
## Regression history
|
||||
|
||||
- Source: ${source ?? "add issue, PR, or anonymized user report reference"}
|
||||
- Fix: add the fixing PR when available
|
||||
`;
|
||||
|
||||
const files = {
|
||||
manifest: join(directory, `${id}.case.json`),
|
||||
fixture: join(directory, `${id}.fixture.mjs`),
|
||||
readme: join(directory, "README.md"),
|
||||
};
|
||||
await Promise.all([
|
||||
writeFile(files.manifest, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }),
|
||||
writeFile(files.fixture, fixture, { flag: "wx" }),
|
||||
writeFile(files.readme, readme, { flag: "wx" }),
|
||||
]);
|
||||
return { directory, files };
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
function hashSeed(value) {
|
||||
const text = String(value);
|
||||
let hash = 2166136261;
|
||||
for (const character of text) {
|
||||
hash ^= character.codePointAt(0);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function mulberry32(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state += 0x6d2b79f5;
|
||||
let value = state;
|
||||
value = Math.imul(value ^ (value >>> 15), value | 1);
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeSeed(value) {
|
||||
const numeric = Number(value);
|
||||
return Number.isSafeInteger(numeric) ? numeric : hashSeed(value);
|
||||
}
|
||||
|
||||
export function describeSeed(value) {
|
||||
const seed = normalizeSeed(value);
|
||||
const random = mulberry32(seed);
|
||||
const choose = (values) => values[Math.floor(random() * values.length)];
|
||||
return {
|
||||
seed,
|
||||
labelMode: choose(["wrapped", "for", "aria"]),
|
||||
nestingDepth: 1 + Math.floor(random() * 4),
|
||||
hydrationDelayMs: choose([0, 40, 120, 250]),
|
||||
decoy: choose(["none", "disabled-input", "duplicate-text"]),
|
||||
fieldOrder: choose(["text-first", "select-first"]),
|
||||
idSuffix: hashSeed(`${seed}:id`).toString(36).slice(0, 6),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { createServer } from "node:http";
|
||||
|
||||
import { loadFixtureRegistry } from "./fixture-registry.mjs";
|
||||
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9._-]{1,120}$/;
|
||||
|
||||
function validRunId(value) {
|
||||
return typeof value === "string" && RUN_ID_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function json(response, status, value) {
|
||||
response.writeHead(status, {
|
||||
"cache-control": "no-store",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
});
|
||||
response.end(`${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function html(response, status, value) {
|
||||
response.writeHead(status, {
|
||||
"cache-control": "no-store",
|
||||
"content-type": "text/html; charset=utf-8",
|
||||
});
|
||||
response.end(value);
|
||||
}
|
||||
|
||||
async function readJson(request) {
|
||||
let raw = "";
|
||||
for await (const chunk of request) {
|
||||
raw += chunk;
|
||||
if (raw.length > 64 * 1024) {
|
||||
throw new Error("request body exceeds 64 KiB");
|
||||
}
|
||||
}
|
||||
return raw ? JSON.parse(raw) : {};
|
||||
}
|
||||
|
||||
export function createEvalServer({ host = "127.0.0.1", port = 0, fixtureRegistry } = {}) {
|
||||
const runs = new Map();
|
||||
let sequence = 0;
|
||||
let server;
|
||||
let registry = fixtureRegistry;
|
||||
|
||||
function eventsFor(runId) {
|
||||
if (!runs.has(runId)) runs.set(runId, []);
|
||||
return runs.get(runId);
|
||||
}
|
||||
|
||||
function record(runId, type, data = {}, metadata = {}) {
|
||||
if (!validRunId(runId)) return null;
|
||||
const event = {
|
||||
sequence: ++sequence,
|
||||
timestamp: new Date().toISOString(),
|
||||
type,
|
||||
data,
|
||||
...metadata,
|
||||
};
|
||||
eventsFor(runId).push(event);
|
||||
return event;
|
||||
}
|
||||
|
||||
function snapshot(runId) {
|
||||
return { runId, events: [...(runs.get(runId) ?? [])] };
|
||||
}
|
||||
|
||||
function reset(runId) {
|
||||
runs.set(runId, []);
|
||||
}
|
||||
|
||||
async function handle(request, response) {
|
||||
const origin = `http://${request.headers.host ?? `${host}:${port}`}`;
|
||||
const url = new URL(request.url ?? "/", origin);
|
||||
|
||||
if (request.method === "GET" && url.pathname === "/healthz") {
|
||||
return json(response, 200, { ok: true });
|
||||
}
|
||||
|
||||
if (request.method === "POST" && url.pathname === "/api/events") {
|
||||
const body = await readJson(request);
|
||||
if (!validRunId(body.runId) || typeof body.type !== "string") {
|
||||
return json(response, 400, { error: "runId and type are required" });
|
||||
}
|
||||
const event = record(body.runId, body.type, body.data ?? {}, {
|
||||
path: typeof body.path === "string" ? body.path : undefined,
|
||||
});
|
||||
return json(response, 202, { accepted: true, sequence: event.sequence });
|
||||
}
|
||||
|
||||
const runMatch = url.pathname.match(/^\/api\/runs\/([^/]+)$/);
|
||||
if (runMatch) {
|
||||
const runId = decodeURIComponent(runMatch[1]);
|
||||
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
|
||||
if (request.method === "GET") return json(response, 200, snapshot(runId));
|
||||
if (request.method === "DELETE") {
|
||||
reset(runId);
|
||||
return json(response, 200, { runId, reset: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (request.method === "GET" && url.pathname === "/api/ping") {
|
||||
const runId = url.searchParams.get("run");
|
||||
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
|
||||
record(runId, "network.ping", { token: "NETWORK-73" }, { path: url.pathname });
|
||||
return json(response, 200, { ok: true, token: "NETWORK-73" });
|
||||
}
|
||||
|
||||
if (request.method !== "GET") return json(response, 405, { error: "method not allowed" });
|
||||
|
||||
const runId = url.searchParams.get("run") ?? "manual";
|
||||
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
|
||||
|
||||
const fixture = registry.render(url.pathname, {
|
||||
runId,
|
||||
query: url.searchParams,
|
||||
record: (type, data, metadata) => record(runId, type, data, metadata),
|
||||
});
|
||||
if (!fixture) return html(response, 404, "<!doctype html><h1>Not found</h1>");
|
||||
record(runId, "page.request", { path: url.pathname }, { path: url.pathname });
|
||||
return html(response, 200, fixture);
|
||||
}
|
||||
|
||||
return {
|
||||
record,
|
||||
reset,
|
||||
snapshot,
|
||||
async start() {
|
||||
if (server) throw new Error("eval server is already running");
|
||||
registry ??= await loadFixtureRegistry();
|
||||
server = createServer((request, response) => {
|
||||
handle(request, response).catch((error) => {
|
||||
json(response, 500, { error: error instanceof Error ? error.message : String(error) });
|
||||
});
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(port, host, resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("unexpected server address");
|
||||
return { host, port: address.port, baseUrl: `http://${host}:${address.port}` };
|
||||
},
|
||||
async stop() {
|
||||
if (!server) return;
|
||||
const active = server;
|
||||
server = undefined;
|
||||
await new Promise((resolve, reject) => {
|
||||
active.close((error) => (error ? reject(error) : resolve()));
|
||||
active.closeAllConnections?.();
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
function durationFor(result) {
|
||||
if (result.execution?.durationMs !== undefined) return result.execution.durationMs;
|
||||
return (result.steps ?? []).reduce((sum, step) => sum + step.durationMs, 0);
|
||||
}
|
||||
|
||||
export function summarizeReports(reports) {
|
||||
const groups = new Map();
|
||||
for (const report of reports) {
|
||||
for (const result of report.results ?? []) {
|
||||
const adapter = result.adapter ?? report.adapter ?? "unknown";
|
||||
const variant = result.variant ?? report.variant ?? "unspecified";
|
||||
const key = `${adapter}\u0000${variant}`;
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, {
|
||||
adapter,
|
||||
variant,
|
||||
runs: 0,
|
||||
passed: 0,
|
||||
fullyVerified: 0,
|
||||
executionFailures: 0,
|
||||
errors: 0,
|
||||
durationMs: 0,
|
||||
toolCalls: 0,
|
||||
toolCallSamples: 0,
|
||||
});
|
||||
}
|
||||
const group = groups.get(key);
|
||||
group.runs += 1;
|
||||
group.durationMs += durationFor(result);
|
||||
group.errors += result.metrics?.errorCount ?? (result.executionError ? 1 : 0);
|
||||
if (result.metrics?.toolCallCount !== null && result.metrics?.toolCallCount !== undefined) {
|
||||
group.toolCalls += result.metrics.toolCallCount;
|
||||
group.toolCallSamples += 1;
|
||||
}
|
||||
const executionFailed =
|
||||
Boolean(result.executionError) ||
|
||||
result.execution?.timedOut ||
|
||||
(result.execution?.exitCode !== undefined && result.execution.exitCode !== 0);
|
||||
if (executionFailed) group.executionFailures += 1;
|
||||
if (result.verification?.status !== "failed" && !executionFailed) group.passed += 1;
|
||||
if (result.verification?.status === "passed" && !executionFailed) group.fullyVerified += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return [...groups.values()].map((group) => ({
|
||||
adapter: group.adapter,
|
||||
variant: group.variant,
|
||||
runs: group.runs,
|
||||
passRate: group.runs ? group.passed / group.runs : 0,
|
||||
fullyVerifiedRate: group.runs ? group.fullyVerified / group.runs : 0,
|
||||
executionFailures: group.executionFailures,
|
||||
errors: group.errors,
|
||||
averageToolCalls: group.toolCallSamples
|
||||
? Math.round((group.toolCalls / group.toolCallSamples) * 10) / 10
|
||||
: null,
|
||||
averageDurationMs: group.runs ? Math.round(group.durationMs / group.runs) : 0,
|
||||
}));
|
||||
}
|
||||
|
||||
export function printSummary(rows) {
|
||||
const printable = rows.map((row) => ({
|
||||
adapter: row.adapter,
|
||||
variant: row.variant,
|
||||
runs: row.runs,
|
||||
pass: `${Math.round(row.passRate * 100)}%`,
|
||||
verified: `${Math.round(row.fullyVerifiedRate * 100)}%`,
|
||||
exec_failures: row.executionFailures,
|
||||
errors: row.errors,
|
||||
avg_tools: row.averageToolCalls ?? "-",
|
||||
avg_ms: row.averageDurationMs,
|
||||
}));
|
||||
console.table(printable);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { loadCaseManifests } from "./case-loader.mjs";
|
||||
|
||||
let casesPromise;
|
||||
|
||||
export function allTasks({ reload = false } = {}) {
|
||||
if (reload || !casesPromise) casesPromise = loadCaseManifests();
|
||||
return casesPromise;
|
||||
}
|
||||
|
||||
export function listTasks(cases) {
|
||||
return cases.map(({ id, title, suite, tags, seed }) => ({ id, title, suite, tags, seed }));
|
||||
}
|
||||
|
||||
export function getTask(taskId, cases) {
|
||||
const task = cases.find(({ id }) => id === taskId);
|
||||
if (!task) {
|
||||
const ids = cases.map(({ id }) => id).join(", ");
|
||||
throw new Error(`unknown case ${JSON.stringify(taskId)}; expected one of: ${ids}`);
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
export function taskUrl(task, baseUrl, runId, seed = task.seed) {
|
||||
const url = new URL(task.startPath, baseUrl);
|
||||
url.searchParams.set("run", runId);
|
||||
if (seed !== undefined) url.searchParams.set("seed", String(seed));
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function expandTemplate(template, values) {
|
||||
return Object.entries(values).reduce(
|
||||
(result, [name, value]) => result.replaceAll(`{${name}}`, String(value)),
|
||||
template,
|
||||
);
|
||||
}
|
||||
|
||||
export function renderPrompt(task, { baseUrl, runId, locale = "zh-CN", seed = task.seed }) {
|
||||
const template = task.prompts[locale];
|
||||
if (!template) {
|
||||
throw new Error(`unsupported locale ${JSON.stringify(locale)}; expected en or zh-CN`);
|
||||
}
|
||||
return expandTemplate(template, {
|
||||
baseUrl,
|
||||
runId: encodeURIComponent(runId),
|
||||
seed: seed ?? "",
|
||||
url: taskUrl(task, baseUrl, runId, seed),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://browser-skill.local/schemas/browser-eval-case.schema.json",
|
||||
"title": "BrowserSkill browser evaluation case",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schemaVersion",
|
||||
"id",
|
||||
"title",
|
||||
"suite",
|
||||
"tags",
|
||||
"fixture",
|
||||
"prompts",
|
||||
"coverage",
|
||||
"assertions",
|
||||
"smoke"
|
||||
],
|
||||
"properties": {
|
||||
"$schema": { "type": "string" },
|
||||
"schemaVersion": { "const": 1 },
|
||||
"id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
|
||||
"title": { "type": "string", "minLength": 1 },
|
||||
"order": { "type": "integer", "minimum": 0 },
|
||||
"suite": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
|
||||
"uniqueItems": true
|
||||
},
|
||||
"fixture": {
|
||||
"type": "object",
|
||||
"required": ["startPath"],
|
||||
"properties": { "startPath": { "type": "string", "pattern": "^/" } },
|
||||
"additionalProperties": false
|
||||
},
|
||||
"prompts": {
|
||||
"type": "object",
|
||||
"required": ["en", "zh-CN"],
|
||||
"properties": {
|
||||
"en": { "type": "string", "minLength": 1 },
|
||||
"zh-CN": { "type": "string", "minLength": 1 }
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"coverage": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
|
||||
"assertions": {
|
||||
"type": "object",
|
||||
"required": ["site", "response", "adapter"],
|
||||
"properties": {
|
||||
"site": { "type": "array", "items": { "type": "object" } },
|
||||
"response": { "type": "array", "items": { "type": "object" } },
|
||||
"adapter": { "type": "array", "items": { "type": "object" } }
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"smoke": {
|
||||
"type": "object",
|
||||
"required": ["steps"],
|
||||
"properties": {
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["action"],
|
||||
"properties": { "action": { "type": "string" } },
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"seed": { "type": "integer" },
|
||||
"source": { "type": "object" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { runCommandTask } from "../lib/agent-runner.mjs";
|
||||
import { loadCaseManifests, validateCaseManifest } from "../lib/case-loader.mjs";
|
||||
import { buildOperationCoverage } from "../lib/coverage.mjs";
|
||||
import { loadFixtureRegistry } from "../lib/fixture-registry.mjs";
|
||||
import { verifyTask } from "../lib/oracle.mjs";
|
||||
import { runProcess } from "../lib/process.mjs";
|
||||
import { repositorySummary, validateRepositoryCases } from "../lib/repository-validation.mjs";
|
||||
import { scaffoldCase } from "../lib/scaffold-case.mjs";
|
||||
import { describeSeed } from "../lib/seeded-generator.mjs";
|
||||
import { createEvalServer } from "../lib/server.mjs";
|
||||
import { summarizeReports } from "../lib/summary.mjs";
|
||||
import { allTasks, getTask, renderPrompt } from "../lib/tasks.mjs";
|
||||
|
||||
const cases = await allTasks();
|
||||
const fixtureRegistry = await loadFixtureRegistry();
|
||||
|
||||
async function withServer(run) {
|
||||
const server = createEvalServer({ fixtureRegistry });
|
||||
const info = await server.start();
|
||||
try {
|
||||
await run(server, info);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
}
|
||||
|
||||
test("case manifests are discovered, ordered, and grouped into suites", () => {
|
||||
assert.deepEqual(
|
||||
cases.map(({ id }) => id),
|
||||
[
|
||||
"navigation-history",
|
||||
"form-controls",
|
||||
"hover-inspect",
|
||||
"tabs",
|
||||
"diagnostics",
|
||||
"mobile-emulation",
|
||||
"generated-form",
|
||||
],
|
||||
);
|
||||
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
|
||||
});
|
||||
|
||||
test("repository validation links every case to a fixture and valid workflow evidence", () => {
|
||||
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
|
||||
const summary = repositorySummary(cases, fixtureRegistry);
|
||||
assert.equal(summary.cases, 7);
|
||||
assert.equal(summary.fixtureModules, 8);
|
||||
assert.equal(summary.fixtureRoutes, 12);
|
||||
});
|
||||
|
||||
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
|
||||
const invalid = {
|
||||
schemaVersion: 1,
|
||||
id: "invalid-case",
|
||||
title: "Invalid",
|
||||
suite: "regression",
|
||||
tags: ["regression"],
|
||||
fixture: { startPath: "/invalid" },
|
||||
prompts: { en: "Open {url}", "zh-CN": "打开 {url}" },
|
||||
coverage: ["page.teleport"],
|
||||
assertions: { site: [], response: [], adapter: [] },
|
||||
smoke: { steps: [{ action: "select", selector: "#choice", values: [] }] },
|
||||
typo: true,
|
||||
};
|
||||
const errors = validateCaseManifest(invalid);
|
||||
assert.ok(errors.some((error) => error.includes("unknown top-level property typo")));
|
||||
assert.ok(errors.some((error) => error.includes("unknown operation page.teleport")));
|
||||
assert.ok(errors.some((error) => error.includes("values must be a non-empty array")));
|
||||
});
|
||||
|
||||
test("fixture server exposes deterministic task and seeded matrix markers", async () => {
|
||||
await withServer(async (_server, { baseUrl }) => {
|
||||
const fixtures = [
|
||||
["/navigation/detail?run=test-markers", "NAV-42"],
|
||||
["/form?run=test-markers", '<option value="two">Two</option>'],
|
||||
["/hover-inspect?run=test-markers", "INSPECT-27"],
|
||||
["/tabs/child?run=test-markers", "TAB-29"],
|
||||
["/diagnostics?run=test-markers", "CONSOLE-61"],
|
||||
["/responsive?run=test-markers", "VIEWPORT-84"],
|
||||
["/matrix/generated-form?run=test-markers&seed=17", "SEED-17"],
|
||||
];
|
||||
for (const [path, marker] of fixtures) {
|
||||
const response = await fetch(`${baseUrl}${path}`);
|
||||
assert.equal(response.status, 200);
|
||||
assert.match(await response.text(), new RegExp(marker));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("server records and resets run-scoped events", async () => {
|
||||
await withServer(async (server, { baseUrl }) => {
|
||||
const response = await fetch(`${baseUrl}/api/events`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ runId: "event-test", type: "example", data: { value: 7 } }),
|
||||
});
|
||||
assert.equal(response.status, 202);
|
||||
assert.equal(server.snapshot("event-test").events[0].data.value, 7);
|
||||
const resetResponse = await fetch(`${baseUrl}/api/runs/event-test`, { method: "DELETE" });
|
||||
assert.equal(resetResponse.status, 200);
|
||||
assert.deepEqual(server.snapshot("event-test").events, []);
|
||||
});
|
||||
});
|
||||
|
||||
test("form case oracle checks submitted values and separates adapter evidence", () => {
|
||||
const task = getTask("form-controls", cases);
|
||||
const events = [
|
||||
{ type: "form.enter_pressed", data: {} },
|
||||
{
|
||||
type: "form.submitted",
|
||||
data: { text: "agent-parity", notes: "BrowserSkill works", choice: "two" },
|
||||
},
|
||||
{ type: "page.shown", data: { path: "/result" } },
|
||||
];
|
||||
const partial = verifyTask(task, { events, responseText: "The page says Received!" });
|
||||
assert.equal(partial.status, "passed-with-unverified");
|
||||
assert.equal(partial.summary.unverified, 2);
|
||||
|
||||
const complete = verifyTask(task, {
|
||||
events,
|
||||
responseText: "The page says Received!",
|
||||
adapterEvidence: { screenshotCreated: true, sessionStopped: true },
|
||||
});
|
||||
assert.equal(complete.status, "passed");
|
||||
});
|
||||
|
||||
test("navigation case fails when reload is missing", () => {
|
||||
const task = getTask("navigation-history", cases);
|
||||
const verification = verifyTask(task, {
|
||||
events: [
|
||||
{ type: "navigation.detail.clicked", data: {} },
|
||||
{ type: "page.shown", data: { path: "/navigation/start" } },
|
||||
{ type: "page.shown", data: { path: "/navigation/start" } },
|
||||
{ type: "page.shown", data: { path: "/navigation/detail" } },
|
||||
{ type: "page.shown", data: { path: "/navigation/detail" } },
|
||||
],
|
||||
responseText: "NAV-42",
|
||||
});
|
||||
assert.equal(verification.status, "failed");
|
||||
assert.equal(
|
||||
verification.checks.find(({ label }) => label === "detail page was reloaded").status,
|
||||
"failed",
|
||||
);
|
||||
});
|
||||
|
||||
test("prompt rendering is run-scoped, localized, and seed-aware", () => {
|
||||
const task = getTask("generated-form", cases);
|
||||
const input = { baseUrl: "http://127.0.0.1:4173", runId: "prompt-test", seed: 17 };
|
||||
assert.match(renderPrompt(task, { ...input, locale: "en" }), /seed=17/);
|
||||
assert.match(renderPrompt(task, { ...input, locale: "zh-CN" }), /seeded form/);
|
||||
});
|
||||
|
||||
test("command adapter carries a run from process launch through oracle verification", async () => {
|
||||
await withServer(async (server, serverInfo) => {
|
||||
const script = `
|
||||
const [baseUrl, runId] = process.argv.slice(1);
|
||||
const send = (type, data) => fetch(baseUrl + "/api/events", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ runId, type, data }),
|
||||
});
|
||||
await send("navigation.detail.clicked", {});
|
||||
await send("page.shown", { path: "/navigation/start" });
|
||||
await send("page.shown", { path: "/navigation/start" });
|
||||
await send("page.shown", { path: "/navigation/detail" });
|
||||
await send("page.shown", { path: "/navigation/detail" });
|
||||
await send("page.shown", { path: "/navigation/detail", navigationType: "reload" });
|
||||
console.log("NAV-42");
|
||||
`;
|
||||
const result = await runCommandTask({
|
||||
server,
|
||||
serverInfo,
|
||||
task: getTask("navigation-history", cases),
|
||||
adapter: {
|
||||
command: process.execPath,
|
||||
args: ["--input-type=module", "-e", script, "{baseUrl}", "{runId}"],
|
||||
variant: "fake-test",
|
||||
},
|
||||
adapterName: "fake-agent",
|
||||
configDirectory: process.cwd(),
|
||||
iteration: 1,
|
||||
locale: "en",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
assert.equal(result.execution.exitCode, 0);
|
||||
assert.equal(result.verification.status, "passed-with-unverified");
|
||||
assert.equal(result.eventCount, 6);
|
||||
});
|
||||
});
|
||||
|
||||
test("report summary keeps partial verification distinct from full verification", () => {
|
||||
const rows = summarizeReports([
|
||||
{
|
||||
results: [
|
||||
{
|
||||
adapter: "agent-a",
|
||||
variant: "granular-28",
|
||||
execution: { exitCode: 0, durationMs: 100 },
|
||||
metrics: { errorCount: 0, toolCallCount: 4 },
|
||||
verification: { status: "passed-with-unverified" },
|
||||
},
|
||||
{
|
||||
adapter: "agent-a",
|
||||
variant: "granular-28",
|
||||
execution: { exitCode: 0, durationMs: 300 },
|
||||
metrics: { errorCount: 1, toolCallCount: 6 },
|
||||
verification: { status: "passed" },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.equal(rows[0].passRate, 1);
|
||||
assert.equal(rows[0].fullyVerifiedRate, 0.5);
|
||||
assert.equal(rows[0].averageDurationMs, 200);
|
||||
assert.equal(rows[0].averageToolCalls, 5);
|
||||
assert.equal(rows[0].errors, 1);
|
||||
});
|
||||
|
||||
test("coverage inventory contains all 28 operations and three manual lanes", () => {
|
||||
const coverage = buildOperationCoverage(cases);
|
||||
const operations = coverage.map(({ operation }) => operation);
|
||||
assert.equal(operations.length, 28);
|
||||
assert.equal(new Set(operations).size, 28);
|
||||
assert.deepEqual(
|
||||
coverage.filter(({ directSmoke }) => !directSmoke).map(({ operation }) => operation),
|
||||
["tabs.borrow", "tabs.return", "assist.request-help"],
|
||||
);
|
||||
assert.deepEqual(coverage.find(({ operation }) => operation === "interact.fill").smokeCases, [
|
||||
"form-controls",
|
||||
"generated-form",
|
||||
]);
|
||||
});
|
||||
|
||||
test("seeded dimensions are reproducible and vary across seeds", () => {
|
||||
assert.deepEqual(describeSeed("badcase-17"), describeSeed("badcase-17"));
|
||||
assert.notDeepEqual(describeSeed("badcase-17"), describeSeed("badcase-18"));
|
||||
assert.equal(describeSeed(17).seed, 17);
|
||||
});
|
||||
|
||||
test("badcase scaffold creates a loadable, escaped, self-contained case", async () => {
|
||||
const temporaryRoot = await mkdtemp(join(tmpdir(), "browser-eval-scaffold-"));
|
||||
try {
|
||||
const title = "Quoted `title` ${notCode} <unsafe>";
|
||||
const result = await scaffoldCase({
|
||||
id: "reported-timeout",
|
||||
title,
|
||||
suite: "regression",
|
||||
source: "issue-123",
|
||||
rootCasesDirectory: temporaryRoot,
|
||||
});
|
||||
const loaded = await loadCaseManifests({ casesDirectory: temporaryRoot });
|
||||
assert.equal(loaded[0].id, "reported-timeout");
|
||||
assert.equal(loaded[0].source.reference, "issue-123");
|
||||
|
||||
const generatedRegistry = await loadFixtureRegistry({ roots: [temporaryRoot] });
|
||||
const html = generatedRegistry.render("/cases/reported-timeout", {
|
||||
runId: "scaffold-test",
|
||||
query: new URLSearchParams(),
|
||||
record() {},
|
||||
});
|
||||
assert.match(html, /Quoted `title` \$\{notCode\} <unsafe>/);
|
||||
assert.match(await readFile(result.files.readme, "utf8"), /issue-123/);
|
||||
} finally {
|
||||
await rm(temporaryRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI accepts pnpm separator and supports suite and tag filters", async () => {
|
||||
const cli = resolve("evals/browser/cli.mjs");
|
||||
const coverageResult = await runProcess(process.execPath, [cli, "--", "coverage", "--json"], {
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
assert.equal(coverageResult.exitCode, 0, coverageResult.stderr);
|
||||
assert.equal(JSON.parse(coverageResult.stdout).length, 28);
|
||||
|
||||
const coreResult = await runProcess(
|
||||
process.execPath,
|
||||
[cli, "list", "--suite", "core", "--json"],
|
||||
{
|
||||
timeoutMs: 5_000,
|
||||
},
|
||||
);
|
||||
assert.equal(coreResult.exitCode, 0, coreResult.stderr);
|
||||
assert.equal(JSON.parse(coreResult.stdout).length, 6);
|
||||
|
||||
const formResult = await runProcess(process.execPath, [cli, "list", "--tag", "form", "--json"], {
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
assert.equal(formResult.exitCode, 0, formResult.stderr);
|
||||
assert.deepEqual(
|
||||
JSON.parse(formResult.stdout).map(({ id }) => id),
|
||||
["form-controls", "generated-form"],
|
||||
);
|
||||
});
|
||||
|
||||
test("CLI validation and seed generation are machine-readable", async () => {
|
||||
const cli = resolve("evals/browser/cli.mjs");
|
||||
const validation = await runProcess(process.execPath, [cli, "validate", "--json"], {
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
assert.equal(validation.exitCode, 0, validation.stderr);
|
||||
assert.equal(JSON.parse(validation.stdout).ok, true);
|
||||
|
||||
const generated = await runProcess(
|
||||
process.execPath,
|
||||
[cli, "generate", "--seed", "known-badcase", "--json"],
|
||||
{ timeoutMs: 5_000 },
|
||||
);
|
||||
assert.equal(generated.exitCode, 0, generated.stderr);
|
||||
assert.equal(typeof JSON.parse(generated.stdout).hydrationDelayMs, "number");
|
||||
|
||||
const generatedSet = await runProcess(
|
||||
process.execPath,
|
||||
[cli, "generate", "--seed", "4,7,14", "--json"],
|
||||
{ timeoutMs: 5_000 },
|
||||
);
|
||||
assert.equal(generatedSet.exitCode, 0, generatedSet.stderr);
|
||||
assert.deepEqual(
|
||||
JSON.parse(generatedSet.stdout).map(({ seed }) => seed),
|
||||
[4, 7, 14],
|
||||
);
|
||||
});
|
||||
@@ -4,6 +4,9 @@
|
||||
"packageManager": "pnpm@10.17.0",
|
||||
"scripts": {
|
||||
"cli:build": "cargo run -p bsk-protocol --bin dump-schema --locked && pnpm -r build",
|
||||
"eval:browser": "node evals/browser/cli.mjs",
|
||||
"eval:browser:check": "node evals/browser/cli.mjs validate && node --test evals/browser/tests/*.test.mjs",
|
||||
"eval:browser:test": "node --test evals/browser/tests/*.test.mjs",
|
||||
"ext:build": "pnpm --filter @browser-skill/extension build",
|
||||
"ext:build:zip": "pnpm --filter @browser-skill/extension zip",
|
||||
"ext:dev": "pnpm --filter @browser-skill/extension dev",
|
||||
|
||||
@@ -497,9 +497,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs. Visual-only surfaces require image understanding and their refs are " +
|
||||
"screenshot-only; keep using any DOM/AX semantics rendered alongside them. " +
|
||||
"Read-only: never submits input.";
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
@@ -571,8 +569,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
name: "browser_click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_snapshot/browser_observe, or a CSS selector. A visual Surface requires the " +
|
||||
"captureId and image coordinates returned by browser_screenshot.",
|
||||
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -589,23 +586,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
type: "integer",
|
||||
description: "Number of consecutive presses (double-click = 2).",
|
||||
},
|
||||
modifiers: {
|
||||
type: "array",
|
||||
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
|
||||
description: "Keyboard modifiers held during the click.",
|
||||
},
|
||||
captureId: {
|
||||
type: "string",
|
||||
description: "Short-lived Surface capture id returned by browser_screenshot.",
|
||||
},
|
||||
imageX: {
|
||||
type: "number",
|
||||
description: "X coordinate in capture-image pixels; requires captureId and imageY.",
|
||||
},
|
||||
imageY: {
|
||||
type: "number",
|
||||
description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
@@ -616,7 +596,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
tabId: { type: "integer", required: true },
|
||||
x: { type: "number", required: true },
|
||||
y: { type: "number", required: true },
|
||||
captureId: { type: "string" },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
@@ -628,44 +607,17 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const pointArgCount =
|
||||
Number(args.captureId !== undefined) +
|
||||
Number(args.imageX !== undefined) +
|
||||
Number(args.imageY !== undefined);
|
||||
if (pointArgCount !== 0 && pointArgCount !== 3) {
|
||||
throw new Error("captureId, imageX, and imageY must be given together");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_click");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
||||
cmdArgs.push("--modifiers", args.modifiers.join(","));
|
||||
}
|
||||
if (args.captureId !== undefined) {
|
||||
cmdArgs.push(
|
||||
"--capture",
|
||||
args.captureId,
|
||||
"--image-x",
|
||||
String(args.imageX),
|
||||
"--image-y",
|
||||
String(args.imageY),
|
||||
);
|
||||
}
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
||||
tab_id: number;
|
||||
x: number;
|
||||
y: number;
|
||||
capture_id?: string;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
x: reply.x,
|
||||
y: reply.y,
|
||||
...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
@@ -808,17 +760,12 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
name: "browser_screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"This is the supported operation for visual surface refs emitted by browser_observe. Only use " +
|
||||
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
|
||||
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
|
||||
"not guess content or coordinates. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
ref: {
|
||||
type: "string",
|
||||
description:
|
||||
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
|
||||
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
@@ -832,8 +779,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
byteSize: { type: "integer", required: true },
|
||||
captureId: { type: "string" },
|
||||
captureExpiresAt: { type: "integer" },
|
||||
image: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
@@ -853,11 +798,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
value.image !== undefined
|
||||
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
||||
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
||||
const captureText =
|
||||
value.captureId !== undefined
|
||||
? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
|
||||
: "";
|
||||
const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
|
||||
const blocks: ContentBlock[] = [{ type: "text", text }];
|
||||
if (value.image !== undefined) {
|
||||
blocks.push({
|
||||
type: "image",
|
||||
@@ -896,7 +837,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
height: number;
|
||||
path: string;
|
||||
byte_size: number;
|
||||
capture?: { id: string; expires_at: number };
|
||||
};
|
||||
writtenPath = reply.path;
|
||||
const data = await readFile(reply.path);
|
||||
@@ -910,9 +850,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
width: reply.width,
|
||||
height: reply.height,
|
||||
byteSize: reply.byte_size,
|
||||
...(reply.capture !== undefined
|
||||
? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
|
||||
: {}),
|
||||
...(ref !== undefined
|
||||
? {
|
||||
image: {
|
||||
|
||||
@@ -377,12 +377,12 @@ describe("interaction tools", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
};
|
||||
|
||||
it("browser_click maps target, button, click count, and modifiers", async () => {
|
||||
it("browser_click maps target, button, and click count", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const value = (await click?.execute(
|
||||
{ target: "@e1", button: "right", clickCount: 2, modifiers: ["ctrl", "shift"] },
|
||||
{ target: "@e1", button: "right", clickCount: 2 },
|
||||
makeExec(),
|
||||
)) as { session: string; x: number; y: number };
|
||||
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
|
||||
@@ -394,43 +394,6 @@ describe("interaction tools", () => {
|
||||
"right",
|
||||
"--click-count",
|
||||
"2",
|
||||
"--modifiers",
|
||||
"ctrl,shift",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_click forwards a complete Surface capture coordinate transaction", async () => {
|
||||
const { tools, calls } = setup({
|
||||
...responses,
|
||||
click: {
|
||||
tab_id: 7,
|
||||
used_ref: "e1",
|
||||
x: 110,
|
||||
y: 70,
|
||||
capture_id: "sc_test",
|
||||
image_x: 200,
|
||||
image_y: 100,
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const value = await click?.execute(
|
||||
{ target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
|
||||
makeExec(),
|
||||
);
|
||||
|
||||
expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
|
||||
expect(calls[1].args).toEqual([
|
||||
"click",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"200",
|
||||
"--image-y",
|
||||
"100",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
@@ -851,32 +814,6 @@ describe("browser_screenshot", () => {
|
||||
expect(rendered?.every((block) => block.type === "text")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns Surface capture metadata alongside the screenshot", async () => {
|
||||
const path = pngFile();
|
||||
const { tools } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: {
|
||||
tab_id: 7,
|
||||
width: 800,
|
||||
height: 600,
|
||||
format: "png",
|
||||
path,
|
||||
byte_size: 8,
|
||||
capture: { id: "sc_test", expires_at: 1700000030000 },
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
|
||||
|
||||
expect(value).toMatchObject({
|
||||
captureId: "sc_test",
|
||||
captureExpiresAt: 1700000030000,
|
||||
});
|
||||
const rendered = screenshot?.output.render({ session: "s1" }, value as never);
|
||||
expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
|
||||
});
|
||||
|
||||
it("commits the image through the attachment store on an image-capable route", async () => {
|
||||
const path = pngFile();
|
||||
const services = {
|
||||
|
||||
@@ -29,66 +29,6 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("renders visual surfaces as screenshot-only refs", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 2,
|
||||
backendNodeId: 99,
|
||||
frameId: "child",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
label: "Data grid",
|
||||
memberCount: 2,
|
||||
},
|
||||
];
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
backendNodeId: 99,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("renders an unnamed visual surface with a stable generic label", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
name: "canvas visual surface",
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
@@ -15,8 +15,6 @@ export type {
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomRef,
|
||||
VomRefCapability,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
Loaded 100 of 103 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user