mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c48a53269e | ||
|
|
3bbd6ab19f |
No files matched your search
@@ -247,6 +247,41 @@ describe("App", () => {
|
|||||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||||
|
mockUseConnectionState.mockReturnValue({
|
||||||
|
snapshot: {
|
||||||
|
...baseSnapshot,
|
||||||
|
state: "version_skew",
|
||||||
|
instanceId: "03c3e47f",
|
||||||
|
handshake: {
|
||||||
|
server: "bh",
|
||||||
|
version: mockDaemonVersion,
|
||||||
|
protocol_version: "1.0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
statusState: "version_skew",
|
||||||
|
setLabel,
|
||||||
|
setConnectionEnabled,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
expect(screen.getByText("已连接")).toBeTruthy();
|
||||||
|
expect(screen.getByText("可升级")).toBeTruthy();
|
||||||
|
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||||
|
expect(screen.queryByText("Action needed")).toBeNull();
|
||||||
|
expect(screen.queryByText("兼容")).toBeNull();
|
||||||
|
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||||
|
expect(warning.textContent).toContain("CLI 协议");
|
||||||
|
expect(warning.textContent).toContain("扩展协议");
|
||||||
|
|
||||||
|
openRecordView();
|
||||||
|
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||||
|
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||||
|
fireEvent.click(copyButton);
|
||||||
|
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("control hints toggle", () => {
|
describe("control hints toggle", () => {
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ export function App() {
|
|||||||
}, [copiedTick]);
|
}, [copiedTick]);
|
||||||
|
|
||||||
const isSkewed = statusState === "version_skew";
|
const isSkewed = statusState === "version_skew";
|
||||||
|
const connectionLive = statusState === "connected" || isSkewed;
|
||||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||||
const extensionVersion = snapshot.extensionVersion || "—";
|
const extensionVersion = snapshot.extensionVersion || "—";
|
||||||
@@ -90,7 +91,7 @@ export function App() {
|
|||||||
setCopiedInstanceId(true);
|
setCopiedInstanceId(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||||
const recordPurpose = purposeDraft.trim();
|
const recordPurpose = purposeDraft.trim();
|
||||||
const recordStartUrl = startUrlDraft.trim();
|
const recordStartUrl = startUrlDraft.trim();
|
||||||
const recordCommand = snapshot.instanceId
|
const recordCommand = snapshot.instanceId
|
||||||
@@ -206,12 +207,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
<p
|
<p
|
||||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||||
data-slot="popup-version-skew-warning"
|
data-slot="popup-version-skew-warning"
|
||||||
>
|
>
|
||||||
{t("popup.versionSkewWarning", {
|
{t("popup.versionSkewWarning", {
|
||||||
extensionProtocol: PROTOCOL_VERSION,
|
extensionProtocol: PROTOCOL_VERSION,
|
||||||
daemonProtocol,
|
cliProtocol: daemonProtocol,
|
||||||
})}
|
})}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -13,8 +13,6 @@ describe("RefStore", () => {
|
|||||||
backendNodeId: 42,
|
backendNodeId: 42,
|
||||||
tabId: 7,
|
tabId: 7,
|
||||||
generation: 0,
|
generation: 0,
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
});
|
});
|
||||||
expect(s.size()).toBe(1);
|
expect(s.size()).toBe(1);
|
||||||
expect(s.isEmpty()).toBe(false);
|
expect(s.isEmpty()).toBe(false);
|
||||||
@@ -50,31 +48,4 @@ describe("RefStore", () => {
|
|||||||
cdpSessionId: "child-session",
|
cdpSessionId: "child-session",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("preserves explicit capabilities for visual surface refs", () => {
|
|
||||||
const s = new RefStore();
|
|
||||||
s.set("e1", 42, {
|
|
||||||
tabId: 7,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(s.resolveEntry("e1")).toMatchObject({
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
|
|
||||||
const s = new RefStore();
|
|
||||||
s.set("e1", 42, { tabId: 7, kind: "surface" });
|
|
||||||
s.set("e2", 43, {
|
|
||||||
tabId: 7,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
|
||||||
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
|
|
||||||
|
|
||||||
function input(): SurfaceCaptureInput {
|
|
||||||
return {
|
|
||||||
sessionId: "aa11",
|
|
||||||
tabId: 7,
|
|
||||||
navigationIdentity: "navigation",
|
|
||||||
surface: {
|
|
||||||
ref: "e3",
|
|
||||||
frameId: "main",
|
|
||||||
backendNodeId: 99,
|
|
||||||
observationGeneration: 2,
|
|
||||||
},
|
|
||||||
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
|
|
||||||
imageWidth: 200,
|
|
||||||
imageHeight: 100,
|
|
||||||
viewportSignature: "viewport",
|
|
||||||
frameProjectionSignature: "frame",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("SurfaceCaptureStore", () => {
|
|
||||||
it("creates metadata-only, short-lived captures and consumes them once", () => {
|
|
||||||
let now = 1_000;
|
|
||||||
const store = new SurfaceCaptureStore({
|
|
||||||
now: () => now,
|
|
||||||
ttlMs: 500,
|
|
||||||
createId: () => "sc_test",
|
|
||||||
});
|
|
||||||
const capture = store.create(input());
|
|
||||||
|
|
||||||
expect(capture).toMatchObject({
|
|
||||||
id: "sc_test",
|
|
||||||
createdAt: 1_000,
|
|
||||||
expiresAt: 1_500,
|
|
||||||
consumed: false,
|
|
||||||
});
|
|
||||||
expect(capture).not.toHaveProperty("imageBase64");
|
|
||||||
expect(store.consume("sc_test")).toMatchObject({ ok: true });
|
|
||||||
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
|
|
||||||
|
|
||||||
now = 1_600;
|
|
||||||
expect(store.size()).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects expired and unknown captures", () => {
|
|
||||||
let now = 1_000;
|
|
||||||
const store = new SurfaceCaptureStore({
|
|
||||||
now: () => now,
|
|
||||||
ttlMs: 10,
|
|
||||||
createId: () => "sc_expired",
|
|
||||||
});
|
|
||||||
store.create(input());
|
|
||||||
now = 1_010;
|
|
||||||
|
|
||||||
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
|
|
||||||
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clears every capture on session cleanup", () => {
|
|
||||||
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
|
|
||||||
store.create(input());
|
|
||||||
store.clear();
|
|
||||||
expect(store.size()).toBe(0);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,12 +1,10 @@
|
|||||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||||
import { RefStore } from "./ref-store";
|
import { RefStore } from "./ref-store";
|
||||||
import { SurfaceCaptureStore } from "./surface-capture-store";
|
|
||||||
|
|
||||||
export interface SessionContext {
|
export interface SessionContext {
|
||||||
sessionId: string;
|
sessionId: string;
|
||||||
agentWindowId: number;
|
agentWindowId: number;
|
||||||
refStore: RefStore;
|
refStore: RefStore;
|
||||||
surfaceCaptures: SurfaceCaptureStore;
|
|
||||||
borrowedTabs: Map<number, BorrowedTab>;
|
borrowedTabs: Map<number, BorrowedTab>;
|
||||||
createdAtMs: number;
|
createdAtMs: number;
|
||||||
}
|
}
|
||||||
@@ -188,7 +186,6 @@ export class SessionManager {
|
|||||||
sessionId,
|
sessionId,
|
||||||
agentWindowId: windowId,
|
agentWindowId: windowId,
|
||||||
refStore: new RefStore(),
|
refStore: new RefStore(),
|
||||||
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
|
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
createdAtMs: this.now(),
|
createdAtMs: this.now(),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
import type { Rect } from "@browser-skill/vom";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||||
*
|
*
|
||||||
@@ -14,17 +12,12 @@ import type { Rect } from "@browser-skill/vom";
|
|||||||
* inside the `SessionContext`.
|
* inside the `SessionContext`.
|
||||||
*/
|
*/
|
||||||
export type BackendNodeId = number;
|
export type BackendNodeId = number;
|
||||||
export type RefCapability = "interact" | "screenshot";
|
|
||||||
export type RefTargetKind = "dom" | "surface";
|
|
||||||
|
|
||||||
export interface RefEntry {
|
export interface RefEntry {
|
||||||
backendNodeId: BackendNodeId;
|
backendNodeId: BackendNodeId;
|
||||||
tabId: number | null;
|
tabId: number | null;
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
cdpSessionId?: string;
|
cdpSessionId?: string;
|
||||||
visibleRect?: Rect;
|
|
||||||
kind: RefTargetKind;
|
|
||||||
capabilities: RefCapability[];
|
|
||||||
generation: number;
|
generation: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -35,19 +28,8 @@ export type RefInput =
|
|||||||
tabId: number;
|
tabId: number;
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
cdpSessionId?: string;
|
cdpSessionId?: string;
|
||||||
visibleRect?: Rect;
|
|
||||||
kind?: RefTargetKind;
|
|
||||||
capabilities?: RefCapability[];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
function capabilitiesFor(
|
|
||||||
kind: RefTargetKind,
|
|
||||||
capabilities: RefCapability[] | undefined,
|
|
||||||
): RefCapability[] {
|
|
||||||
if (kind === "surface") return ["screenshot"];
|
|
||||||
return capabilities ?? ["interact", "screenshot"];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RefStore {
|
export class RefStore {
|
||||||
private readonly map = new Map<string, RefEntry>();
|
private readonly map = new Map<string, RefEntry>();
|
||||||
private generation = 0;
|
private generation = 0;
|
||||||
@@ -88,20 +70,13 @@ export class RefStore {
|
|||||||
tabId?: number;
|
tabId?: number;
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
cdpSessionId?: string;
|
cdpSessionId?: string;
|
||||||
visibleRect?: Rect;
|
|
||||||
kind?: RefTargetKind;
|
|
||||||
capabilities?: RefCapability[];
|
|
||||||
} = {},
|
} = {},
|
||||||
): void {
|
): void {
|
||||||
const kind = opts.kind ?? "dom";
|
|
||||||
this.map.set(normaliseRef(ref), {
|
this.map.set(normaliseRef(ref), {
|
||||||
backendNodeId: id,
|
backendNodeId: id,
|
||||||
tabId: opts.tabId ?? null,
|
tabId: opts.tabId ?? null,
|
||||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||||
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
|
|
||||||
kind,
|
|
||||||
capabilities: capabilitiesFor(kind, opts.capabilities),
|
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -119,20 +94,14 @@ export class RefStore {
|
|||||||
return {
|
return {
|
||||||
backendNodeId: input,
|
backendNodeId: input,
|
||||||
tabId: null,
|
tabId: null,
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const kind = input.kind ?? "dom";
|
|
||||||
return {
|
return {
|
||||||
backendNodeId: input.backendNodeId,
|
backendNodeId: input.backendNodeId,
|
||||||
tabId: input.tabId,
|
tabId: input.tabId,
|
||||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||||
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
|
|
||||||
kind,
|
|
||||||
capabilities: capabilitiesFor(kind, input.capabilities),
|
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
import type { Rect } from "@browser-skill/vom";
|
|
||||||
|
|
||||||
export const SURFACE_CAPTURE_TTL_MS = 30_000;
|
|
||||||
|
|
||||||
export interface SurfaceCapture {
|
|
||||||
id: string;
|
|
||||||
sessionId: string;
|
|
||||||
tabId: number;
|
|
||||||
navigationIdentity: string;
|
|
||||||
surface: {
|
|
||||||
ref: string;
|
|
||||||
frameId?: string;
|
|
||||||
backendNodeId: number;
|
|
||||||
observationGeneration: number;
|
|
||||||
};
|
|
||||||
topViewportRect: Rect;
|
|
||||||
imageWidth: number;
|
|
||||||
imageHeight: number;
|
|
||||||
viewportSignature: string;
|
|
||||||
frameProjectionSignature: string;
|
|
||||||
createdAt: number;
|
|
||||||
expiresAt: number;
|
|
||||||
consumed: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type SurfaceCaptureInput = Omit<
|
|
||||||
SurfaceCapture,
|
|
||||||
"id" | "createdAt" | "expiresAt" | "consumed"
|
|
||||||
>;
|
|
||||||
|
|
||||||
export type SurfaceCaptureConsumeResult =
|
|
||||||
| { ok: true; capture: SurfaceCapture }
|
|
||||||
| { ok: false; reason: "not_found" | "expired" | "consumed" };
|
|
||||||
|
|
||||||
export interface SurfaceCaptureStoreOptions {
|
|
||||||
now?: () => number;
|
|
||||||
ttlMs?: number;
|
|
||||||
createId?: () => string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function randomCaptureId(): string {
|
|
||||||
const bytes = crypto.getRandomValues(new Uint8Array(12));
|
|
||||||
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
|
|
||||||
export class SurfaceCaptureStore {
|
|
||||||
private readonly captures = new Map<string, SurfaceCapture>();
|
|
||||||
private readonly now: () => number;
|
|
||||||
private readonly ttlMs: number;
|
|
||||||
private readonly createId: () => string;
|
|
||||||
|
|
||||||
constructor(options: SurfaceCaptureStoreOptions = {}) {
|
|
||||||
this.now = options.now ?? Date.now;
|
|
||||||
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
|
|
||||||
this.createId = options.createId ?? randomCaptureId;
|
|
||||||
}
|
|
||||||
|
|
||||||
create(input: SurfaceCaptureInput): SurfaceCapture {
|
|
||||||
this.purgeExpired();
|
|
||||||
const createdAt = this.now();
|
|
||||||
const capture: SurfaceCapture = {
|
|
||||||
...input,
|
|
||||||
id: this.createId(),
|
|
||||||
createdAt,
|
|
||||||
expiresAt: createdAt + this.ttlMs,
|
|
||||||
consumed: false,
|
|
||||||
};
|
|
||||||
this.captures.set(capture.id, capture);
|
|
||||||
return capture;
|
|
||||||
}
|
|
||||||
|
|
||||||
consume(id: string): SurfaceCaptureConsumeResult {
|
|
||||||
const capture = this.captures.get(id);
|
|
||||||
if (!capture) return { ok: false, reason: "not_found" };
|
|
||||||
if (capture.expiresAt <= this.now()) {
|
|
||||||
this.captures.delete(id);
|
|
||||||
return { ok: false, reason: "expired" };
|
|
||||||
}
|
|
||||||
if (capture.consumed) return { ok: false, reason: "consumed" };
|
|
||||||
capture.consumed = true;
|
|
||||||
return { ok: true, capture };
|
|
||||||
}
|
|
||||||
|
|
||||||
clear(): void {
|
|
||||||
this.captures.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
size(): number {
|
|
||||||
this.purgeExpired();
|
|
||||||
return this.captures.size;
|
|
||||||
}
|
|
||||||
|
|
||||||
private purgeExpired(): void {
|
|
||||||
const now = this.now();
|
|
||||||
for (const [id, capture] of this.captures) {
|
|
||||||
if (capture.expiresAt <= now) this.captures.delete(id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,6 @@ import {
|
|||||||
parseKeySpec,
|
parseKeySpec,
|
||||||
resolveKeyDescriptor,
|
resolveKeyDescriptor,
|
||||||
} from "../interaction";
|
} from "../interaction";
|
||||||
import { captureSurfaceEnvironment } from "../surface-target";
|
|
||||||
|
|
||||||
function fakeAgentWindow(ids: number[]) {
|
function fakeAgentWindow(ids: number[]) {
|
||||||
let i = 0;
|
let i = 0;
|
||||||
@@ -34,15 +33,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
if (!h && method === "Page.getLayoutMetrics") {
|
if (!h && method === "Page.getLayoutMetrics") {
|
||||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
}
|
}
|
||||||
if (
|
|
||||||
!h &&
|
|
||||||
method === "Runtime.evaluate" &&
|
|
||||||
String((params as { expression?: string })?.expression ?? "").includes(
|
|
||||||
"bsk-focus-target-probe",
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
return { result: { value: { hasFocus: true, activeTag: "input" } } };
|
|
||||||
}
|
|
||||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return h(params);
|
return h(params);
|
||||||
};
|
};
|
||||||
@@ -54,10 +44,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
params?: object,
|
params?: object,
|
||||||
) => Promise<T>,
|
) => Promise<T>,
|
||||||
trackSessionTab: vi.fn(),
|
trackSessionTab: vi.fn(),
|
||||||
getFrameGraph: vi.fn(async (tabId) => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [{ frameId: "main", target: { tabId } }],
|
|
||||||
})),
|
|
||||||
};
|
};
|
||||||
const tabsApi = {
|
const tabsApi = {
|
||||||
get: vi.fn(
|
get: vi.fn(
|
||||||
@@ -68,71 +54,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
return { cdp, tabsApi, sent };
|
return { cdp, tabsApi, sent };
|
||||||
}
|
}
|
||||||
|
|
||||||
function fillRuntimeReadback(value: string) {
|
|
||||||
return (params: unknown) => {
|
|
||||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
|
||||||
if (fn.includes("bsk-input-readback")) {
|
|
||||||
return { result: { value: { supported: true, value } } };
|
|
||||||
}
|
|
||||||
return { result: { type: "undefined" } };
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
|
||||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const context = await manager.start("aa11");
|
|
||||||
context.refStore.set("e3", 99, {
|
|
||||||
tabId: 4,
|
|
||||||
kind: "surface",
|
|
||||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
|
||||||
});
|
|
||||||
const fake = makeFakeCdp({
|
|
||||||
"Page.getFrameTree": () => ({
|
|
||||||
frameTree: {
|
|
||||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
"Page.getLayoutMetrics": () => ({
|
|
||||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
|
||||||
cssVisualViewport: {
|
|
||||||
clientWidth: 800,
|
|
||||||
clientHeight: 600,
|
|
||||||
pageX: 0,
|
|
||||||
pageY: 0,
|
|
||||||
scale: 1,
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
|
||||||
"Runtime.evaluate": () => ({
|
|
||||||
result: { value: { overlayHostPresent: false, overlayHostConnected: false } },
|
|
||||||
}),
|
|
||||||
"Input.dispatchMouseEvent": (params) => {
|
|
||||||
onMouse?.(params as Record<string, unknown>);
|
|
||||||
return {};
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const environment = await captureSurfaceEnvironment(fake.cdp, 4, undefined);
|
|
||||||
if ("code" in environment) throw new Error(environment.message);
|
|
||||||
const entry = context.refStore.resolveEntry("e3");
|
|
||||||
if (!entry) throw new Error("missing Surface ref");
|
|
||||||
const capture = context.surfaceCaptures.create({
|
|
||||||
sessionId: "aa11",
|
|
||||||
tabId: 4,
|
|
||||||
navigationIdentity: environment.navigationIdentity,
|
|
||||||
surface: {
|
|
||||||
ref: "e3",
|
|
||||||
backendNodeId: 99,
|
|
||||||
observationGeneration: entry.generation,
|
|
||||||
},
|
|
||||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
|
||||||
imageWidth: 200,
|
|
||||||
imageHeight: 80,
|
|
||||||
viewportSignature: environment.viewportSignature,
|
|
||||||
frameProjectionSignature: environment.frameProjectionSignature,
|
|
||||||
});
|
|
||||||
return { manager, context, fake, capture };
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("modifiersBitfield", () => {
|
describe("modifiersBitfield", () => {
|
||||||
it("matches CDP's expected bit layout", () => {
|
it("matches CDP's expected bit layout", () => {
|
||||||
expect(modifiersBitfield([])).toBe(0);
|
expect(modifiersBitfield([])).toBe(0);
|
||||||
@@ -202,29 +123,6 @@ describe("handleClick", () => {
|
|||||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e3", 1234, {
|
|
||||||
tabId: 4,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
});
|
|
||||||
const fake = makeFakeCdp({});
|
|
||||||
|
|
||||||
const res = await handleClick(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", ref: "@e3" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(res).toMatchObject({
|
|
||||||
code: "permission_denied",
|
|
||||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
|
||||||
});
|
|
||||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
|
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
@@ -269,74 +167,6 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each([
|
|
||||||
"left",
|
|
||||||
"middle",
|
|
||||||
"right",
|
|
||||||
] as const)("uses the common click executor for a %s-button Surface click", async (button) => {
|
|
||||||
const { manager, fake, capture } = await makeSurfaceClickFixture();
|
|
||||||
const result = await handleClick(
|
|
||||||
manager,
|
|
||||||
{
|
|
||||||
session_id: "aa11",
|
|
||||||
ref: "@e3",
|
|
||||||
capture_id: capture.id,
|
|
||||||
image_x: 50,
|
|
||||||
image_y: 20,
|
|
||||||
button,
|
|
||||||
click_count: 2,
|
|
||||||
modifiers: ["ctrl", "shift"],
|
|
||||||
},
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
|
||||||
expect(result).toMatchObject({
|
|
||||||
used_ref: "e3",
|
|
||||||
capture_id: capture.id,
|
|
||||||
image_x: 50,
|
|
||||||
image_y: 20,
|
|
||||||
x: 35,
|
|
||||||
y: 30,
|
|
||||||
});
|
|
||||||
const mouse = fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent");
|
|
||||||
expect(mouse).toHaveLength(3);
|
|
||||||
expect(mouse[1].params).toMatchObject({
|
|
||||||
type: "mousePressed",
|
|
||||||
button,
|
|
||||||
clickCount: 2,
|
|
||||||
modifiers: 2 | 8,
|
|
||||||
});
|
|
||||||
expect(mouse[2].params).toMatchObject({
|
|
||||||
type: "mouseReleased",
|
|
||||||
button,
|
|
||||||
clickCount: 2,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("consumes a Surface capture when the common click executor fails", async () => {
|
|
||||||
let mouseCalls = 0;
|
|
||||||
const { manager, fake, capture } = await makeSurfaceClickFixture((params) => {
|
|
||||||
mouseCalls += 1;
|
|
||||||
if (params.type === "mousePressed") throw new Error("input failed");
|
|
||||||
});
|
|
||||||
const params = {
|
|
||||||
session_id: "aa11",
|
|
||||||
ref: "@e3",
|
|
||||||
capture_id: capture.id,
|
|
||||||
image_x: 50,
|
|
||||||
image_y: 20,
|
|
||||||
};
|
|
||||||
|
|
||||||
expect(
|
|
||||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
|
||||||
).toMatchObject({ code: "cdp_failed" });
|
|
||||||
expect(mouseCalls).toBe(2);
|
|
||||||
expect(
|
|
||||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
|
||||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
@@ -756,7 +586,7 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||||
"Runtime.callFunctionOn": fillRuntimeReadback("hello"),
|
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
@@ -789,9 +619,6 @@ describe("handleFill", () => {
|
|||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||||
"Runtime.callFunctionOn": (p) => {
|
"Runtime.callFunctionOn": (p) => {
|
||||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
if (fn.includes("bsk-input-readback")) {
|
|
||||||
return { result: { value: { supported: true, value: "existingx" } } };
|
|
||||||
}
|
|
||||||
// No "this.value = ''" clearing on a no-clear path; only the
|
// No "this.value = ''" clearing on a no-clear path; only the
|
||||||
// post-input dispatchEvent.
|
// post-input dispatchEvent.
|
||||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||||
@@ -805,7 +632,7 @@ describe("handleFill", () => {
|
|||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
);
|
);
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
expect(res.value_length).toBe(9);
|
expect(res.value_length).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("treats contenteditable=true as fillable", async () => {
|
it("treats contenteditable=true as fillable", async () => {
|
||||||
@@ -823,7 +650,7 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||||
"Runtime.callFunctionOn": fillRuntimeReadback("rich"),
|
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
@@ -834,88 +661,6 @@ describe("handleFill", () => {
|
|||||||
expect("code" in res).toBe(false);
|
expect("code" in res).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e1", 77, {
|
|
||||||
tabId: 4,
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
});
|
|
||||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
|
||||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child-frame",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 12,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}));
|
|
||||||
const childCalls: string[] = [];
|
|
||||||
fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
|
|
||||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
|
||||||
childCalls.push(method);
|
|
||||||
if (method === "DOM.describeNode") {
|
|
||||||
return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
|
|
||||||
if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
|
|
||||||
if (method === "Input.insertText") {
|
|
||||||
expect(params).toEqual({ text: "中文输入" });
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
if (method === "Runtime.callFunctionOn") {
|
|
||||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
|
||||||
return fn.includes("bsk-input-readback")
|
|
||||||
? { result: { value: { supported: true, value: "中文输入" } } }
|
|
||||||
: { result: { type: "undefined" } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child call ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"];
|
|
||||||
|
|
||||||
const res = await handleFill(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", ref: "@e1", value: "中文输入" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in res) throw new Error(JSON.stringify(res));
|
|
||||||
expect(res.value_length).toBe(4);
|
|
||||||
expect(childCalls).toContain("DOM.focus");
|
|
||||||
expect(childCalls).toContain("Input.insertText");
|
|
||||||
expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
|
|
||||||
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("returns input_not_applied when a controlled input resets the value", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e1", 8, { tabId: 4 });
|
|
||||||
const fake = makeFakeCdp({
|
|
||||||
"DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
|
|
||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
|
||||||
"DOM.focus": () => ({}),
|
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
|
|
||||||
"Input.insertText": () => ({}),
|
|
||||||
"Runtime.callFunctionOn": fillRuntimeReadback(""),
|
|
||||||
});
|
|
||||||
|
|
||||||
const res = await handleFill(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", ref: "@e1", value: "rejected" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(res).toMatchObject({
|
|
||||||
code: "cdp_failed",
|
|
||||||
data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
it("stops before Input.insertText when abort fires after clearing", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
@@ -1110,125 +855,6 @@ describe("handlePress", () => {
|
|||||||
expect(res.key).toBe("Enter");
|
expect(res.key).toBe("Enter");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("routes ref-targeted key events through the OOPIF session", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e1", 555, {
|
|
||||||
tabId: 4,
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
});
|
|
||||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
|
||||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child-frame",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 12,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}));
|
|
||||||
const childCalls: string[] = [];
|
|
||||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
|
||||||
childCalls.push(method);
|
|
||||||
return {};
|
|
||||||
}) as CdpRunner["sendToTarget"];
|
|
||||||
|
|
||||||
const res = await handlePress(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", key: "Enter", ref: "@e1" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
expectPressOk(res);
|
|
||||||
expect(childCalls).toEqual([
|
|
||||||
"DOM.scrollIntoViewIfNeeded",
|
|
||||||
"DOM.focus",
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
]);
|
|
||||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
await sm.start("aa11");
|
|
||||||
const fake = makeFakeCdp({
|
|
||||||
"Runtime.evaluate": () => ({
|
|
||||||
result: { value: { hasFocus: true, activeTag: "iframe" } },
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child-frame",
|
|
||||||
parentFrameId: "main",
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}));
|
|
||||||
const childCalls: string[] = [];
|
|
||||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
|
||||||
childCalls.push(method);
|
|
||||||
if (method === "Runtime.evaluate") {
|
|
||||||
return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}) as CdpRunner["sendToTarget"];
|
|
||||||
|
|
||||||
const res = await handlePress(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", key: "a" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
expectPressOk(res);
|
|
||||||
expect(childCalls).toEqual([
|
|
||||||
"Runtime.evaluate",
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("refuses an untargeted press when focus ownership is ambiguous", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
await sm.start("aa11");
|
|
||||||
const fake = makeFakeCdp({
|
|
||||||
"Runtime.evaluate": () => ({
|
|
||||||
result: { value: { hasFocus: true, activeTag: "input" } },
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{ frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
|
|
||||||
],
|
|
||||||
}));
|
|
||||||
fake.cdp.sendToTarget = vi.fn(async () => ({
|
|
||||||
result: { value: { hasFocus: true, activeTag: "textarea" } },
|
|
||||||
})) as CdpRunner["sendToTarget"];
|
|
||||||
|
|
||||||
const res = await handlePress(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", key: "Enter" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(res).toMatchObject({
|
|
||||||
code: "cdp_failed",
|
|
||||||
data: { reason: "focus_target_unresolved", candidate_count: 2 },
|
|
||||||
});
|
|
||||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("stops before focus when abort fires after press scroll", async () => {
|
it("stops before focus when abort fires after press scroll", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -1,119 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import { dispatchMouseClick } from "../mouse-input";
|
|
||||||
import type { CdpRunner } from "../shared";
|
|
||||||
|
|
||||||
function fakeCdp(onSend?: (method: string, params: Record<string, unknown>) => void): {
|
|
||||||
cdp: CdpRunner;
|
|
||||||
events: Record<string, unknown>[];
|
|
||||||
} {
|
|
||||||
const events: Record<string, unknown>[] = [];
|
|
||||||
return {
|
|
||||||
events,
|
|
||||||
cdp: {
|
|
||||||
send: vi.fn(async (_tabId, method, params) => {
|
|
||||||
const event = params as Record<string, unknown>;
|
|
||||||
events.push(event);
|
|
||||||
onSend?.(method, event);
|
|
||||||
return {} as never;
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("dispatchMouseClick", () => {
|
|
||||||
it("emits one move, press, and release sequence", async () => {
|
|
||||||
const { cdp, events } = fakeCdp();
|
|
||||||
|
|
||||||
expect(
|
|
||||||
await dispatchMouseClick(
|
|
||||||
cdp,
|
|
||||||
4,
|
|
||||||
{ x: 12, y: 34 },
|
|
||||||
{
|
|
||||||
button: "left",
|
|
||||||
clickCount: 1,
|
|
||||||
modifiers: 0,
|
|
||||||
moveSettleMs: 0,
|
|
||||||
},
|
|
||||||
),
|
|
||||||
).toBe("completed");
|
|
||||||
|
|
||||||
expect(events).toEqual([
|
|
||||||
{
|
|
||||||
type: "mouseMoved",
|
|
||||||
x: 12,
|
|
||||||
y: 34,
|
|
||||||
modifiers: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: "mousePressed",
|
|
||||||
x: 12,
|
|
||||||
y: 34,
|
|
||||||
button: "left",
|
|
||||||
clickCount: 1,
|
|
||||||
modifiers: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: "mouseReleased",
|
|
||||||
x: 12,
|
|
||||||
y: 34,
|
|
||||||
button: "left",
|
|
||||||
clickCount: 1,
|
|
||||||
modifiers: 0,
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("releases a pressed button when cancellation arrives", async () => {
|
|
||||||
const controller = new AbortController();
|
|
||||||
const { cdp, events } = fakeCdp((_method, event) => {
|
|
||||||
if (event.type === "mousePressed") controller.abort();
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(
|
|
||||||
await dispatchMouseClick(
|
|
||||||
cdp,
|
|
||||||
4,
|
|
||||||
{ x: 1, y: 2 },
|
|
||||||
{
|
|
||||||
button: "left",
|
|
||||||
clickCount: 1,
|
|
||||||
modifiers: 0,
|
|
||||||
signal: controller.signal,
|
|
||||||
moveSettleMs: 0,
|
|
||||||
},
|
|
||||||
),
|
|
||||||
).toBe("cancelled");
|
|
||||||
expect(events.at(-1)).toMatchObject({ type: "mouseReleased" });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("settles pointer-move hit testing before pressing", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
try {
|
|
||||||
const { cdp, events } = fakeCdp();
|
|
||||||
const click = dispatchMouseClick(
|
|
||||||
cdp,
|
|
||||||
4,
|
|
||||||
{ x: 1, y: 2 },
|
|
||||||
{
|
|
||||||
button: "left",
|
|
||||||
clickCount: 1,
|
|
||||||
modifiers: 0,
|
|
||||||
moveSettleMs: 32,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(31);
|
|
||||||
expect(events.map((event) => event.type)).toEqual(["mouseMoved"]);
|
|
||||||
await vi.advanceTimersByTimeAsync(1);
|
|
||||||
await click;
|
|
||||||
expect(events.map((event) => event.type)).toEqual([
|
|
||||||
"mouseMoved",
|
|
||||||
"mousePressed",
|
|
||||||
"mouseReleased",
|
|
||||||
]);
|
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -67,9 +67,6 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
|||||||
if (!handler && method === "Page.getLayoutMetrics") {
|
if (!handler && method === "Page.getLayoutMetrics") {
|
||||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
}
|
}
|
||||||
if (!handler && method === "Page.getFrameTree") {
|
|
||||||
return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } };
|
|
||||||
}
|
|
||||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return handler(params);
|
return handler(params);
|
||||||
});
|
});
|
||||||
@@ -303,82 +300,6 @@ describe("handleScreenshot", () => {
|
|||||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e5", 999, {
|
|
||||||
tabId: 7,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
});
|
|
||||||
const { cdp, sent } = makeFakeCdp({
|
|
||||||
"Page.getLayoutMetrics": () => ({
|
|
||||||
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
|
|
||||||
}),
|
|
||||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
|
|
||||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
|
||||||
});
|
|
||||||
|
|
||||||
const res = await handleScreenshot(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
|
||||||
makeScreenshotDeps({
|
|
||||||
cdp,
|
|
||||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
|
||||||
query: vi.fn(),
|
|
||||||
captureVisibleTab: vi.fn(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
|
||||||
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
|
|
||||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
|
||||||
clip?: { width: number; height: number; scale: number };
|
|
||||||
};
|
|
||||||
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
|
|
||||||
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
|
|
||||||
expect(res.capture).toMatchObject({
|
|
||||||
surface_ref: "@e5",
|
|
||||||
coordinate_space: "capture-image-pixel",
|
|
||||||
});
|
|
||||||
expect(ctx.surfaceCaptures.size()).toBe(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("crops a visual surface screenshot to its observation-time visible region", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e5", 999, {
|
|
||||||
tabId: 7,
|
|
||||||
kind: "surface",
|
|
||||||
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
|
|
||||||
});
|
|
||||||
const { cdp, sent } = makeFakeCdp({
|
|
||||||
"Page.getLayoutMetrics": () => ({
|
|
||||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
|
||||||
}),
|
|
||||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
|
||||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
|
||||||
});
|
|
||||||
|
|
||||||
const res = await handleScreenshot(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
|
||||||
makeScreenshotDeps({
|
|
||||||
cdp,
|
|
||||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
|
||||||
query: vi.fn(),
|
|
||||||
captureVisibleTab: vi.fn(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
|
||||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
|
||||||
clip?: { x: number; y: number; width: number; height: number };
|
|
||||||
};
|
|
||||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
|
||||||
expect(res.capture?.id).toMatch(/^sc_/);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
await sm.start("aa11");
|
await sm.start("aa11");
|
||||||
@@ -1735,7 +1656,7 @@ describe("buildVomScene", () => {
|
|||||||
|
|
||||||
const scene = buildVomScene(axNodes, captured);
|
const scene = buildVomScene(axNodes, captured);
|
||||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||||
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
|
expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
|
||||||
);
|
);
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "close"');
|
expect(rendered.text).toContain('@e1 button "close"');
|
||||||
@@ -1811,7 +1732,7 @@ describe("buildVomScene", () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const scene = buildVomScene(axNodes, captured);
|
const scene = buildVomScene(axNodes, captured);
|
||||||
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
|
expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
|
||||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1880,9 +1801,9 @@ describe("buildVomScene", () => {
|
|||||||
|
|
||||||
const scene = buildVomScene(axNodes, captured);
|
const scene = buildVomScene(axNodes, captured);
|
||||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||||
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
|
expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
|
||||||
);
|
);
|
||||||
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
|
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
@@ -2210,7 +2131,7 @@ describe("buildVomScene", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(scene.surfaces).toEqual([
|
expect(scene.surfaces).toEqual([
|
||||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||||
]);
|
]);
|
||||||
expect(renderVom(scene).text).toContain(
|
expect(renderVom(scene).text).toContain(
|
||||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||||
@@ -2772,85 +2693,6 @@ describe("handleSnapshot", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
const root: CdpAxNode = {
|
|
||||||
nodeId: "1",
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
name: { type: "computedString", value: "Canvas app" },
|
|
||||||
backendDOMNodeId: 100,
|
|
||||||
};
|
|
||||||
const strings = ["body", "canvas", "static", "auto"];
|
|
||||||
const i = (value: string) => strings.indexOf(value);
|
|
||||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
|
||||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
|
||||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
|
||||||
}
|
|
||||||
if (method === "DOMSnapshot.captureSnapshot") {
|
|
||||||
return {
|
|
||||||
strings,
|
|
||||||
documents: [
|
|
||||||
{
|
|
||||||
nodes: {
|
|
||||||
parentIndex: [-1, 0],
|
|
||||||
nodeName: [i("body"), i("canvas")],
|
|
||||||
backendNodeId: [100, 200],
|
|
||||||
attributes: [[], []],
|
|
||||||
},
|
|
||||||
layout: {
|
|
||||||
nodeIndex: [0, 1],
|
|
||||||
styles: [
|
|
||||||
[i("static"), i("auto"), i("auto")],
|
|
||||||
[i("static"), i("auto"), i("auto")],
|
|
||||||
],
|
|
||||||
bounds: [
|
|
||||||
[0, 0, 1000, 800],
|
|
||||||
[20, 30, 800, 500],
|
|
||||||
],
|
|
||||||
paintOrders: [0, 1],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected CDP method ${method}`);
|
|
||||||
});
|
|
||||||
const deps = {
|
|
||||||
cdp: {
|
|
||||||
send: send as unknown as <T = unknown>(
|
|
||||||
tabId: number,
|
|
||||||
method: string,
|
|
||||||
params?: object,
|
|
||||||
) => Promise<T>,
|
|
||||||
trackSessionTab: vi.fn(),
|
|
||||||
},
|
|
||||||
tabsApi: {
|
|
||||||
get: vi.fn(
|
|
||||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
|
||||||
),
|
|
||||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
|
||||||
},
|
|
||||||
conditionalSurfaceProbe: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
|
|
||||||
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
|
|
||||||
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
|
|
||||||
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
|
|
||||||
backendNodeId: 200,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
});
|
|
||||||
|
|
||||||
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
|
||||||
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
|
|
||||||
expect(snapshot.text).not.toContain("visual surface");
|
|
||||||
expect(snapshot.ref_count).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
await sm.start("aa11");
|
await sm.start("aa11");
|
||||||
@@ -3313,98 +3155,6 @@ describe("handleSnapshot", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("decorates live observations with active controlled content", async () => {
|
|
||||||
const strings = [
|
|
||||||
"html",
|
|
||||||
"body",
|
|
||||||
"button",
|
|
||||||
"div",
|
|
||||||
"#text",
|
|
||||||
"role",
|
|
||||||
"tab",
|
|
||||||
"aria-selected",
|
|
||||||
"true",
|
|
||||||
"aria-controls",
|
|
||||||
"reviews-panel",
|
|
||||||
"id",
|
|
||||||
"static",
|
|
||||||
"auto",
|
|
||||||
"visible",
|
|
||||||
"1",
|
|
||||||
"A detailed review",
|
|
||||||
];
|
|
||||||
const index = (value: string) => strings.indexOf(value);
|
|
||||||
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
|
|
||||||
const snapshot = {
|
|
||||||
strings,
|
|
||||||
documents: [
|
|
||||||
{
|
|
||||||
nodes: {
|
|
||||||
parentIndex: [-1, 0, 1, 1, 3],
|
|
||||||
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
|
|
||||||
backendNodeId: [10, 11, 12, 13, 14],
|
|
||||||
attributes: [
|
|
||||||
[],
|
|
||||||
[],
|
|
||||||
[
|
|
||||||
index("role"),
|
|
||||||
index("tab"),
|
|
||||||
index("aria-selected"),
|
|
||||||
index("true"),
|
|
||||||
index("aria-controls"),
|
|
||||||
index("reviews-panel"),
|
|
||||||
],
|
|
||||||
[index("id"), index("reviews-panel")],
|
|
||||||
[],
|
|
||||||
],
|
|
||||||
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
|
|
||||||
},
|
|
||||||
layout: {
|
|
||||||
nodeIndex: [0, 1, 2, 3, 4],
|
|
||||||
styles: [style, style, style, style, style],
|
|
||||||
bounds: [
|
|
||||||
[0, 0, 1000, 800],
|
|
||||||
[0, 0, 1000, 800],
|
|
||||||
[20, 20, 120, 40],
|
|
||||||
[20, 80, 500, 200],
|
|
||||||
[20, 80, 200, 20],
|
|
||||||
],
|
|
||||||
paintOrders: [0, 0, 1, 1, 1],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
const ax: CdpAxNode[] = [
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 11,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["reviews"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "reviews",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 12,
|
|
||||||
role: { type: "role", value: "tab" },
|
|
||||||
name: { type: "computedString", value: "Reviews" },
|
|
||||||
properties: [
|
|
||||||
{ name: "selected", value: { value: true } },
|
|
||||||
{ name: "controls", value: { value: "reviews-panel" } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const result = await captureVomObservation(
|
|
||||||
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
|
|
||||||
4,
|
|
||||||
"https://example.com",
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(result.text).toContain('@e1 tab "Reviews"');
|
|
||||||
expect(result.text).toContain("[§ active: Reviews]");
|
|
||||||
expect(result.text).toContain("A detailed review");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||||
const secrets = {
|
const secrets = {
|
||||||
email: "user@example.com",
|
email: "user@example.com",
|
||||||
|
|||||||
@@ -252,23 +252,12 @@ describe("handleSessionStop with auto-return", () => {
|
|||||||
expect(aw.remove).not.toHaveBeenCalled();
|
expect(aw.remove).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clears refs and Surface captures before window teardown", async () => {
|
it("clears the RefStore before window teardown", async () => {
|
||||||
const aw = fakeAgentWindow([100]);
|
const aw = fakeAgentWindow([100]);
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
// Insert a fake ref so we can verify clear() ran.
|
// Insert a fake ref so we can verify clear() ran.
|
||||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||||
ctx.surfaceCaptures.create({
|
|
||||||
sessionId: "aa11",
|
|
||||||
tabId: 7,
|
|
||||||
navigationIdentity: "navigation",
|
|
||||||
surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 },
|
|
||||||
topViewportRect: { x: 0, y: 0, w: 100, h: 50 },
|
|
||||||
imageWidth: 100,
|
|
||||||
imageHeight: 50,
|
|
||||||
viewportSignature: "viewport",
|
|
||||||
frameProjectionSignature: "frame",
|
|
||||||
});
|
|
||||||
const state: FakeState = {
|
const state: FakeState = {
|
||||||
tabs: new Map(),
|
tabs: new Map(),
|
||||||
windowsClosed: new Set(),
|
windowsClosed: new Set(),
|
||||||
@@ -277,6 +266,5 @@ describe("handleSessionStop with auto-return", () => {
|
|||||||
const { tabs, windows } = makeApis(state);
|
const { tabs, windows } = makeApis(state);
|
||||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||||
expect(ctx.surfaceCaptures.size()).toBe(0);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -24,16 +24,10 @@ describe("lookupSnapshotRef", () => {
|
|||||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
|
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||||
backendNodeId: 1234,
|
backendNodeId: 1234,
|
||||||
refKey: "e3",
|
refKey: "e3",
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
generation: 0,
|
|
||||||
});
|
});
|
||||||
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
|
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
|
||||||
backendNodeId: 1234,
|
backendNodeId: 1234,
|
||||||
refKey: "e3",
|
refKey: "e3",
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
generation: 0,
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -59,8 +53,6 @@ describe("resolveSnapshotRef", () => {
|
|||||||
tabId: 4,
|
tabId: 4,
|
||||||
frameId: "child-frame",
|
frameId: "child-frame",
|
||||||
cdpSessionId: "child-session",
|
cdpSessionId: "child-session",
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const expected = {
|
const expected = {
|
||||||
@@ -68,9 +60,6 @@ describe("resolveSnapshotRef", () => {
|
|||||||
refKey: "e3",
|
refKey: "e3",
|
||||||
frameId: "child-frame",
|
frameId: "child-frame",
|
||||||
cdpSessionId: "child-session",
|
cdpSessionId: "child-session",
|
||||||
kind: "dom" as const,
|
|
||||||
capabilities: ["interact", "screenshot"] as const,
|
|
||||||
generation: 0,
|
|
||||||
};
|
};
|
||||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||||
@@ -116,29 +105,6 @@ describe("resolveSnapshotRef", () => {
|
|||||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
|
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||||
backendNodeId: 1234,
|
backendNodeId: 1234,
|
||||||
refKey: "e3",
|
refKey: "e3",
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
generation: 0,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects an operation outside the ref's declared capabilities", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e3", 1234, {
|
|
||||||
tabId: 4,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
|
|
||||||
code: "permission_denied",
|
|
||||||
message: "ref @e3 does not support interact",
|
|
||||||
data: {
|
|
||||||
reason: "ref_capability_denied",
|
|
||||||
required_capability: "interact",
|
|
||||||
kind: "surface",
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import {
|
|
||||||
mapImagePointToViewport,
|
|
||||||
pointInRegion,
|
|
||||||
sameRect,
|
|
||||||
surfaceVisibleRect,
|
|
||||||
} from "../surface-coordinate";
|
|
||||||
|
|
||||||
describe("surface coordinate mapping", () => {
|
|
||||||
it("maps capture image pixels through the actual screenshot dimensions", () => {
|
|
||||||
expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({
|
|
||||||
x: 200,
|
|
||||||
y: 100,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects invalid image coordinates and dimensions", () => {
|
|
||||||
const rect = { x: 0, y: 0, w: 100, h: 50 };
|
|
||||||
expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull();
|
|
||||||
expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull();
|
|
||||||
expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull();
|
|
||||||
expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull();
|
|
||||||
expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull();
|
|
||||||
expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("intersects live and observed visible regions and compares them strictly", () => {
|
|
||||||
const clipped = surfaceVisibleRect(
|
|
||||||
{ x: 0, y: 0, width: 200, height: 100 },
|
|
||||||
{ x: 25, y: 30, w: 50, h: 40 },
|
|
||||||
);
|
|
||||||
expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 });
|
|
||||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 25, y: 30, w: 50, h: 40 })).toBe(
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 26, y: 30, w: 50, h: 40 })).toBe(
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("checks the projected visible region rather than only its bounding box", () => {
|
|
||||||
const triangle = [
|
|
||||||
[
|
|
||||||
{ x: 0, y: 0 },
|
|
||||||
{ x: 100, y: 0 },
|
|
||||||
{ x: 0, y: 100 },
|
|
||||||
],
|
|
||||||
];
|
|
||||||
expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true);
|
|
||||||
expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,306 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
|
||||||
import type { CdpRunner } from "../shared";
|
|
||||||
import {
|
|
||||||
captureSurfaceEnvironment,
|
|
||||||
resolveSurfacePointerTarget,
|
|
||||||
SURFACE_POINTER_MOVE_SETTLE_MS,
|
|
||||||
} from "../surface-target";
|
|
||||||
|
|
||||||
function fakeAgentWindow() {
|
|
||||||
return {
|
|
||||||
create: vi.fn(async () => 100),
|
|
||||||
remove: vi.fn(async () => {}),
|
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function fakeBrowser() {
|
|
||||||
let loaderId = "loader-1";
|
|
||||||
let pageY = 0;
|
|
||||||
let quad = [10, 20, 110, 20, 110, 60, 10, 60];
|
|
||||||
const sent: Array<{ method: string; params?: object }> = [];
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
|
||||||
sent.push({ method, params });
|
|
||||||
if (method === "Page.getFrameTree") {
|
|
||||||
return {
|
|
||||||
frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } },
|
|
||||||
} as never;
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return {
|
|
||||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY },
|
|
||||||
cssVisualViewport: {
|
|
||||||
clientWidth: 800,
|
|
||||||
clientHeight: 600,
|
|
||||||
pageX: 0,
|
|
||||||
pageY,
|
|
||||||
scale: 1,
|
|
||||||
},
|
|
||||||
} as never;
|
|
||||||
}
|
|
||||||
if (method === "DOM.getContentQuads") return { quads: [quad] } as never;
|
|
||||||
throw new Error(`unexpected CDP call ${method}`);
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
trackSessionTab: vi.fn(),
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
cdp,
|
|
||||||
sent,
|
|
||||||
setLoaderId: (value: string) => {
|
|
||||||
loaderId = value;
|
|
||||||
},
|
|
||||||
setPageY: (value: number) => {
|
|
||||||
pageY = value;
|
|
||||||
},
|
|
||||||
setQuad: (value: number[]) => {
|
|
||||||
quad = value;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function setupCapture() {
|
|
||||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
|
||||||
const context = await manager.start("aa11");
|
|
||||||
context.refStore.set("e3", 99, {
|
|
||||||
tabId: 4,
|
|
||||||
kind: "surface",
|
|
||||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
|
||||||
});
|
|
||||||
const browser = fakeBrowser();
|
|
||||||
const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined);
|
|
||||||
if ("code" in environment) throw new Error(environment.message);
|
|
||||||
const entry = context.refStore.resolveEntry("e3");
|
|
||||||
if (!entry) throw new Error("missing ref");
|
|
||||||
const capture = context.surfaceCaptures.create({
|
|
||||||
sessionId: "aa11",
|
|
||||||
tabId: 4,
|
|
||||||
navigationIdentity: environment.navigationIdentity,
|
|
||||||
surface: {
|
|
||||||
ref: "e3",
|
|
||||||
backendNodeId: 99,
|
|
||||||
observationGeneration: entry.generation,
|
|
||||||
},
|
|
||||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
|
||||||
imageWidth: 200,
|
|
||||||
imageHeight: 80,
|
|
||||||
viewportSignature: environment.viewportSignature,
|
|
||||||
frameProjectionSignature: environment.frameProjectionSignature,
|
|
||||||
});
|
|
||||||
return { manager, context, browser, capture };
|
|
||||||
}
|
|
||||||
|
|
||||||
function pointParams(captureId: string) {
|
|
||||||
return {
|
|
||||||
ref: "@e3",
|
|
||||||
captureId,
|
|
||||||
imageX: 50,
|
|
||||||
imageY: 20,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("Surface screenshot-bound pointer target", () => {
|
|
||||||
it("maps one fresh capture coordinate without executing an action", async () => {
|
|
||||||
const { context, browser, capture } = await setupCapture();
|
|
||||||
const result = await resolveSurfacePointerTarget(
|
|
||||||
context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
pointParams(capture.id),
|
|
||||||
{ cdp: browser.cdp },
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
|
||||||
expect(result).toMatchObject({
|
|
||||||
usedRef: "e3",
|
|
||||||
point: { x: 35, y: 30 },
|
|
||||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
|
||||||
capture: { id: capture.id, imageX: 50, imageY: 20 },
|
|
||||||
});
|
|
||||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
|
||||||
|
|
||||||
const repeated = await resolveSurfacePointerTarget(
|
|
||||||
context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
pointParams(capture.id),
|
|
||||||
{ cdp: browser.cdp },
|
|
||||||
);
|
|
||||||
expect(repeated).toMatchObject({
|
|
||||||
code: "permission_denied",
|
|
||||||
data: { reason: "surface_capture_consumed" },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects a new observation generation before sending input", async () => {
|
|
||||||
const { context, browser, capture } = await setupCapture();
|
|
||||||
context.refStore.replace([
|
|
||||||
[
|
|
||||||
"e3",
|
|
||||||
{
|
|
||||||
backendNodeId: 99,
|
|
||||||
tabId: 4,
|
|
||||||
kind: "surface" as const,
|
|
||||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
]);
|
|
||||||
|
|
||||||
const result = await resolveSurfacePointerTarget(
|
|
||||||
context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
pointParams(capture.id),
|
|
||||||
{ cdp: browser.cdp },
|
|
||||||
);
|
|
||||||
expect(result).toMatchObject({
|
|
||||||
code: "permission_denied",
|
|
||||||
data: { reason: "surface_capture_stale" },
|
|
||||||
});
|
|
||||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects navigation, scroll, and visible geometry changes", async () => {
|
|
||||||
for (const mutate of [
|
|
||||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setLoaderId("loader-2"),
|
|
||||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setPageY(20),
|
|
||||||
(browser: ReturnType<typeof fakeBrowser>) =>
|
|
||||||
browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]),
|
|
||||||
]) {
|
|
||||||
const { context, browser, capture } = await setupCapture();
|
|
||||||
mutate(browser);
|
|
||||||
const result = await resolveSurfacePointerTarget(
|
|
||||||
context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
pointParams(capture.id),
|
|
||||||
{ cdp: browser.cdp },
|
|
||||||
);
|
|
||||||
expect(result).toMatchObject({
|
|
||||||
code: "permission_denied",
|
|
||||||
data: { reason: "surface_capture_stale" },
|
|
||||||
});
|
|
||||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it("consumes the capture when coordinate validation fails", async () => {
|
|
||||||
const invalid = await setupCapture();
|
|
||||||
const outside = await resolveSurfacePointerTarget(
|
|
||||||
invalid.context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
{ ...pointParams(invalid.capture.id), imageX: 200 },
|
|
||||||
{ cdp: invalid.browser.cdp },
|
|
||||||
);
|
|
||||||
expect(outside).toMatchObject({
|
|
||||||
code: "invalid_params",
|
|
||||||
data: { reason: "surface_coordinate_invalid" },
|
|
||||||
});
|
|
||||||
expect(
|
|
||||||
await resolveSurfacePointerTarget(
|
|
||||||
invalid.context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
pointParams(invalid.capture.id),
|
|
||||||
{ cdp: invalid.browser.cdp },
|
|
||||||
),
|
|
||||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("maps image coordinates through an OOPIF frame projection", async () => {
|
|
||||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
|
||||||
const context = await manager.start("aa11");
|
|
||||||
context.refStore.set("e3", 99, {
|
|
||||||
tabId: 4,
|
|
||||||
frameId: "child",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
kind: "surface",
|
|
||||||
visibleRect: { x: 120, y: 110, w: 100, h: 40 },
|
|
||||||
});
|
|
||||||
const graph = {
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 77,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
const sent: Array<{ method: string; params?: object }> = [];
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
|
||||||
sent.push({ method, params });
|
|
||||||
if (method === "Page.getFrameTree") {
|
|
||||||
return {
|
|
||||||
frameTree: {
|
|
||||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
|
||||||
},
|
|
||||||
} as never;
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return {
|
|
||||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
|
||||||
cssVisualViewport: {
|
|
||||||
clientWidth: 800,
|
|
||||||
clientHeight: 600,
|
|
||||||
pageX: 0,
|
|
||||||
pageY: 0,
|
|
||||||
scale: 1,
|
|
||||||
},
|
|
||||||
} as never;
|
|
||||||
}
|
|
||||||
if (method === "DOM.getBoxModel") {
|
|
||||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never;
|
|
||||||
}
|
|
||||||
if (method === "Input.dispatchMouseEvent") return {} as never;
|
|
||||||
throw new Error(`unexpected root CDP call ${method}`);
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never;
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never;
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child CDP call ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
getFrameGraph: vi.fn(async () => graph),
|
|
||||||
trackSessionTab: vi.fn(),
|
|
||||||
};
|
|
||||||
const environment = await captureSurfaceEnvironment(cdp, 4, "child");
|
|
||||||
if ("code" in environment) throw new Error(environment.message);
|
|
||||||
const entry = context.refStore.resolveEntry("e3");
|
|
||||||
if (!entry) throw new Error("missing ref");
|
|
||||||
const capture = context.surfaceCaptures.create({
|
|
||||||
sessionId: "aa11",
|
|
||||||
tabId: 4,
|
|
||||||
navigationIdentity: environment.navigationIdentity,
|
|
||||||
surface: {
|
|
||||||
ref: "e3",
|
|
||||||
frameId: "child",
|
|
||||||
backendNodeId: 99,
|
|
||||||
observationGeneration: entry.generation,
|
|
||||||
},
|
|
||||||
topViewportRect: { x: 120, y: 110, w: 100, h: 40 },
|
|
||||||
imageWidth: 200,
|
|
||||||
imageHeight: 80,
|
|
||||||
viewportSignature: environment.viewportSignature,
|
|
||||||
frameProjectionSignature: environment.frameProjectionSignature,
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await resolveSurfacePointerTarget(
|
|
||||||
context,
|
|
||||||
{ tabId: 4 },
|
|
||||||
{
|
|
||||||
ref: "@e3",
|
|
||||||
captureId: capture.id,
|
|
||||||
imageX: 100,
|
|
||||||
imageY: 40,
|
|
||||||
},
|
|
||||||
{ cdp },
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
|
||||||
expect(result).toMatchObject({ point: { x: 170, y: 130 } });
|
|
||||||
expect(sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -11,7 +11,7 @@
|
|||||||
// commands.
|
// commands.
|
||||||
|
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
ClickParams,
|
ClickParams,
|
||||||
@@ -32,7 +32,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
|
|||||||
import { backendNodeToObject } from "./element-geometry";
|
import { backendNodeToObject } from "./element-geometry";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||||
import { dispatchMouseClick } from "./mouse-input";
|
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
@@ -44,7 +43,6 @@ import {
|
|||||||
resolveTargetTab,
|
resolveTargetTab,
|
||||||
} from "./shared";
|
} from "./shared";
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
|
||||||
|
|
||||||
export interface InteractionDeps {
|
export interface InteractionDeps {
|
||||||
cdp: CdpRunner;
|
cdp: CdpRunner;
|
||||||
@@ -161,7 +159,7 @@ async function resolveBackendNode(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (hasRef) {
|
if (hasRef) {
|
||||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
|
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||||
if (isRpcError(resolved)) return resolved;
|
if (isRpcError(resolved)) return resolved;
|
||||||
return {
|
return {
|
||||||
backendNodeId: resolved.backendNodeId,
|
backendNodeId: resolved.backendNodeId,
|
||||||
@@ -217,131 +215,15 @@ async function resolveBackendNode(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
interface InputDispatchContext {
|
|
||||||
target: CdpTarget;
|
|
||||||
cdp: CdpRunner;
|
|
||||||
backendNodeId?: number;
|
|
||||||
frameId?: string;
|
|
||||||
usedRef?: string;
|
|
||||||
usedSelector?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function inputContextForNode(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
node: Awaited<ReturnType<typeof resolveBackendNode>>,
|
|
||||||
): (InputDispatchContext & { backendNodeId: number }) | RpcError {
|
|
||||||
if (isRpcError(node)) return node;
|
|
||||||
return {
|
|
||||||
target: node.cdpTarget,
|
|
||||||
cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
|
|
||||||
backendNodeId: node.backendNodeId,
|
|
||||||
frameId: node.frameId,
|
|
||||||
usedRef: node.usedRef,
|
|
||||||
usedSelector: node.usedSelector,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FocusProbe {
|
|
||||||
hasFocus: boolean;
|
|
||||||
activeTag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
|
||||||
async function resolveFocusedInputContext(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
): Promise<InputDispatchContext | RpcError> {
|
|
||||||
if (!cdp.getFrameGraph) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"focus_target_unresolved",
|
|
||||||
"press cannot determine the focused frame target",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let targets: CdpTarget[];
|
|
||||||
try {
|
|
||||||
const graph = await cdp.getFrameGraph(tabId);
|
|
||||||
const unique = new Map<string, CdpTarget>();
|
|
||||||
unique.set(cdpTargetKey({ tabId }), { tabId });
|
|
||||||
for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
|
|
||||||
targets = [...unique.values()];
|
|
||||||
} catch (err) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"focus_target_unresolved",
|
|
||||||
err instanceof Error ? err.message : String(err),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
|
|
||||||
await Promise.all(
|
|
||||||
targets.map(async (candidate) => {
|
|
||||||
try {
|
|
||||||
const reply = await cdpRunnerForTarget(cdp, candidate).send<{
|
|
||||||
result?: { value?: FocusProbe };
|
|
||||||
}>(tabId, "Runtime.evaluate", {
|
|
||||||
expression: `(() => {
|
|
||||||
/* bsk-focus-target-probe */
|
|
||||||
const active = document.activeElement;
|
|
||||||
return {
|
|
||||||
hasFocus: document.hasFocus(),
|
|
||||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
|
|
||||||
};
|
|
||||||
})()`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
const probe = reply.result?.value;
|
|
||||||
if (probe?.hasFocus) focused.push({ target: candidate, probe });
|
|
||||||
} catch {
|
|
||||||
// A target may navigate or detach while the graph is being probed.
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
// A top document whose active element is an iframe delegates focus to
|
|
||||||
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
|
||||||
// target while same-process frames continue to use the shared root target.
|
|
||||||
const owners = focused.filter(
|
|
||||||
({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
|
|
||||||
);
|
|
||||||
if (owners.length !== 1) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"focus_target_unresolved",
|
|
||||||
owners.length === 0
|
|
||||||
? "press found no unique focused frame target"
|
|
||||||
: "press found multiple focused frame targets",
|
|
||||||
{ candidate_count: owners.length },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const target = owners[0].target;
|
|
||||||
return { target, cdp: cdpRunnerForTarget(cdp, target) };
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// tool.click
|
// tool.click
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
interface ResolvedClickPointerTarget {
|
|
||||||
point: { x: number; y: number };
|
|
||||||
usedRef?: string;
|
|
||||||
usedSelector?: string;
|
|
||||||
moveSettleMs: number;
|
|
||||||
capture?: {
|
|
||||||
id: string;
|
|
||||||
imageX: number;
|
|
||||||
imageY: number;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function handleClick(
|
export async function handleClick(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: ClickParams,
|
params: ClickParams,
|
||||||
deps: InteractionDeps = getDefaultDeps(),
|
deps: InteractionDeps = getDefaultDeps(),
|
||||||
): Promise<ClickResult | RpcError> {
|
): Promise<ClickResult | RpcError> {
|
||||||
const hasSurfacePointParams =
|
|
||||||
params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined;
|
|
||||||
const ctxOrErr = lookupSession(manager, params, "click");
|
const ctxOrErr = lookupSession(manager, params, "click");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const ctx = ctxOrErr;
|
const ctx = ctxOrErr;
|
||||||
@@ -353,67 +235,39 @@ export async function handleClick(
|
|||||||
if (denied) return denied;
|
if (denied) return denied;
|
||||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
const button: MouseButton = params.button ?? "left";
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||||
const clickCount = params.click_count ?? 1;
|
if (isRpcError(node)) return node;
|
||||||
if (!Number.isSafeInteger(clickCount) || clickCount < 1) {
|
|
||||||
return { code: "invalid_params", message: "click_count must be a positive integer" };
|
|
||||||
}
|
|
||||||
const modifiers = modifiersBitfield(params.modifiers);
|
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const pointerTarget: ResolvedClickPointerTarget | RpcError = hasSurfacePointParams
|
const geometry = await resolveNodeGeometry(
|
||||||
? await resolveSurfacePointerTarget(
|
deps.cdp,
|
||||||
ctx,
|
target.tabId,
|
||||||
target,
|
{
|
||||||
{
|
target: node.cdpTarget,
|
||||||
ref: params.ref,
|
backendNodeId: node.backendNodeId,
|
||||||
selector: params.selector,
|
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||||
captureId: params.capture_id,
|
},
|
||||||
imageX: params.image_x,
|
{ scrollIntoView: true },
|
||||||
imageY: params.image_y,
|
);
|
||||||
},
|
if (isRpcError(geometry)) return geometry;
|
||||||
{
|
const centre = geometry.actionPoint;
|
||||||
cdp: deps.cdp,
|
|
||||||
signal: deps.signal,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
: await (async () => {
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
|
||||||
if (isRpcError(node)) return node;
|
|
||||||
const geometry = await resolveNodeGeometry(
|
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
{
|
|
||||||
target: node.cdpTarget,
|
|
||||||
backendNodeId: node.backendNodeId,
|
|
||||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
|
||||||
},
|
|
||||||
{ scrollIntoView: true },
|
|
||||||
);
|
|
||||||
if (isRpcError(geometry)) return geometry;
|
|
||||||
return {
|
|
||||||
point: geometry.actionPoint,
|
|
||||||
usedRef: node.usedRef,
|
|
||||||
usedSelector: node.usedSelector,
|
|
||||||
moveSettleMs: 0,
|
|
||||||
};
|
|
||||||
})();
|
|
||||||
if (isRpcError(pointerTarget)) return pointerTarget;
|
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const overlayBlocking = await checkOverlayAtPoint(
|
const button: MouseButton = params.button ?? "left";
|
||||||
deps.cdp,
|
const clickCount = params.click_count ?? 1;
|
||||||
target.tabId,
|
if (clickCount < 1) {
|
||||||
pointerTarget.point.x,
|
return { code: "invalid_params", message: "click_count must be greater than zero" };
|
||||||
pointerTarget.point.y,
|
}
|
||||||
);
|
const modifiers = modifiersBitfield(params.modifiers);
|
||||||
|
|
||||||
|
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||||
let automationBypassEnabled = false;
|
let automationBypassEnabled = false;
|
||||||
if (overlayBlocking && deps.bypassOverlay) {
|
if (overlayBlocking && deps.bypassOverlay) {
|
||||||
try {
|
try {
|
||||||
@@ -425,16 +279,47 @@ export async function handleClick(
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, {
|
// Move first so hover state activates, then press → release.
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
modifiers,
|
||||||
|
});
|
||||||
|
if (throwIfAborted(deps.signal)) {
|
||||||
|
return { code: "cancelled", message: "click aborted" };
|
||||||
|
}
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mousePressed",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
button,
|
button,
|
||||||
clickCount,
|
clickCount,
|
||||||
modifiers,
|
modifiers,
|
||||||
signal: deps.signal,
|
|
||||||
moveSettleMs: pointerTarget.moveSettleMs,
|
|
||||||
});
|
});
|
||||||
if (dispatch === "cancelled") {
|
if (throwIfAborted(deps.signal)) {
|
||||||
|
try {
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseReleased",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
button,
|
||||||
|
clickCount,
|
||||||
|
modifiers,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err);
|
||||||
|
}
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
}
|
}
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseReleased",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
button,
|
||||||
|
clickCount,
|
||||||
|
modifiers,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -452,17 +337,10 @@ export async function handleClick(
|
|||||||
|
|
||||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
used_ref: pointerTarget.usedRef,
|
used_ref: node.usedRef,
|
||||||
used_selector: pointerTarget.usedSelector,
|
used_selector: node.usedSelector,
|
||||||
x: pointerTarget.point.x,
|
x: centre.x,
|
||||||
y: pointerTarget.point.y,
|
y: centre.y,
|
||||||
...(pointerTarget.capture
|
|
||||||
? {
|
|
||||||
capture_id: pointerTarget.capture.id,
|
|
||||||
image_x: pointerTarget.capture.imageX,
|
|
||||||
image_y: pointerTarget.capture.imageY,
|
|
||||||
}
|
|
||||||
: {}),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -720,9 +598,8 @@ export async function handleFill(
|
|||||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||||
const input = inputContextForNode(deps.cdp, node);
|
if (isRpcError(node)) return node;
|
||||||
if (isRpcError(input)) return input;
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
const nodeCdp = input.cdp;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
@@ -730,7 +607,7 @@ export async function handleFill(
|
|||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{
|
{
|
||||||
backendNodeId: input.backendNodeId,
|
backendNodeId: node.backendNodeId,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
if (!described.node || !isFillable(described.node)) {
|
if (!described.node || !isFillable(described.node)) {
|
||||||
@@ -743,15 +620,15 @@ export async function handleFill(
|
|||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
deps.cdp,
|
deps.cdp,
|
||||||
target.tabId,
|
target.tabId,
|
||||||
input.target,
|
node.cdpTarget,
|
||||||
input.backendNodeId,
|
node.backendNodeId,
|
||||||
input.frameId,
|
node.frameId,
|
||||||
);
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -763,7 +640,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
const clearBefore = params.clear_before ?? true;
|
const clearBefore = params.clear_before ?? true;
|
||||||
@@ -794,7 +671,7 @@ export async function handleFill(
|
|||||||
}
|
}
|
||||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||||
await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
|
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
@@ -807,28 +684,6 @@ export async function handleFill(
|
|||||||
}`,
|
}`,
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
});
|
});
|
||||||
// Give controlled inputs one bounded browser-render turn to accept or
|
|
||||||
// reject the dispatched value before reporting success.
|
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function() {
|
|
||||||
/* bsk-input-sync */
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
let settled = false;
|
|
||||||
const finish = () => {
|
|
||||||
if (settled) return;
|
|
||||||
settled = true;
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve();
|
|
||||||
};
|
|
||||||
const timer = setTimeout(finish, 100);
|
|
||||||
if (typeof requestAnimationFrame !== 'function') { finish(); return; }
|
|
||||||
requestAnimationFrame(() => requestAnimationFrame(finish));
|
|
||||||
});
|
|
||||||
}`,
|
|
||||||
awaitPromise: true,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -836,56 +691,11 @@ export async function handleFill(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
|
||||||
}
|
|
||||||
|
|
||||||
let actualValue: string;
|
|
||||||
try {
|
|
||||||
const readback = await nodeCdp.send<{
|
|
||||||
result?: { value?: { supported?: boolean; value?: string } };
|
|
||||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function() {
|
|
||||||
/* bsk-input-readback */
|
|
||||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
|
||||||
return { supported: true, value: this.value };
|
|
||||||
}
|
|
||||||
if (this.isContentEditable) {
|
|
||||||
return { supported: true, value: this.innerText ?? this.textContent ?? '' };
|
|
||||||
}
|
|
||||||
return { supported: false, value: '' };
|
|
||||||
}`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
const state = readback.result?.value;
|
|
||||||
actualValue = typeof state?.value === "string" ? state.value : "";
|
|
||||||
const applied =
|
|
||||||
state?.supported === true &&
|
|
||||||
(clearBefore
|
|
||||||
? actualValue === params.value
|
|
||||||
: params.value.length === 0 || actualValue.includes(params.value));
|
|
||||||
if (!applied) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"input_not_applied",
|
|
||||||
"the page did not retain the requested input value",
|
|
||||||
{ expected_value_length: params.value.length, actual_value_length: actualValue.length },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"input_not_applied",
|
|
||||||
err instanceof Error ? err.message : String(err),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
used_ref: input.usedRef,
|
used_ref: node.usedRef,
|
||||||
used_selector: input.usedSelector,
|
used_selector: node.usedSelector,
|
||||||
value_length: actualValue.length,
|
value_length: params.value.length,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1060,38 +870,31 @@ export async function handlePress(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let input: InputDispatchContext | RpcError;
|
|
||||||
// Optional focus before key dispatch.
|
// Optional focus before key dispatch.
|
||||||
if (params.ref || params.selector) {
|
if (params.ref || params.selector) {
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||||
const explicitInput = inputContextForNode(deps.cdp, node);
|
if (isRpcError(node)) return node;
|
||||||
if (isRpcError(explicitInput)) return explicitInput;
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
input = explicitInput;
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
deps.cdp,
|
deps.cdp,
|
||||||
target.tabId,
|
target.tabId,
|
||||||
explicitInput.target,
|
node.cdpTarget,
|
||||||
explicitInput.backendNodeId,
|
node.backendNodeId,
|
||||||
explicitInput.frameId,
|
node.frameId,
|
||||||
);
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "press aborted" };
|
return { code: "cancelled", message: "press aborted" };
|
||||||
}
|
}
|
||||||
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
backendNodeId: explicitInput.backendNodeId,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId);
|
|
||||||
if (isRpcError(input)) return input;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
@@ -1105,7 +908,7 @@ export async function handlePress(
|
|||||||
try {
|
try {
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||||
type: "rawKeyDown",
|
type: "rawKeyDown",
|
||||||
key: descriptor.key,
|
key: descriptor.key,
|
||||||
code: descriptor.code,
|
code: descriptor.code,
|
||||||
@@ -1119,7 +922,7 @@ export async function handlePress(
|
|||||||
typeof descriptor.text === "string" &&
|
typeof descriptor.text === "string" &&
|
||||||
descriptor.text.length > 0
|
descriptor.text.length > 0
|
||||||
) {
|
) {
|
||||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||||
type: "char",
|
type: "char",
|
||||||
key: descriptor.key,
|
key: descriptor.key,
|
||||||
code: descriptor.code,
|
code: descriptor.code,
|
||||||
@@ -1136,7 +939,7 @@ export async function handlePress(
|
|||||||
cancelled = true;
|
cancelled = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||||
type: "keyUp",
|
type: "keyUp",
|
||||||
key: descriptor.key,
|
key: descriptor.key,
|
||||||
code: descriptor.code,
|
code: descriptor.code,
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
import type { MouseButton } from "@/transport/types";
|
|
||||||
import type { Point } from "./geometry";
|
|
||||||
import type { CdpRunner } from "./shared";
|
|
||||||
|
|
||||||
interface MouseClickDispatchOptions {
|
|
||||||
button: MouseButton;
|
|
||||||
clickCount: number;
|
|
||||||
modifiers: number;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
moveSettleMs?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
type MouseClickDispatchResult = "completed" | "cancelled";
|
|
||||||
|
|
||||||
function waitForSettle(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
|
|
||||||
if (ms <= 0) return Promise.resolve(signal?.aborted !== true);
|
|
||||||
if (signal?.aborted) return Promise.resolve(false);
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
let settled = false;
|
|
||||||
const finish = (completed: boolean) => {
|
|
||||||
if (settled) return;
|
|
||||||
settled = true;
|
|
||||||
clearTimeout(timer);
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
resolve(completed);
|
|
||||||
};
|
|
||||||
const timer = setTimeout(() => finish(true), ms);
|
|
||||||
const onAbort = () => finish(false);
|
|
||||||
signal?.addEventListener("abort", onAbort, { once: true });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dispatch one coherent CDP mouse click in top-viewport CSS coordinates.
|
|
||||||
*
|
|
||||||
* Keeping the sequence here prevents DOM and Surface interaction paths from
|
|
||||||
* duplicating cancellation and best-effort release behavior. Callers that
|
|
||||||
* target asynchronously hit-tested content may request a settle interval.
|
|
||||||
*/
|
|
||||||
export async function dispatchMouseClick(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
point: Point,
|
|
||||||
options: MouseClickDispatchOptions,
|
|
||||||
): Promise<MouseClickDispatchResult> {
|
|
||||||
const { button, clickCount, modifiers, signal, moveSettleMs = 0 } = options;
|
|
||||||
let pressed = false;
|
|
||||||
const release = () =>
|
|
||||||
cdp.send(tabId, "Input.dispatchMouseEvent", {
|
|
||||||
type: "mouseReleased",
|
|
||||||
x: point.x,
|
|
||||||
y: point.y,
|
|
||||||
button,
|
|
||||||
clickCount,
|
|
||||||
modifiers,
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
|
||||||
type: "mouseMoved",
|
|
||||||
x: point.x,
|
|
||||||
y: point.y,
|
|
||||||
modifiers,
|
|
||||||
});
|
|
||||||
if (!(await waitForSettle(moveSettleMs, signal))) return "cancelled";
|
|
||||||
|
|
||||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
|
||||||
type: "mousePressed",
|
|
||||||
x: point.x,
|
|
||||||
y: point.y,
|
|
||||||
button,
|
|
||||||
clickCount,
|
|
||||||
modifiers,
|
|
||||||
});
|
|
||||||
pressed = true;
|
|
||||||
|
|
||||||
if (signal?.aborted) {
|
|
||||||
await release();
|
|
||||||
pressed = false;
|
|
||||||
return "cancelled";
|
|
||||||
}
|
|
||||||
|
|
||||||
await release();
|
|
||||||
pressed = false;
|
|
||||||
return "completed";
|
|
||||||
} catch (error) {
|
|
||||||
if (pressed) {
|
|
||||||
try {
|
|
||||||
await release();
|
|
||||||
} catch (releaseError) {
|
|
||||||
console.debug("[bsk mouse-input] best-effort mouse release failed", releaseError);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -243,7 +243,6 @@ export async function handleSessionStop(
|
|||||||
|
|
||||||
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
||||||
ctx.refStore.clear();
|
ctx.refStore.clear();
|
||||||
ctx.surfaceCaptures.clear();
|
|
||||||
|
|
||||||
clearRecordingForSession(params.session_id);
|
clearRecordingForSession(params.session_id);
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,8 @@
|
|||||||
// compound CDP node identity via the session RefStore, and build stable
|
// compound CDP node identity via the session RefStore, and build stable
|
||||||
// `ref_not_found` errors for hard-failure tool paths.
|
// `ref_not_found` errors for hard-failure tool paths.
|
||||||
|
|
||||||
import type { Rect } from "@browser-skill/vom";
|
|
||||||
import type { SessionContext } from "@/session-manager/manager";
|
import type { SessionContext } from "@/session-manager/manager";
|
||||||
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
|
import { normaliseRef } from "@/session-manager/ref-store";
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
|
|
||||||
@@ -13,10 +12,6 @@ export interface SnapshotRefLookup {
|
|||||||
refKey: string;
|
refKey: string;
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
cdpSessionId?: string;
|
cdpSessionId?: string;
|
||||||
visibleRect?: Rect;
|
|
||||||
kind: RefTargetKind;
|
|
||||||
capabilities: RefCapability[];
|
|
||||||
generation: number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||||
@@ -42,10 +37,6 @@ export function lookupSnapshotRef(
|
|||||||
refKey,
|
refKey,
|
||||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||||
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
|
|
||||||
kind: entry.kind,
|
|
||||||
capabilities: entry.capabilities,
|
|
||||||
generation: entry.generation,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,7 +49,6 @@ export function resolveSnapshotRef(
|
|||||||
ctx: SessionContext,
|
ctx: SessionContext,
|
||||||
ref: string,
|
ref: string,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
requiredCapability?: RefCapability,
|
|
||||||
): SnapshotRefLookup | RpcError {
|
): SnapshotRefLookup | RpcError {
|
||||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||||
if (looked === null) {
|
if (looked === null) {
|
||||||
@@ -68,13 +58,5 @@ export function resolveSnapshotRef(
|
|||||||
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
|
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
|
|
||||||
return rpcError(
|
|
||||||
"permission_denied",
|
|
||||||
"ref_capability_denied",
|
|
||||||
`ref ${ref} does not support ${requiredCapability}`,
|
|
||||||
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return looked;
|
return looked;
|
||||||
}
|
}
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
import type { Rect } from "@browser-skill/vom";
|
|
||||||
import type { Point, Region, ViewportRect } from "./geometry";
|
|
||||||
|
|
||||||
const RECT_EPSILON = 0.01;
|
|
||||||
|
|
||||||
export function surfaceVisibleRect(
|
|
||||||
liveRect: ViewportRect,
|
|
||||||
observedRect: Rect | undefined,
|
|
||||||
): Rect | null {
|
|
||||||
const observed = observedRect ?? {
|
|
||||||
x: liveRect.x,
|
|
||||||
y: liveRect.y,
|
|
||||||
w: liveRect.width,
|
|
||||||
h: liveRect.height,
|
|
||||||
};
|
|
||||||
const x = Math.max(liveRect.x, observed.x);
|
|
||||||
const y = Math.max(liveRect.y, observed.y);
|
|
||||||
const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w);
|
|
||||||
const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h);
|
|
||||||
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function sameRect(a: Rect, b: Rect): boolean {
|
|
||||||
return (
|
|
||||||
Math.abs(a.x - b.x) <= RECT_EPSILON &&
|
|
||||||
Math.abs(a.y - b.y) <= RECT_EPSILON &&
|
|
||||||
Math.abs(a.w - b.w) <= RECT_EPSILON &&
|
|
||||||
Math.abs(a.h - b.h) <= RECT_EPSILON
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function mapImagePointToViewport(
|
|
||||||
rect: Rect,
|
|
||||||
imageWidth: number,
|
|
||||||
imageHeight: number,
|
|
||||||
imageX: number,
|
|
||||||
imageY: number,
|
|
||||||
): Point | null {
|
|
||||||
if (
|
|
||||||
!Number.isFinite(imageX) ||
|
|
||||||
!Number.isFinite(imageY) ||
|
|
||||||
!Number.isSafeInteger(imageWidth) ||
|
|
||||||
!Number.isSafeInteger(imageHeight) ||
|
|
||||||
imageWidth <= 0 ||
|
|
||||||
imageHeight <= 0 ||
|
|
||||||
imageX < 0 ||
|
|
||||||
imageY < 0 ||
|
|
||||||
imageX >= imageWidth ||
|
|
||||||
imageY >= imageHeight ||
|
|
||||||
!Number.isFinite(rect.x) ||
|
|
||||||
!Number.isFinite(rect.y) ||
|
|
||||||
!Number.isFinite(rect.w) ||
|
|
||||||
!Number.isFinite(rect.h) ||
|
|
||||||
rect.w <= 0 ||
|
|
||||||
rect.h <= 0
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
x: rect.x + (imageX / imageWidth) * rect.w,
|
|
||||||
y: rect.y + (imageY / imageHeight) * rect.h,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function pointInPolygon(point: Point, polygon: Point[]): boolean {
|
|
||||||
for (let index = 0; index < polygon.length; index += 1) {
|
|
||||||
const a = polygon[index];
|
|
||||||
const b = polygon[(index + 1) % polygon.length];
|
|
||||||
const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x);
|
|
||||||
const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9;
|
|
||||||
const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9;
|
|
||||||
if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true;
|
|
||||||
}
|
|
||||||
let inside = false;
|
|
||||||
for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) {
|
|
||||||
const a = polygon[index];
|
|
||||||
const b = polygon[previous];
|
|
||||||
const intersects =
|
|
||||||
a.y > point.y !== b.y > point.y &&
|
|
||||||
point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x;
|
|
||||||
if (intersects) inside = !inside;
|
|
||||||
}
|
|
||||||
return inside;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function pointInRegion(point: Point, region: Region): boolean {
|
|
||||||
return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon));
|
|
||||||
}
|
|
||||||
@@ -1,260 +0,0 @@
|
|||||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
|
||||||
import type { SessionContext } from "@/session-manager/manager";
|
|
||||||
import type { RpcError } from "@/transport/types";
|
|
||||||
import { rpcError } from "./errors";
|
|
||||||
import { resolveNodeGeometry } from "./frame-geometry";
|
|
||||||
import { type CdpRunner, isRpcError, normaliseRef } from "./shared";
|
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
|
||||||
import {
|
|
||||||
mapImagePointToViewport,
|
|
||||||
pointInRegion,
|
|
||||||
sameRect,
|
|
||||||
surfaceVisibleRect,
|
|
||||||
} from "./surface-coordinate";
|
|
||||||
|
|
||||||
export interface SurfaceCaptureEnvironment {
|
|
||||||
navigationIdentity: string;
|
|
||||||
viewportSignature: string;
|
|
||||||
frameProjectionSignature: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SurfaceTargetResolverDeps {
|
|
||||||
cdp: CdpRunner;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SurfacePointerTargetInput {
|
|
||||||
ref?: string;
|
|
||||||
selector?: string;
|
|
||||||
captureId?: string;
|
|
||||||
imageX?: number;
|
|
||||||
imageY?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ResolvedSurfacePointerTarget {
|
|
||||||
point: { x: number; y: number };
|
|
||||||
usedRef: string;
|
|
||||||
moveSettleMs: number;
|
|
||||||
capture: {
|
|
||||||
id: string;
|
|
||||||
imageX: number;
|
|
||||||
imageY: number;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Canvas-style controls commonly update their pointer hit-test on the next
|
|
||||||
// animation frame. Leave a scheduling boundary after moving and before pressing.
|
|
||||||
export const SURFACE_POINTER_MOVE_SETTLE_MS = 32;
|
|
||||||
|
|
||||||
function stale(message: string): RpcError {
|
|
||||||
return rpcError("permission_denied", "surface_capture_stale", message);
|
|
||||||
}
|
|
||||||
|
|
||||||
function finiteObject(value: unknown): unknown {
|
|
||||||
if (Array.isArray(value)) return value.map(finiteObject);
|
|
||||||
if (typeof value !== "object" || value === null) return value;
|
|
||||||
return Object.fromEntries(
|
|
||||||
Object.entries(value)
|
|
||||||
.filter(
|
|
||||||
([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)),
|
|
||||||
)
|
|
||||||
.sort(([a], [b]) => a.localeCompare(b))
|
|
||||||
.map(([key, entry]) => [key, finiteObject(entry)]),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null {
|
|
||||||
if (!frameId) return "top";
|
|
||||||
const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
|
||||||
const path: Array<{
|
|
||||||
frameId: string;
|
|
||||||
parentFrameId?: string;
|
|
||||||
ownerBackendNodeId?: number;
|
|
||||||
targetSessionId?: string;
|
|
||||||
}> = [];
|
|
||||||
const seen = new Set<string>();
|
|
||||||
let current = frames.get(frameId);
|
|
||||||
while (current) {
|
|
||||||
if (seen.has(current.frameId)) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
path.push({
|
|
||||||
frameId: current.frameId,
|
|
||||||
...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}),
|
|
||||||
...(current.ownerBackendNodeId !== undefined
|
|
||||||
? { ownerBackendNodeId: current.ownerBackendNodeId }
|
|
||||||
: {}),
|
|
||||||
...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}),
|
|
||||||
});
|
|
||||||
if (!current.parentFrameId) return JSON.stringify(path);
|
|
||||||
const parent = frames.get(current.parentFrameId);
|
|
||||||
if (!parent) return null;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function captureSurfaceEnvironment(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
frameId: string | undefined,
|
|
||||||
): Promise<SurfaceCaptureEnvironment | RpcError> {
|
|
||||||
try {
|
|
||||||
const [tree, metrics, graph] = await Promise.all([
|
|
||||||
cdp.send<{
|
|
||||||
frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } };
|
|
||||||
}>(tabId, "Page.getFrameTree", {}),
|
|
||||||
cdp.send<Record<string, unknown>>(tabId, "Page.getLayoutMetrics", {}),
|
|
||||||
frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null),
|
|
||||||
]);
|
|
||||||
const root = tree.frameTree?.frame;
|
|
||||||
if (!root?.id || !root.loaderId) {
|
|
||||||
return { code: "cdp_failed", message: "could not identify the current navigation" };
|
|
||||||
}
|
|
||||||
const frameProjectionSignature = frameId
|
|
||||||
? graph
|
|
||||||
? framePathSignature(graph, frameId)
|
|
||||||
: null
|
|
||||||
: "top";
|
|
||||||
if (!frameProjectionSignature) {
|
|
||||||
return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` };
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }),
|
|
||||||
viewportSignature: JSON.stringify(
|
|
||||||
finiteObject({
|
|
||||||
cssLayoutViewport: metrics.cssLayoutViewport,
|
|
||||||
cssVisualViewport: metrics.cssVisualViewport,
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
frameProjectionSignature,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function validatePointParams(input: SurfacePointerTargetInput): RpcError | null {
|
|
||||||
if (!input.captureId || typeof input.captureId !== "string") {
|
|
||||||
return { code: "invalid_params", message: "surface point click requires capture_id" };
|
|
||||||
}
|
|
||||||
if (typeof input.imageX !== "number" || !Number.isFinite(input.imageX)) {
|
|
||||||
return { code: "invalid_params", message: "image_x must be a finite number" };
|
|
||||||
}
|
|
||||||
if (typeof input.imageY !== "number" || !Number.isFinite(input.imageY)) {
|
|
||||||
return { code: "invalid_params", message: "image_y must be a finite number" };
|
|
||||||
}
|
|
||||||
if (!input.ref || input.selector) {
|
|
||||||
return {
|
|
||||||
code: "invalid_params",
|
|
||||||
message: "surface point click requires one Surface ref and does not accept a selector",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError {
|
|
||||||
const code = reason === "not_found" ? "not_found" : "permission_denied";
|
|
||||||
return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function aborted(signal: AbortSignal | undefined): RpcError | null {
|
|
||||||
return signal?.aborted
|
|
||||||
? { code: "cancelled", message: "surface target resolution aborted" }
|
|
||||||
: null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Resolve one fresh Surface capture coordinate into a validated top-viewport pointer target. */
|
|
||||||
export async function resolveSurfacePointerTarget(
|
|
||||||
ctx: SessionContext,
|
|
||||||
target: { tabId: number; url?: string },
|
|
||||||
input: SurfacePointerTargetInput,
|
|
||||||
deps: SurfaceTargetResolverDeps,
|
|
||||||
): Promise<ResolvedSurfacePointerTarget | RpcError> {
|
|
||||||
const invalid = validatePointParams(input);
|
|
||||||
if (invalid) return invalid;
|
|
||||||
const earlyAbort = aborted(deps.signal);
|
|
||||||
if (earlyAbort) return earlyAbort;
|
|
||||||
|
|
||||||
const consumed = ctx.surfaceCaptures.consume(input.captureId as string);
|
|
||||||
if (!consumed.ok) return captureConsumeError(consumed.reason);
|
|
||||||
const capture = consumed.capture;
|
|
||||||
const postConsumeAbort = aborted(deps.signal);
|
|
||||||
if (postConsumeAbort) return postConsumeAbort;
|
|
||||||
|
|
||||||
if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) {
|
|
||||||
return stale("surface capture belongs to a different session or tab");
|
|
||||||
}
|
|
||||||
if (normaliseRef(input.ref as string) !== capture.surface.ref) {
|
|
||||||
return stale("surface ref does not match the capture");
|
|
||||||
}
|
|
||||||
const node = resolveSnapshotRef(ctx, input.ref as string, target.tabId, "screenshot");
|
|
||||||
if (isRpcError(node)) return stale(node.message);
|
|
||||||
if (
|
|
||||||
node.kind !== "surface" ||
|
|
||||||
node.backendNodeId !== capture.surface.backendNodeId ||
|
|
||||||
node.frameId !== capture.surface.frameId ||
|
|
||||||
node.generation !== capture.surface.observationGeneration
|
|
||||||
) {
|
|
||||||
return stale("surface observation generation or node identity changed");
|
|
||||||
}
|
|
||||||
|
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
|
||||||
try {
|
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
|
||||||
} catch (error) {
|
|
||||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
|
||||||
}
|
|
||||||
const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId);
|
|
||||||
if (isRpcError(environment)) return environment;
|
|
||||||
if (
|
|
||||||
environment.navigationIdentity !== capture.navigationIdentity ||
|
|
||||||
environment.viewportSignature !== capture.viewportSignature ||
|
|
||||||
environment.frameProjectionSignature !== capture.frameProjectionSignature
|
|
||||||
) {
|
|
||||||
return stale("navigation, viewport, zoom, scroll, or frame projection changed");
|
|
||||||
}
|
|
||||||
|
|
||||||
const geometry = await resolveNodeGeometry(
|
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
{
|
|
||||||
target: {
|
|
||||||
tabId: target.tabId,
|
|
||||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
|
||||||
},
|
|
||||||
backendNodeId: node.backendNodeId,
|
|
||||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
|
||||||
},
|
|
||||||
{ scrollIntoView: false },
|
|
||||||
);
|
|
||||||
if (isRpcError(geometry)) return stale(geometry.message);
|
|
||||||
const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect);
|
|
||||||
if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) {
|
|
||||||
return stale("surface visible region changed");
|
|
||||||
}
|
|
||||||
const point = mapImagePointToViewport(
|
|
||||||
capture.topViewportRect,
|
|
||||||
capture.imageWidth,
|
|
||||||
capture.imageHeight,
|
|
||||||
input.imageX as number,
|
|
||||||
input.imageY as number,
|
|
||||||
);
|
|
||||||
if (!point || !pointInRegion(point, geometry.topVisibleRegions)) {
|
|
||||||
return rpcError(
|
|
||||||
"invalid_params",
|
|
||||||
"surface_coordinate_invalid",
|
|
||||||
"image coordinate is outside the captured Surface",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
point,
|
|
||||||
usedRef: capture.surface.ref,
|
|
||||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
|
||||||
capture: {
|
|
||||||
id: capture.id,
|
|
||||||
imageX: input.imageX as number,
|
|
||||||
imageY: input.imageY as number,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -36,7 +36,7 @@ function fakeSnapshotReply() {
|
|||||||
},
|
},
|
||||||
layout: {
|
layout: {
|
||||||
nodeIndex: [1, 2, 3],
|
nodeIndex: [1, 2, 3],
|
||||||
// Legacy fixture: omitted style columns fall back to visible defaults.
|
// styles columns follow requested computedStyles order: [position, pointer-events]
|
||||||
styles: [
|
styles: [
|
||||||
[i("static"), i("auto")],
|
[i("static"), i("auto")],
|
||||||
[i("fixed"), i("auto")],
|
[i("fixed"), i("auto")],
|
||||||
@@ -682,84 +682,6 @@ describe("captureViewModel", () => {
|
|||||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("captures painted visibility as DOM fallback evidence", async () => {
|
|
||||||
const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
|
|
||||||
const i = (value: string) => S.indexOf(value);
|
|
||||||
const snapshot = {
|
|
||||||
strings: S,
|
|
||||||
documents: [
|
|
||||||
{
|
|
||||||
nodes: {
|
|
||||||
parentIndex: [-1, 0, 1, 1, 1],
|
|
||||||
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
|
||||||
backendNodeId: [10, 11, 12, 13, 14],
|
|
||||||
attributes: [[], [], [], [], []],
|
|
||||||
},
|
|
||||||
layout: {
|
|
||||||
nodeIndex: [1, 2, 3, 4],
|
|
||||||
styles: [
|
|
||||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
|
||||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
|
||||||
[i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
|
|
||||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
|
||||||
],
|
|
||||||
bounds: [
|
|
||||||
[0, 0, 1000, 800],
|
|
||||||
[10, 10, 100, 30],
|
|
||||||
[10, 50, 100, 30],
|
|
||||||
[10, 90, 100, 30],
|
|
||||||
],
|
|
||||||
paintOrders: [0, 1, 2, 3],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
|
||||||
|
|
||||||
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
|
||||||
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
|
||||||
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
|
|
||||||
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
|
|
||||||
const i = (value: string) => S.indexOf(value);
|
|
||||||
const snapshot = {
|
|
||||||
strings: S,
|
|
||||||
documents: [
|
|
||||||
{
|
|
||||||
nodes: {
|
|
||||||
parentIndex: [-1, 0, 1, 2],
|
|
||||||
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
|
|
||||||
backendNodeId: [10, 11, 12, 13],
|
|
||||||
attributes: [[], [], [], []],
|
|
||||||
},
|
|
||||||
layout: {
|
|
||||||
nodeIndex: [1, 2, 3],
|
|
||||||
styles: [
|
|
||||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
|
||||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
|
||||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
|
||||||
],
|
|
||||||
bounds: [
|
|
||||||
[0, 0, 1000, 800],
|
|
||||||
[10, 10, 200, 100],
|
|
||||||
[10, 10, 200, 100],
|
|
||||||
],
|
|
||||||
paintOrders: [0, 1, 2],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
|
||||||
const canvas = nodes.find((node) => node.backendNodeId === 13);
|
|
||||||
|
|
||||||
expect(canvas?.rendered).toBe(true);
|
|
||||||
expect(canvas?.visuallySuppressed).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_t: number, method: string) => {
|
send: vi.fn(async (_t: number, method: string) => {
|
||||||
|
|||||||
@@ -132,67 +132,4 @@ describe("captureFrameData", () => {
|
|||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
|
||||||
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
|
||||||
const captured: CapturedViewModel = {
|
|
||||||
nodes: [owner],
|
|
||||||
viewport: { width: 1000, height: 800 },
|
|
||||||
iframeNodes: new Map(),
|
|
||||||
frameNodes: new Map([["main", [owner]]]),
|
|
||||||
rootFrameId: "main",
|
|
||||||
excludedBackendNodeIds: new Set(),
|
|
||||||
};
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId, method) => {
|
|
||||||
if (method === "Accessibility.enable") return {};
|
|
||||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
|
||||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected root ${method}`);
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
|
||||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
|
||||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
|
||||||
if (method === "Accessibility.getFullAXTree") {
|
|
||||||
return {
|
|
||||||
nodes: [
|
|
||||||
{
|
|
||||||
nodeId: "button",
|
|
||||||
backendDOMNodeId: 101,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Continue" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected ${method}`);
|
|
||||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
|
||||||
getFrameGraph: vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 10,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
})),
|
|
||||||
};
|
|
||||||
|
|
||||||
const documents = await captureFrameData(cdp, 4, captured);
|
|
||||||
const childDocument = documents.find((document) => document.frameId === "child");
|
|
||||||
|
|
||||||
expect(childDocument?.axNodes).toEqual([
|
|
||||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
|
||||||
]);
|
|
||||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
|
||||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
@@ -1,183 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
|
||||||
import type { CdpRunner } from "../../shared";
|
|
||||||
import type { CapturedNode } from "../capture";
|
|
||||||
import type { FrameDocument, FrameOwnedAxNode } from "../frame-document";
|
|
||||||
import { probeTooltipNames } from "../name-enrichment";
|
|
||||||
import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph";
|
|
||||||
import { stableIdentifierName } from "../semantic-graph/name-evidence";
|
|
||||||
import { frameBackendKey } from "../semantic-graph/types";
|
|
||||||
|
|
||||||
function document(
|
|
||||||
target: CdpTarget,
|
|
||||||
backendNodeId: number,
|
|
||||||
accessibleName?: string,
|
|
||||||
frameId = "child",
|
|
||||||
x = 10,
|
|
||||||
): FrameDocument<FrameOwnedAxNode> {
|
|
||||||
const node: CapturedNode = {
|
|
||||||
backendNodeId,
|
|
||||||
parentBackendNodeId: null,
|
|
||||||
frameId,
|
|
||||||
tag: "button",
|
|
||||||
attrs: {
|
|
||||||
id: "toolInsertRecord",
|
|
||||||
"data-test-id": "toolInsertRecord",
|
|
||||||
...(accessibleName ? { "aria-label": accessibleName } : {}),
|
|
||||||
},
|
|
||||||
rect: { x, y: 20, w: 40, h: 30 },
|
|
||||||
localRect: { x: 2, y: 3, w: 40, h: 30 },
|
|
||||||
paintOrder: 1,
|
|
||||||
position: "static",
|
|
||||||
pointerEvents: "auto",
|
|
||||||
cursor: "pointer",
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
frameId,
|
|
||||||
parentFrameId: "main",
|
|
||||||
contextScopeId: "child",
|
|
||||||
target,
|
|
||||||
axNodes: [],
|
|
||||||
domNodes: [node],
|
|
||||||
url: "https://example.com/frame",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function semantics(documents: FrameDocument<FrameOwnedAxNode>[]) {
|
|
||||||
return resolveSemanticGraph(
|
|
||||||
buildSemanticGraph({
|
|
||||||
documents,
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: documents[0]?.frameId,
|
|
||||||
}),
|
|
||||||
{ identifierFallback: false },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("tooltip name enrichment", () => {
|
|
||||||
it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => {
|
|
||||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
|
||||||
let hovered = false;
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId, method, params) => {
|
|
||||||
if (method === "Input.dispatchMouseEvent") {
|
|
||||||
hovered = (params as { x?: number }).x === 30;
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 11 };
|
|
||||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } };
|
|
||||||
if (method === "Runtime.callFunctionOn") {
|
|
||||||
return { result: { value: hovered ? ["插入行"] : [] } };
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
};
|
|
||||||
|
|
||||||
const documents = [document(target, 501)];
|
|
||||||
const graph = semantics(documents);
|
|
||||||
const names = await probeTooltipNames(cdp, 7, documents, graph);
|
|
||||||
|
|
||||||
expect(names.get(frameBackendKey("child", 501))).toBe("插入行");
|
|
||||||
expect(cdp.sendToTarget).toHaveBeenCalledWith(
|
|
||||||
target,
|
|
||||||
"Page.createIsolatedWorld",
|
|
||||||
expect.objectContaining({ frameId: "child" }),
|
|
||||||
);
|
|
||||||
|
|
||||||
const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length;
|
|
||||||
const targetSendsBeforeCache = vi.mocked(
|
|
||||||
cdp.sendToTarget as NonNullable<CdpRunner["sendToTarget"]>,
|
|
||||||
).mock.calls.length;
|
|
||||||
const cached = await probeTooltipNames(cdp, 7, documents, graph);
|
|
||||||
expect(cached.get(frameBackendKey("child", 501))).toBe("插入行");
|
|
||||||
expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache);
|
|
||||||
expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not probe controls that already have a higher-confidence name", async () => {
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
|
||||||
};
|
|
||||||
|
|
||||||
const documents = [document({ tabId: 8 }, 601, "Add a new record")];
|
|
||||||
const names = await probeTooltipNames(cdp, 8, documents, semantics(documents));
|
|
||||||
|
|
||||||
expect(names.size).toBe(0);
|
|
||||||
expect(cdp.send).not.toHaveBeenCalled();
|
|
||||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not collect supplemental names from form controls", async () => {
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
|
||||||
};
|
|
||||||
const inputDocument = document({ tabId: 8 }, 602);
|
|
||||||
inputDocument.domNodes[0] = {
|
|
||||||
...inputDocument.domNodes[0],
|
|
||||||
tag: "input",
|
|
||||||
attrs: { type: "email" },
|
|
||||||
formValue: "user@example.com",
|
|
||||||
formDefaultValue: "",
|
|
||||||
formState: "filled",
|
|
||||||
};
|
|
||||||
|
|
||||||
const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
|
|
||||||
|
|
||||||
expect(names.size).toBe(0);
|
|
||||||
expect(cdp.send).not.toHaveBeenCalled();
|
|
||||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
|
||||||
let hoveredX = -10;
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId, method, params) => {
|
|
||||||
if (method === "Input.dispatchMouseEvent") {
|
|
||||||
hoveredX = (params as { x: number }).x;
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (target, method, params) => {
|
|
||||||
if (method === "Page.createIsolatedWorld") {
|
|
||||||
return { executionContextId: target.sessionId === "left-session" ? 21 : 22 };
|
|
||||||
}
|
|
||||||
if (method === "DOM.resolveNode") {
|
|
||||||
return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } };
|
|
||||||
}
|
|
||||||
if (method === "Runtime.callFunctionOn") {
|
|
||||||
const objectId = (params as { objectId: string }).objectId;
|
|
||||||
const value =
|
|
||||||
objectId === "left" && hoveredX === 30
|
|
||||||
? ["Left action"]
|
|
||||||
: objectId === "right" && hoveredX === 130
|
|
||||||
? ["Right action"]
|
|
||||||
: [];
|
|
||||||
return { result: { value } };
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
};
|
|
||||||
const documents = [
|
|
||||||
document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10),
|
|
||||||
document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110),
|
|
||||||
];
|
|
||||||
|
|
||||||
const names = await probeTooltipNames(cdp, 9, documents, semantics(documents));
|
|
||||||
|
|
||||||
expect(names.get(frameBackendKey("left", 701))).toBe("Left action");
|
|
||||||
expect(names.get(frameBackendKey("right", 701))).toBe("Right action");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("stable identifier evidence", () => {
|
|
||||||
it("accepts readable action identifiers and rejects framework-generated ids", () => {
|
|
||||||
expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record");
|
|
||||||
expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account");
|
|
||||||
expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined();
|
|
||||||
expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,256 +0,0 @@
|
|||||||
import type { VomScene } from "@browser-skill/vom";
|
|
||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import type { CapturedNode } from "../capture";
|
|
||||||
import type { CapturedFrameDocument } from "../frame-capture";
|
|
||||||
import {
|
|
||||||
clusterRenderedSurfaces,
|
|
||||||
discoverRenderedSurfaces,
|
|
||||||
projectRenderedSurfaces,
|
|
||||||
} from "../rendered-surfaces";
|
|
||||||
import type { SemanticAxNode } from "../semantic-graph";
|
|
||||||
|
|
||||||
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
|
||||||
return {
|
|
||||||
backendNodeId,
|
|
||||||
parentBackendNodeId: null,
|
|
||||||
frameId: "main",
|
|
||||||
ownerFrameBackendNodeId: null,
|
|
||||||
tag: "canvas",
|
|
||||||
attrs: {},
|
|
||||||
rect: { x: 10, y: 20, w: 800, h: 500 },
|
|
||||||
localRect: { x: 10, y: 20, w: 800, h: 500 },
|
|
||||||
paintOrder: 1,
|
|
||||||
position: "static",
|
|
||||||
pointerEvents: "auto",
|
|
||||||
rendered: true,
|
|
||||||
...overrides,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function document(
|
|
||||||
domNodes: CapturedNode[],
|
|
||||||
axNodes: SemanticAxNode[] = [],
|
|
||||||
): CapturedFrameDocument<SemanticAxNode> {
|
|
||||||
return {
|
|
||||||
frameId: domNodes[0]?.frameId ?? "main",
|
|
||||||
contextScopeId: domNodes[0]?.frameId ?? "main",
|
|
||||||
target: { tabId: 4 },
|
|
||||||
domNodes,
|
|
||||||
axNodes,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("rendered surface discovery", () => {
|
|
||||||
it("finds prominent rendered canvases and excludes hidden descendants", () => {
|
|
||||||
const visible = domNode(2, { parentBackendNodeId: 1 });
|
|
||||||
const hidden = domNode(3, { parentBackendNodeId: 4 });
|
|
||||||
const surfaces = discoverRenderedSurfaces(
|
|
||||||
[
|
|
||||||
document([
|
|
||||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
|
||||||
visible,
|
|
||||||
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
|
|
||||||
hidden,
|
|
||||||
]),
|
|
||||||
],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(surfaces).toHaveLength(1);
|
|
||||||
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps tiny canvases as individually addressable exact discoveries", () => {
|
|
||||||
const surfaces = discoverRenderedSurfaces(
|
|
||||||
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
);
|
|
||||||
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps a surface when the canvas only has partial native accessibility content", () => {
|
|
||||||
const axNodes = [
|
|
||||||
{
|
|
||||||
nodeId: "canvas",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "presentation" },
|
|
||||||
childIds: ["button"],
|
|
||||||
frameId: "main",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "button",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
frameId: "main",
|
|
||||||
},
|
|
||||||
] as SemanticAxNode[];
|
|
||||||
expect(
|
|
||||||
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
|
|
||||||
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
|
|
||||||
const canvas = domNode(2, {
|
|
||||||
attrs: {
|
|
||||||
id: "must-not-be-a-label",
|
|
||||||
class: "must-not-be-a-label",
|
|
||||||
"data-testid": "must-not-be-a-label",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
|
|
||||||
expect.not.objectContaining({
|
|
||||||
label: expect.anything(),
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("treats blank explicit Canvas names as unnamed", () => {
|
|
||||||
expect(
|
|
||||||
discoverRenderedSurfaces(
|
|
||||||
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
),
|
|
||||||
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
|
|
||||||
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
|
|
||||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
|
||||||
|
|
||||||
expect(
|
|
||||||
discoverRenderedSurfaces(
|
|
||||||
[document([owner, canvas])],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
new Set([1]),
|
|
||||||
),
|
|
||||||
).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("excludes canvases suppressed by a fully transparent ancestor", () => {
|
|
||||||
const owner = domNode(1, {
|
|
||||||
tag: "div",
|
|
||||||
rect: { x: 0, y: 0, w: 900, h: 600 },
|
|
||||||
visuallySuppressed: true,
|
|
||||||
});
|
|
||||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
|
||||||
|
|
||||||
expect(
|
|
||||||
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
|
|
||||||
).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
|
|
||||||
const owner = domNode(1, {
|
|
||||||
tag: "div",
|
|
||||||
rect: { x: 10, y: 20, w: 100, h: 70 },
|
|
||||||
overflowX: "hidden",
|
|
||||||
overflowY: "hidden",
|
|
||||||
});
|
|
||||||
const partial = domNode(2, {
|
|
||||||
parentBackendNodeId: 1,
|
|
||||||
rect: { x: 55, y: 30, w: 140, h: 60 },
|
|
||||||
});
|
|
||||||
const fullyClipped = domNode(3, {
|
|
||||||
parentBackendNodeId: 1,
|
|
||||||
rect: { x: 140, y: 30, w: 80, h: 50 },
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(
|
|
||||||
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
|
|
||||||
width: 1280,
|
|
||||||
height: 720,
|
|
||||||
}),
|
|
||||||
).toEqual([
|
|
||||||
expect.objectContaining({
|
|
||||||
backendNodeId: 2,
|
|
||||||
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("rendered surface clustering and projection", () => {
|
|
||||||
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
|
|
||||||
const body = domNode(1, { tag: "body", rect: null, localRect: null });
|
|
||||||
const surfaces = discoverRenderedSurfaces(
|
|
||||||
[
|
|
||||||
document([
|
|
||||||
body,
|
|
||||||
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
|
|
||||||
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
|
|
||||||
]),
|
|
||||||
],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
);
|
|
||||||
const groups = clusterRenderedSurfaces(surfaces);
|
|
||||||
const scene: VomScene = {
|
|
||||||
viewport: { width: 1280, height: 720 },
|
|
||||||
nodes: [
|
|
||||||
{
|
|
||||||
id: 10,
|
|
||||||
parentId: null,
|
|
||||||
backendNodeId: 1,
|
|
||||||
frameId: "main",
|
|
||||||
tag: "body",
|
|
||||||
role: "RootWebArea",
|
|
||||||
rect: null,
|
|
||||||
paintOrder: 0,
|
|
||||||
position: "static",
|
|
||||||
pointerEvents: "auto",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
expect(groups).toHaveLength(1);
|
|
||||||
expect(groups[0]?.members).toHaveLength(2);
|
|
||||||
expect(groups[0]?.label).toBeUndefined();
|
|
||||||
expect(
|
|
||||||
projectRenderedSurfaces(
|
|
||||||
scene,
|
|
||||||
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
|
|
||||||
groups,
|
|
||||||
),
|
|
||||||
).toMatchObject({
|
|
||||||
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not merge a contained canvas with a larger independent canvas", () => {
|
|
||||||
const groups = clusterRenderedSurfaces([
|
|
||||||
...discoverRenderedSurfaces(
|
|
||||||
[
|
|
||||||
document([
|
|
||||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
|
||||||
domNode(2, { parentBackendNodeId: 1 }),
|
|
||||||
domNode(3, {
|
|
||||||
parentBackendNodeId: 1,
|
|
||||||
rect: { x: 30, y: 40, w: 200, h: 100 },
|
|
||||||
}),
|
|
||||||
]),
|
|
||||||
],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
|
|
||||||
expect(groups).toHaveLength(2);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not merge coincident canvases owned by different containers", () => {
|
|
||||||
const groups = clusterRenderedSurfaces(
|
|
||||||
discoverRenderedSurfaces(
|
|
||||||
[
|
|
||||||
document([
|
|
||||||
domNode(1, { tag: "section", rect: null, localRect: null }),
|
|
||||||
domNode(2, { tag: "section", rect: null, localRect: null }),
|
|
||||||
domNode(3, { parentBackendNodeId: 1 }),
|
|
||||||
domNode(4, { parentBackendNodeId: 2 }),
|
|
||||||
]),
|
|
||||||
],
|
|
||||||
{ width: 1280, height: 720 },
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(groups).toHaveLength(2);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,782 +0,0 @@
|
|||||||
import { isVomReferenceNode, renderVom } from "@browser-skill/vom";
|
|
||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import type { CapturedNode } from "../capture";
|
|
||||||
import type { FrameDocument } from "../frame-document";
|
|
||||||
import { buildSemanticVomScene, type SemanticAxNode } from "../semantic-graph";
|
|
||||||
|
|
||||||
function dom(
|
|
||||||
backendNodeId: number,
|
|
||||||
parentBackendNodeId: number | null,
|
|
||||||
tag: string,
|
|
||||||
attrs: Record<string, string> = {},
|
|
||||||
textContent?: string,
|
|
||||||
): CapturedNode {
|
|
||||||
return {
|
|
||||||
backendNodeId,
|
|
||||||
parentBackendNodeId,
|
|
||||||
tag,
|
|
||||||
attrs,
|
|
||||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
|
||||||
paintOrder: backendNodeId,
|
|
||||||
position: "static",
|
|
||||||
pointerEvents: "auto",
|
|
||||||
...(textContent ? { textContent } : {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function document(
|
|
||||||
frameId: string,
|
|
||||||
axNodes: SemanticAxNode[],
|
|
||||||
domNodes: CapturedNode[],
|
|
||||||
frame?: Pick<FrameDocument<SemanticAxNode>, "parentFrameId" | "ownerBackendNodeId">,
|
|
||||||
): FrameDocument<SemanticAxNode> {
|
|
||||||
return {
|
|
||||||
frameId,
|
|
||||||
contextScopeId: frameId,
|
|
||||||
target: { tabId: 1 },
|
|
||||||
axNodes,
|
|
||||||
domNodes,
|
|
||||||
...frame,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("semantic VOM graph", () => {
|
|
||||||
it("preserves backend-less AX structure without making it referenceable", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["toolbar"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "toolbar",
|
|
||||||
parentId: "root",
|
|
||||||
role: { type: "role", value: "toolbar" },
|
|
||||||
name: { type: "computedString", value: "Formatting" },
|
|
||||||
childIds: ["bold", "italic"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "bold",
|
|
||||||
parentId: "toolbar",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Bold" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "italic",
|
|
||||||
parentId: "toolbar",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Italic" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(1, null, "body"), dom(2, 1, "button"), dom(3, 1, "button")],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const toolbar = scene.nodes.find((node) => node.role === "toolbar");
|
|
||||||
const buttons = scene.nodes.filter((node) => node.role === "button");
|
|
||||||
expect(toolbar).toEqual(expect.objectContaining({ name: "Formatting", referenceable: false }));
|
|
||||||
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
|
||||||
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
|
||||||
expect(renderVom(scene).refs.map((ref) => ref.backendNodeId)).toEqual([2, 3]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("resolves missing AX names through one DOM naming pipeline", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["labelled", "titled", "text"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "labelled",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "titled",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "text",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
dom(1, null, "body"),
|
|
||||||
dom(2, 1, "button", { "aria-label": "Add row" }),
|
|
||||||
dom(3, 1, "button", { title: "Settings" }),
|
|
||||||
dom(4, 1, "button"),
|
|
||||||
dom(5, 4, "span", {}, "Save"),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(scene.nodes.filter((node) => node.role === "button").map((node) => node.name)).toEqual([
|
|
||||||
"Add row",
|
|
||||||
"Settings",
|
|
||||||
"Save",
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("aggregates AX text at the semantic text frontier", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["covered", "inline-only", "repeated", "labelled"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "covered",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "paragraph" },
|
|
||||||
childIds: ["covered-static"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "covered-static",
|
|
||||||
parentId: "covered",
|
|
||||||
role: { type: "role", value: "StaticText" },
|
|
||||||
name: { type: "computedString", value: "Covered text" },
|
|
||||||
childIds: ["covered-inline"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "covered-inline",
|
|
||||||
parentId: "covered-static",
|
|
||||||
role: { type: "role", value: "InlineTextBox" },
|
|
||||||
name: { type: "computedString", value: "Covered text" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "inline-only",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "paragraph" },
|
|
||||||
childIds: ["inline-leaf"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "inline-leaf",
|
|
||||||
parentId: "inline-only",
|
|
||||||
role: { type: "role", value: "InlineTextBox" },
|
|
||||||
name: { type: "computedString", value: "Inline fallback" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "repeated",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "paragraph" },
|
|
||||||
childIds: ["first-go", "second-go"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "first-go",
|
|
||||||
parentId: "repeated",
|
|
||||||
role: { type: "role", value: "StaticText" },
|
|
||||||
name: { type: "computedString", value: "Go" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "second-go",
|
|
||||||
parentId: "repeated",
|
|
||||||
role: { type: "role", value: "StaticText" },
|
|
||||||
name: { type: "computedString", value: "Go" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "labelled",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 5,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
childIds: ["visible-label"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "visible-label",
|
|
||||||
parentId: "labelled",
|
|
||||||
role: { type: "role", value: "StaticText" },
|
|
||||||
name: { type: "computedString", value: "Visible text" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
dom(1, null, "body"),
|
|
||||||
dom(2, 1, "p"),
|
|
||||||
dom(3, 1, "p"),
|
|
||||||
dom(4, 1, "p"),
|
|
||||||
dom(5, 1, "button", { "aria-label": "Explicit action" }),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(scene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Covered text");
|
|
||||||
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Inline fallback");
|
|
||||||
expect(scene.nodes.find((node) => node.backendNodeId === 4)?.name).toBe("Go Go");
|
|
||||||
expect(scene.nodes.find((node) => node.backendNodeId === 5)?.name).toBe("Explicit action");
|
|
||||||
expect(renderVom(scene).text).not.toContain("Covered text Covered text");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uses stable identifiers only as a final name fallback", () => {
|
|
||||||
const base = document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["fallback", "labelled", "container"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "fallback",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "labelled",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Localized name" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "container",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
dom(1, null, "body"),
|
|
||||||
dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }),
|
|
||||||
dom(3, 1, "button", { id: "toolIgnoredFallback" }),
|
|
||||||
dom(4, 1, "div", { id: "toolbarContainer" }),
|
|
||||||
],
|
|
||||||
);
|
|
||||||
const staticScene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [base],
|
|
||||||
});
|
|
||||||
const enrichedScene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [base],
|
|
||||||
supplementalNames: new Map([["main\u00002", "插入行"]]),
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record");
|
|
||||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name");
|
|
||||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined();
|
|
||||||
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行");
|
|
||||||
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe(
|
|
||||||
"Localized name",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("isolates equal backend node ids in sibling frames", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 1000, height: 800 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["owner-a", "owner-b"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "owner-a",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 10,
|
|
||||||
role: { type: "role", value: "Iframe" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "owner-b",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 20,
|
|
||||||
role: { type: "role", value: "Iframe" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(1, null, "body"), dom(10, 1, "iframe"), dom(20, 1, "iframe")],
|
|
||||||
),
|
|
||||||
document(
|
|
||||||
"frame-a",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "button-a",
|
|
||||||
backendDOMNodeId: 5,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
childIds: ["text-a"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "text-a",
|
|
||||||
parentId: "button-a",
|
|
||||||
role: { type: "role", value: "StaticText" },
|
|
||||||
name: { type: "computedString", value: "Accept" },
|
|
||||||
childIds: ["inline-a"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "inline-a",
|
|
||||||
parentId: "text-a",
|
|
||||||
role: { type: "role", value: "InlineTextBox" },
|
|
||||||
name: { type: "computedString", value: "Accept" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(5, null, "button")],
|
|
||||||
{ parentFrameId: "main", ownerBackendNodeId: 10 },
|
|
||||||
),
|
|
||||||
document(
|
|
||||||
"frame-b",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "button-b",
|
|
||||||
backendDOMNodeId: 5,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
childIds: ["text-b"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "text-b",
|
|
||||||
parentId: "button-b",
|
|
||||||
role: { type: "role", value: "StaticText" },
|
|
||||||
name: { type: "computedString", value: "Decline" },
|
|
||||||
childIds: ["inline-b"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "inline-b",
|
|
||||||
parentId: "text-b",
|
|
||||||
role: { type: "role", value: "InlineTextBox" },
|
|
||||||
name: { type: "computedString", value: "Decline" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(5, null, "button")],
|
|
||||||
{ parentFrameId: "main", ownerBackendNodeId: 20 },
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const frameButtons = scene.nodes.filter((node) => node.backendNodeId === 5);
|
|
||||||
const owners = new Map(
|
|
||||||
scene.nodes
|
|
||||||
.filter((node) => node.backendNodeId === 10 || node.backendNodeId === 20)
|
|
||||||
.map((node) => [node.backendNodeId, node.id]),
|
|
||||||
);
|
|
||||||
expect(frameButtons.map((node) => node.frameId)).toEqual(["frame-a", "frame-b"]);
|
|
||||||
expect(frameButtons.map((node) => node.name)).toEqual(["Accept", "Decline"]);
|
|
||||||
expect(frameButtons.map((node) => node.parentId)).toEqual([owners.get(10), owners.get(20)]);
|
|
||||||
expect(renderVom(scene).refs.map((ref) => [ref.frameId, ref.backendNodeId])).toEqual([
|
|
||||||
["frame-a", 5],
|
|
||||||
["frame-b", 5],
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("requires render evidence before promoting DOM fallback semantics", () => {
|
|
||||||
const hiddenOwner = {
|
|
||||||
...dom(10, 1, "iframe", { hidden: "" }),
|
|
||||||
rect: null,
|
|
||||||
localRect: null,
|
|
||||||
rendered: false,
|
|
||||||
};
|
|
||||||
const hiddenButton = {
|
|
||||||
...dom(2, 1, "button", { "aria-label": "Hidden action" }),
|
|
||||||
rendered: false,
|
|
||||||
};
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(1, null, "body"), hiddenButton, hiddenOwner],
|
|
||||||
),
|
|
||||||
document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 10,
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
|
|
||||||
expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
|
|
||||||
expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
|
|
||||||
expect(renderVom(scene).refs).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[],
|
|
||||||
[dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(renderVom(scene).refs).toEqual([
|
|
||||||
expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("lets explicit DOM structure override ignored AX metadata", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["toolbar"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "toolbar",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
ignored: true,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["wrapper"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "wrapper",
|
|
||||||
parentId: "toolbar",
|
|
||||||
backendDOMNodeId: 5,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["bold", "italic"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "bold",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Bold" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "italic",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Italic" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
dom(1, null, "body"),
|
|
||||||
dom(2, 1, "div", { role: "toolbar" }),
|
|
||||||
dom(5, 2, "div"),
|
|
||||||
dom(3, 5, "button"),
|
|
||||||
dom(4, 5, "button"),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const toolbar = scene.nodes.find((node) => node.backendNodeId === 2);
|
|
||||||
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
|
||||||
expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" }));
|
|
||||||
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
|
||||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
|
||||||
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("infers an unnamed group only from independent interaction branches", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["wrapper"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "wrapper",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["first", "second"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "first",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "First" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "second",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Second" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const group = scene.nodes.find((node) => node.backendNodeId === 2);
|
|
||||||
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
|
||||||
expect(group).toEqual(expect.objectContaining({ role: "group" }));
|
|
||||||
expect(group?.name).toBeUndefined();
|
|
||||||
expect(group && isVomReferenceNode(group)).toBe(false);
|
|
||||||
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
|
||||||
|
|
||||||
const rendered = renderVom(scene);
|
|
||||||
const lines = rendered.text.split("\n");
|
|
||||||
const groupLine = lines.findIndex((line) => line.trim() === "group");
|
|
||||||
expect(groupLine).toBeGreaterThanOrEqual(0);
|
|
||||||
for (const ref of rendered.refs) {
|
|
||||||
expect(ref.line).toBeGreaterThan(groupLine);
|
|
||||||
expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it("never promotes a form value into its semantic name through supplemental evidence", () => {
|
|
||||||
const secret = "user@example.com";
|
|
||||||
const input = {
|
|
||||||
...dom(2, 1, "input", { type: "email" }),
|
|
||||||
formValue: secret,
|
|
||||||
formDefaultValue: "",
|
|
||||||
formState: "filled" as const,
|
|
||||||
};
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
supplementalNames: new Map([["main\u00002", secret]]),
|
|
||||||
identifierFallback: false,
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["email"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "email",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "textbox" },
|
|
||||||
value: { value: secret },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(1, null, "body"), input],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const field = scene.nodes.find((node) => node.backendNodeId === 2);
|
|
||||||
expect(field?.name).toBeUndefined();
|
|
||||||
const rendered = renderVom(scene, { redactValues: true });
|
|
||||||
expect(rendered.text).toContain("•••");
|
|
||||||
expect(JSON.stringify(rendered)).not.toContain(secret);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps uncertain single-branch wrappers flat", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["wrapper"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "wrapper",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["button"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "button",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Only action" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
|
||||||
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe(
|
|
||||||
scene.nodes.find((node) => node.backendNodeId === 1)?.id,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not infer across mixed content boundaries", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["wrapper"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "wrapper",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["first", "second", "paragraph"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "first",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "First" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "second",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Second" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "paragraph",
|
|
||||||
parentId: "wrapper",
|
|
||||||
backendDOMNodeId: 5,
|
|
||||||
role: { type: "role", value: "paragraph" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
dom(1, null, "body"),
|
|
||||||
dom(2, 1, "div"),
|
|
||||||
dom(3, 2, "button"),
|
|
||||||
dom(4, 2, "button"),
|
|
||||||
dom(5, 2, "p", {}, "Description"),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps only the innermost reliable inferred group", () => {
|
|
||||||
const scene = buildSemanticVomScene({
|
|
||||||
viewport: { width: 800, height: 600 },
|
|
||||||
rootFrameId: "main",
|
|
||||||
documents: [
|
|
||||||
document(
|
|
||||||
"main",
|
|
||||||
[
|
|
||||||
{
|
|
||||||
nodeId: "root",
|
|
||||||
backendDOMNodeId: 1,
|
|
||||||
role: { type: "role", value: "RootWebArea" },
|
|
||||||
childIds: ["outer"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "outer",
|
|
||||||
parentId: "root",
|
|
||||||
backendDOMNodeId: 2,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["inner", "third"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "inner",
|
|
||||||
parentId: "outer",
|
|
||||||
backendDOMNodeId: 3,
|
|
||||||
role: { type: "role", value: "generic" },
|
|
||||||
childIds: ["first", "second"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "first",
|
|
||||||
parentId: "inner",
|
|
||||||
backendDOMNodeId: 4,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "First" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "second",
|
|
||||||
parentId: "inner",
|
|
||||||
backendDOMNodeId: 5,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Second" },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
nodeId: "third",
|
|
||||||
parentId: "outer",
|
|
||||||
backendDOMNodeId: 6,
|
|
||||||
role: { type: "role", value: "button" },
|
|
||||||
name: { type: "computedString", value: "Third" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
dom(1, null, "body"),
|
|
||||||
dom(2, 1, "div"),
|
|
||||||
dom(3, 2, "div"),
|
|
||||||
dom(4, 3, "button"),
|
|
||||||
dom(5, 3, "button"),
|
|
||||||
dom(6, 2, "button"),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const groups = scene.nodes.filter((node) => node.role === "group");
|
|
||||||
expect(groups.map((node) => node.backendNodeId)).toEqual([3]);
|
|
||||||
expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,25 +1,16 @@
|
|||||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||||
// shared string table); requested computed styles are decoded through
|
// shared string table); we request exactly three computed styles so the
|
||||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||||
|
|
||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
import { clearHover, waitForHover } from "./hover-perception";
|
|
||||||
|
|
||||||
const REQUESTED_STYLES = [
|
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||||
"position",
|
|
||||||
"pointer-events",
|
|
||||||
"cursor",
|
|
||||||
"visibility",
|
|
||||||
"opacity",
|
|
||||||
"overflow-x",
|
|
||||||
"overflow-y",
|
|
||||||
] as const;
|
|
||||||
const STYLE_COL = Object.fromEntries(
|
const STYLE_COL = Object.fromEntries(
|
||||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||||
@@ -47,16 +38,6 @@ export interface CapturedNode {
|
|||||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||||
*/
|
*/
|
||||||
cursor?: string;
|
cursor?: string;
|
||||||
overflowX?: string;
|
|
||||||
overflowY?: string;
|
|
||||||
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
|
|
||||||
visuallySuppressed?: boolean;
|
|
||||||
/**
|
|
||||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
|
||||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
|
||||||
* nodes remain authoritative even when they are outside the viewport.
|
|
||||||
*/
|
|
||||||
rendered?: boolean;
|
|
||||||
textContent?: string;
|
textContent?: string;
|
||||||
formState?: "empty" | "filled" | "default";
|
formState?: "empty" | "filled" | "default";
|
||||||
formValue?: string;
|
formValue?: string;
|
||||||
@@ -479,6 +460,36 @@ function hoverStateExpression(): string {
|
|||||||
})()`;
|
})()`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
reject(captureAbortError());
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||||
|
await cdp
|
||||||
|
.send(tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: -10,
|
||||||
|
y: -10,
|
||||||
|
})
|
||||||
|
.catch(() =>
|
||||||
|
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
||||||
|
// best effort
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function capturedText(node: CapturedNode): string | undefined {
|
function capturedText(node: CapturedNode): string | undefined {
|
||||||
const value =
|
const value =
|
||||||
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||||
@@ -658,7 +669,7 @@ async function probeHoverSurfaces(
|
|||||||
try {
|
try {
|
||||||
await clearHover(cdp, tabId);
|
await clearHover(cdp, tabId);
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
await wait(HOVER_SETTLE_MS, options.signal);
|
||||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: hoverStateExpression(),
|
expression: hoverStateExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
@@ -673,7 +684,7 @@ async function probeHoverSurfaces(
|
|||||||
y: candidate.y,
|
y: candidate.y,
|
||||||
});
|
});
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
await wait(HOVER_SETTLE_MS, options.signal);
|
||||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: hoverStateExpression(),
|
expression: hoverStateExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
@@ -806,10 +817,6 @@ function parseDocumentNodes(
|
|||||||
const posCol = STYLE_COL.position;
|
const posCol = STYLE_COL.position;
|
||||||
const peCol = STYLE_COL["pointer-events"];
|
const peCol = STYLE_COL["pointer-events"];
|
||||||
const cursorCol = STYLE_COL.cursor;
|
const cursorCol = STYLE_COL.cursor;
|
||||||
const visibilityCol = STYLE_COL.visibility;
|
|
||||||
const opacityCol = STYLE_COL.opacity;
|
|
||||||
const overflowXCol = STYLE_COL["overflow-x"];
|
|
||||||
const overflowYCol = STYLE_COL["overflow-y"];
|
|
||||||
const inputValues = sparseIndexMap(dn.inputValue);
|
const inputValues = sparseIndexMap(dn.inputValue);
|
||||||
const textValues = sparseIndexMap(dn.textValue);
|
const textValues = sparseIndexMap(dn.textValue);
|
||||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||||
@@ -835,7 +842,6 @@ function parseDocumentNodes(
|
|||||||
|
|
||||||
// Build CapturedNodes.
|
// Build CapturedNodes.
|
||||||
const nodes: CapturedNode[] = [];
|
const nodes: CapturedNode[] = [];
|
||||||
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
|
|
||||||
for (let n = 0; n < count; n++) {
|
for (let n = 0; n < count; n++) {
|
||||||
if (excluded[n]) continue;
|
if (excluded[n]) continue;
|
||||||
const backendNodeId = dn.backendNodeId[n];
|
const backendNodeId = dn.backendNodeId[n];
|
||||||
@@ -855,11 +861,6 @@ function parseDocumentNodes(
|
|||||||
let position = "static";
|
let position = "static";
|
||||||
let pointerEvents = "auto";
|
let pointerEvents = "auto";
|
||||||
let cursor = "auto";
|
let cursor = "auto";
|
||||||
let overflowX = "visible";
|
|
||||||
let overflowY = "visible";
|
|
||||||
let rendered = false;
|
|
||||||
let visuallySuppressed =
|
|
||||||
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
|
|
||||||
const li = layoutByNode.get(n);
|
const li = layoutByNode.get(n);
|
||||||
if (li !== undefined) {
|
if (li !== undefined) {
|
||||||
const b = dl?.bounds?.[li];
|
const b = dl?.bounds?.[li];
|
||||||
@@ -880,21 +881,7 @@ function parseDocumentNodes(
|
|||||||
position = str(strings, styleRow[posCol]) || "static";
|
position = str(strings, styleRow[posCol]) || "static";
|
||||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
|
||||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
|
||||||
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
|
|
||||||
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
|
|
||||||
visuallySuppressed ||=
|
|
||||||
visibility === "hidden" ||
|
|
||||||
visibility === "collapse" ||
|
|
||||||
(Number.parseFloat(opacity) || 0) <= 0;
|
|
||||||
rendered =
|
|
||||||
localRect !== null &&
|
|
||||||
visibility !== "hidden" &&
|
|
||||||
visibility !== "collapse" &&
|
|
||||||
(Number.parseFloat(opacity) || 0) > 0;
|
|
||||||
}
|
}
|
||||||
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
|
|
||||||
|
|
||||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||||
// geometry and are never queried by callers.
|
// geometry and are never queried by callers.
|
||||||
@@ -929,10 +916,6 @@ function parseDocumentNodes(
|
|||||||
position,
|
position,
|
||||||
pointerEvents,
|
pointerEvents,
|
||||||
cursor,
|
cursor,
|
||||||
overflowX,
|
|
||||||
overflowY,
|
|
||||||
visuallySuppressed,
|
|
||||||
rendered,
|
|
||||||
textContent,
|
textContent,
|
||||||
...(formValue !== undefined ? { formValue } : {}),
|
...(formValue !== undefined ? { formValue } : {}),
|
||||||
...(tag === "input" || tag === "textarea"
|
...(tag === "input" || tag === "textarea"
|
||||||
@@ -1037,13 +1020,7 @@ export async function captureViewModel(
|
|||||||
options: CaptureViewModelOptions = {},
|
options: CaptureViewModelOptions = {},
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
let metrics: LayoutMetricsReply = {};
|
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
try {
|
|
||||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
|
||||||
} catch (error) {
|
|
||||||
if (isAbortError(error)) throw error;
|
|
||||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
|
||||||
}
|
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const dpr = devicePixelRatio(metrics);
|
const dpr = devicePixelRatio(metrics);
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
|||||||
@@ -33,19 +33,25 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
|
|||||||
|
|
||||||
function transformFrameNodes(
|
function transformFrameNodes(
|
||||||
nodes: CapturedNode[],
|
nodes: CapturedNode[],
|
||||||
projection: GeometryProjection | null,
|
projection: GeometryProjection,
|
||||||
): CapturedNode[] {
|
): CapturedNode[] {
|
||||||
return nodes.map((node) => ({
|
return nodes.map((node) => ({
|
||||||
...node,
|
...node,
|
||||||
rect: projection
|
rect: (() => {
|
||||||
? (() => {
|
const bounds = projectRectToViewport(node.rect, projection);
|
||||||
const bounds = projectRectToViewport(node.rect, projection);
|
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
})(),
|
||||||
})()
|
|
||||||
: null,
|
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||||
|
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||||
|
const parentNodes = frame.parentFrameId
|
||||||
|
? captured.frameNodes?.get(frame.parentFrameId)
|
||||||
|
: captured.nodes;
|
||||||
|
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
async function captureMissingFrameDocuments(
|
async function captureMissingFrameDocuments(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -59,20 +65,14 @@ async function captureMissingFrameDocuments(
|
|||||||
|
|
||||||
for (const frame of graph.frames) {
|
for (const frame of graph.frames) {
|
||||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||||
|
const owner = frameOwnerNode(captured, frame);
|
||||||
|
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||||
try {
|
try {
|
||||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||||
signal,
|
signal,
|
||||||
});
|
});
|
||||||
let projection: GeometryProjection | null = null;
|
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||||
try {
|
if (!projection) continue;
|
||||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
|
||||||
} catch (err) {
|
|
||||||
if (isAbortError(err)) throw err;
|
|
||||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
|
||||||
frameId: frame.frameId,
|
|
||||||
err,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||||
@@ -85,13 +85,11 @@ async function captureMissingFrameDocuments(
|
|||||||
for (const [childFrameId, nodes] of childFrames) {
|
for (const [childFrameId, nodes] of childFrames) {
|
||||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||||
}
|
}
|
||||||
if (frame.ownerBackendNodeId !== undefined) {
|
captured.iframeNodes.set(
|
||||||
captured.iframeNodes.set(
|
frame.ownerBackendNodeId,
|
||||||
frame.ownerBackendNodeId,
|
captured.frameNodes.get(frame.frameId) ?? [],
|
||||||
captured.frameNodes.get(frame.frameId) ?? [],
|
);
|
||||||
);
|
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
|
||||||
}
|
|
||||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||||
|
|||||||
@@ -1,37 +0,0 @@
|
|||||||
import type { CdpRunner } from "../shared";
|
|
||||||
|
|
||||||
function abortError(): Error {
|
|
||||||
const error = new Error("hover perception aborted");
|
|
||||||
error.name = "AbortError";
|
|
||||||
return error;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function waitForHover(ms: number, signal?: AbortSignal): Promise<void> {
|
|
||||||
if (signal?.aborted) throw abortError();
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
const timer = setTimeout(() => {
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
resolve();
|
|
||||||
}, ms);
|
|
||||||
const onAbort = () => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
reject(abortError());
|
|
||||||
};
|
|
||||||
signal?.addEventListener("abort", onAbort, { once: true });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
|
||||||
await cdp
|
|
||||||
.send(tabId, "Input.dispatchMouseEvent", {
|
|
||||||
type: "mouseMoved",
|
|
||||||
x: -10,
|
|
||||||
y: -10,
|
|
||||||
})
|
|
||||||
.catch(() =>
|
|
||||||
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
|
||||||
// Best effort cleanup.
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,296 +0,0 @@
|
|||||||
import { isVomReferenceNode } from "@browser-skill/vom";
|
|
||||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
|
||||||
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
|
||||||
import { clearHover, waitForHover } from "./hover-perception";
|
|
||||||
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
|
||||||
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
|
||||||
|
|
||||||
const MAX_TOOLTIP_CANDIDATES = 8;
|
|
||||||
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
|
||||||
const TOOLTIP_SETTLE_MS = 250;
|
|
||||||
const CACHE_TTL_MS = 5 * 60_000;
|
|
||||||
const MAX_CACHE_ENTRIES = 500;
|
|
||||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
|
||||||
|
|
||||||
interface TooltipCacheEntry {
|
|
||||||
name: string;
|
|
||||||
expiresAt: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
|
||||||
document: FrameDocument<T>;
|
|
||||||
backendNodeId: number;
|
|
||||||
x: number;
|
|
||||||
y: number;
|
|
||||||
stableIdentifier: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface TooltipNameProbeOptions {
|
|
||||||
signal?: AbortSignal;
|
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
|
||||||
|
|
||||||
function cacheKey<T extends FrameOwnedAxNode>(
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
backendNodeId: number,
|
|
||||||
): string {
|
|
||||||
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function cachedName<T extends FrameOwnedAxNode>(
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
backendNodeId: number,
|
|
||||||
): string | undefined {
|
|
||||||
const key = cacheKey(document, backendNodeId);
|
|
||||||
const entry = tooltipCache.get(key);
|
|
||||||
if (!entry) return undefined;
|
|
||||||
if (entry.expiresAt <= Date.now()) {
|
|
||||||
tooltipCache.delete(key);
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
return entry.name;
|
|
||||||
}
|
|
||||||
|
|
||||||
function storeName<T extends FrameOwnedAxNode>(
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
backendNodeId: number,
|
|
||||||
name: string,
|
|
||||||
): void {
|
|
||||||
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
|
||||||
const oldest = tooltipCache.keys().next().value;
|
|
||||||
if (oldest !== undefined) tooltipCache.delete(oldest);
|
|
||||||
}
|
|
||||||
tooltipCache.set(cacheKey(document, backendNodeId), {
|
|
||||||
name,
|
|
||||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function tooltipCollectorFunction(): string {
|
|
||||||
return `function() {
|
|
||||||
const ownRoot = this && typeof this.getRootNode === "function" ? this.getRootNode() : null;
|
|
||||||
const roots = ownRoot && ownRoot !== document ? [ownRoot, document] : [document];
|
|
||||||
const items = [];
|
|
||||||
const seen = new Set();
|
|
||||||
for (const root of roots) {
|
|
||||||
for (const el of Array.from(root.querySelectorAll("[role='tooltip']"))) {
|
|
||||||
const style = getComputedStyle(el);
|
|
||||||
const rect = el.getBoundingClientRect();
|
|
||||||
if (rect.width <= 0 || rect.height <= 0) continue;
|
|
||||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
|
||||||
const text = String(el.getAttribute("aria-label") || el.getAttribute("title") || el.textContent || "")
|
|
||||||
.replace(/\\s+/g, " ")
|
|
||||||
.trim();
|
|
||||||
if (!text || text.length > 160 || seen.has(text)) continue;
|
|
||||||
seen.add(text);
|
|
||||||
items.push(text);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items.slice(0, 20);
|
|
||||||
}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function cleanTooltipDiff(before: string[], after: string[]): string | undefined {
|
|
||||||
const existing = new Set(before.map((value) => value.toLowerCase()));
|
|
||||||
const added = [...new Set(after.map((value) => value.replace(/\s+/g, " ").trim()))].filter(
|
|
||||||
(value) => value && !existing.has(value.toLowerCase()),
|
|
||||||
);
|
|
||||||
return added.length === 1 ? added[0] : undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function candidates<T extends FrameOwnedAxNode>(
|
|
||||||
documents: FrameDocument<T>[],
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
): TooltipCandidate<T>[] {
|
|
||||||
const documentByFrame = new Map(documents.map((document) => [document.frameId, document]));
|
|
||||||
const byFrame = new Map<string, TooltipCandidate<T>[]>();
|
|
||||||
for (const node of graph.nodes.values()) {
|
|
||||||
if (
|
|
||||||
node.vom.name ||
|
|
||||||
FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") ||
|
|
||||||
!node.referenceable ||
|
|
||||||
node.backendNodeId === undefined ||
|
|
||||||
!isVomReferenceNode({
|
|
||||||
...node.vom,
|
|
||||||
id: 0,
|
|
||||||
parentId: null,
|
|
||||||
backendNodeId: node.backendNodeId,
|
|
||||||
frameId: node.frameId,
|
|
||||||
referenceable: true,
|
|
||||||
})
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const document = documentByFrame.get(node.frameId);
|
|
||||||
const rect = node.dom?.rect;
|
|
||||||
if (!document || !rect || rect.w <= 0 || rect.h <= 0) continue;
|
|
||||||
if (
|
|
||||||
rect.x + rect.w <= 0 ||
|
|
||||||
rect.y + rect.h <= 0 ||
|
|
||||||
rect.x >= graph.viewport.width ||
|
|
||||||
rect.y >= graph.viewport.height
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
node.vom.disabled ||
|
|
||||||
node.vom.inert ||
|
|
||||||
node.vom.pointerEvents === "none" ||
|
|
||||||
node.vom.sensitive
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const frameCandidates = byFrame.get(node.frameId) ?? [];
|
|
||||||
frameCandidates.push({
|
|
||||||
document,
|
|
||||||
backendNodeId: node.backendNodeId,
|
|
||||||
x: (Math.max(0, rect.x) + Math.min(graph.viewport.width, rect.x + rect.w)) / 2,
|
|
||||||
y: (Math.max(0, rect.y) + Math.min(graph.viewport.height, rect.y + rect.h)) / 2,
|
|
||||||
stableIdentifier: stableIdentifierName(node.dom?.attrs ?? {}) !== undefined,
|
|
||||||
});
|
|
||||||
byFrame.set(node.frameId, frameCandidates);
|
|
||||||
}
|
|
||||||
|
|
||||||
const out: TooltipCandidate<T>[] = [];
|
|
||||||
for (const stableIdentifier of [true, false]) {
|
|
||||||
const queues = documents.map((document) =>
|
|
||||||
(byFrame.get(document.frameId) ?? []).filter(
|
|
||||||
(candidate) => candidate.stableIdentifier === stableIdentifier,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
for (let index = 0; queues.some((queue) => index < queue.length); index += 1) {
|
|
||||||
for (const queue of queues) {
|
|
||||||
const candidate = queue[index];
|
|
||||||
if (candidate) out.push(candidate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function executionContextId<T extends FrameOwnedAxNode>(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
): Promise<number | undefined> {
|
|
||||||
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
|
||||||
cdp,
|
|
||||||
document.target,
|
|
||||||
"Page.createIsolatedWorld",
|
|
||||||
{
|
|
||||||
frameId: document.frameId,
|
|
||||||
worldName: "bsk-vom-tooltip",
|
|
||||||
grantUniveralAccess: false,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
return result.executionContextId;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolvedObjectId<T extends FrameOwnedAxNode>(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
contextId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
): Promise<string | undefined> {
|
|
||||||
const result = await sendToCdpTarget<{
|
|
||||||
object?: { objectId?: string };
|
|
||||||
}>(cdp, document.target, "DOM.resolveNode", {
|
|
||||||
backendNodeId,
|
|
||||||
executionContextId: contextId,
|
|
||||||
});
|
|
||||||
return result.object?.objectId;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function tooltipTexts<T extends FrameOwnedAxNode>(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
objectId: string,
|
|
||||||
): Promise<string[]> {
|
|
||||||
const result = await sendToCdpTarget<{
|
|
||||||
result?: { value?: string[] };
|
|
||||||
}>(cdp, document.target, "Runtime.callFunctionOn", {
|
|
||||||
functionDeclaration: tooltipCollectorFunction(),
|
|
||||||
objectId,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
return result.result?.value ?? [];
|
|
||||||
}
|
|
||||||
|
|
||||||
async function releaseObject<T extends FrameOwnedAxNode>(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
document: FrameDocument<T>,
|
|
||||||
objectId: string,
|
|
||||||
): Promise<void> {
|
|
||||||
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObject", { objectId }).catch(
|
|
||||||
() => undefined,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
documents: FrameDocument<T>[],
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
options: TooltipNameProbeOptions = {},
|
|
||||||
): Promise<Map<string, string>> {
|
|
||||||
const names = new Map<string, string>();
|
|
||||||
const pending: TooltipCandidate<T>[] = [];
|
|
||||||
for (const candidate of candidates(documents, graph)) {
|
|
||||||
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
|
||||||
if (cached) {
|
|
||||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
|
||||||
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
|
||||||
pending.push(candidate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (pending.length === 0) return names;
|
|
||||||
|
|
||||||
const started = Date.now();
|
|
||||||
const contexts = new Map<string, number>();
|
|
||||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
|
||||||
try {
|
|
||||||
for (const candidate of pending) {
|
|
||||||
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
|
||||||
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
|
||||||
let objectId: string | undefined;
|
|
||||||
try {
|
|
||||||
let contextId = contexts.get(candidate.document.frameId);
|
|
||||||
if (contextId === undefined) {
|
|
||||||
contextId = await executionContextId(cdp, candidate.document);
|
|
||||||
if (contextId === undefined) continue;
|
|
||||||
contexts.set(candidate.document.frameId, contextId);
|
|
||||||
}
|
|
||||||
objectId = await resolvedObjectId(
|
|
||||||
cdp,
|
|
||||||
candidate.document,
|
|
||||||
contextId,
|
|
||||||
candidate.backendNodeId,
|
|
||||||
);
|
|
||||||
if (!objectId) continue;
|
|
||||||
await clearHover(cdp, tabId);
|
|
||||||
const before = await tooltipTexts(cdp, candidate.document, objectId);
|
|
||||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
|
||||||
type: "mouseMoved",
|
|
||||||
x: candidate.x,
|
|
||||||
y: candidate.y,
|
|
||||||
});
|
|
||||||
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
|
||||||
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
|
||||||
const name = cleanTooltipDiff(before, after);
|
|
||||||
if (!name) continue;
|
|
||||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
|
||||||
storeName(candidate.document, candidate.backendNodeId, name);
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
|
||||||
} finally {
|
|
||||||
if (objectId) await releaseObject(cdp, candidate.document, objectId);
|
|
||||||
await clearHover(cdp, tabId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await clearHover(cdp, tabId);
|
|
||||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
|
||||||
}
|
|
||||||
return names;
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
import type { Rect } from "@browser-skill/vom";
|
|
||||||
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
|
||||||
|
|
||||||
const STACK_IOU_RATIO = 0.9;
|
|
||||||
const STACK_SIZE_RATIO = 0.9;
|
|
||||||
|
|
||||||
function area(rect: Rect): number {
|
|
||||||
return Math.max(0, rect.w) * Math.max(0, rect.h);
|
|
||||||
}
|
|
||||||
|
|
||||||
function intersectionArea(a: Rect, b: Rect): number {
|
|
||||||
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
|
|
||||||
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
|
|
||||||
return width * height;
|
|
||||||
}
|
|
||||||
|
|
||||||
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
|
||||||
if (a.frameId !== b.frameId) return false;
|
|
||||||
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
|
|
||||||
const aArea = area(a.visibleRect);
|
|
||||||
const bArea = area(b.visibleRect);
|
|
||||||
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
|
|
||||||
const union = aArea + bArea - intersection;
|
|
||||||
const widthRatio =
|
|
||||||
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
|
|
||||||
const heightRatio =
|
|
||||||
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
|
|
||||||
return (
|
|
||||||
union > 0 &&
|
|
||||||
intersection / union >= STACK_IOU_RATIO &&
|
|
||||||
widthRatio >= STACK_SIZE_RATIO &&
|
|
||||||
heightRatio >= STACK_SIZE_RATIO
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
|
|
||||||
const representative = [...members].sort(
|
|
||||||
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
|
|
||||||
)[0];
|
|
||||||
return {
|
|
||||||
frameId: representative.frameId,
|
|
||||||
members,
|
|
||||||
representative,
|
|
||||||
label: members.map((surface) => surface.label).find((label) => label !== undefined),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
|
|
||||||
const groups: RenderedSurface[][] = [];
|
|
||||||
for (const surface of surfaces) {
|
|
||||||
const matching = groups.find((group) =>
|
|
||||||
group.some((member) => sameVisualStack(member, surface)),
|
|
||||||
);
|
|
||||||
if (matching) matching.push(surface);
|
|
||||||
else groups.push([surface]);
|
|
||||||
}
|
|
||||||
return groups
|
|
||||||
.map(toGroup)
|
|
||||||
.sort(
|
|
||||||
(a, b) =>
|
|
||||||
a.frameId.localeCompare(b.frameId) ||
|
|
||||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
|
||||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
|
||||||
import type { CapturedNode } from "../capture";
|
|
||||||
import type { CapturedFrameDocument } from "../frame-capture";
|
|
||||||
import type { SemanticAxNode } from "../semantic-graph";
|
|
||||||
import type { RenderedSurface } from "./types";
|
|
||||||
|
|
||||||
function clean(value: string | undefined): string | undefined {
|
|
||||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
|
||||||
return normalized || undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
|
|
||||||
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
|
|
||||||
}
|
|
||||||
|
|
||||||
function intersectRect(a: Rect, b: Rect): Rect | null {
|
|
||||||
const x = Math.max(a.x, b.x);
|
|
||||||
const y = Math.max(a.y, b.y);
|
|
||||||
const right = Math.min(a.x + a.w, b.x + b.w);
|
|
||||||
const bottom = Math.min(a.y + a.h, b.y + b.h);
|
|
||||||
if (right <= x || bottom <= y) return null;
|
|
||||||
return { x, y, w: right - x, h: bottom - y };
|
|
||||||
}
|
|
||||||
|
|
||||||
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
|
|
||||||
|
|
||||||
function clipToOverflowAncestors(
|
|
||||||
node: CapturedNode,
|
|
||||||
rect: Rect,
|
|
||||||
nodesByBackend: ReadonlyMap<number, CapturedNode>,
|
|
||||||
): Rect | null {
|
|
||||||
let clipped: Rect | null = rect;
|
|
||||||
let parentBackendNodeId = node.parentBackendNodeId;
|
|
||||||
let guard = 0;
|
|
||||||
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
|
|
||||||
const parent = nodesByBackend.get(parentBackendNodeId);
|
|
||||||
if (!parent) break;
|
|
||||||
if (parent.rect) {
|
|
||||||
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
|
|
||||||
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
|
|
||||||
if (clipX || clipY) {
|
|
||||||
const clipRect = {
|
|
||||||
x: clipX ? parent.rect.x : clipped.x,
|
|
||||||
y: clipY ? parent.rect.y : clipped.y,
|
|
||||||
w: clipX ? parent.rect.w : clipped.w,
|
|
||||||
h: clipY ? parent.rect.h : clipped.h,
|
|
||||||
};
|
|
||||||
clipped = intersectRect(clipped, clipRect);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
parentBackendNodeId = parent.parentBackendNodeId;
|
|
||||||
guard += 1;
|
|
||||||
}
|
|
||||||
return clipped;
|
|
||||||
}
|
|
||||||
|
|
||||||
function excludedByDomPolicy(
|
|
||||||
node: CapturedNode,
|
|
||||||
nodesByBackend: Map<number, CapturedNode>,
|
|
||||||
excludedBackendNodeIds: ReadonlySet<number>,
|
|
||||||
): boolean {
|
|
||||||
let current: CapturedNode | undefined = node;
|
|
||||||
let guard = 0;
|
|
||||||
while (current && guard <= nodesByBackend.size) {
|
|
||||||
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
|
|
||||||
if (current.visuallySuppressed === true) return true;
|
|
||||||
const attrs = current.attrs;
|
|
||||||
if (
|
|
||||||
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
|
||||||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
|
||||||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
current =
|
|
||||||
current.parentBackendNodeId === null
|
|
||||||
? undefined
|
|
||||||
: nodesByBackend.get(current.parentBackendNodeId);
|
|
||||||
guard += 1;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function discoverRenderedSurfaces(
|
|
||||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
|
||||||
viewport: Viewport,
|
|
||||||
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
|
|
||||||
): RenderedSurface[] {
|
|
||||||
const surfaces: RenderedSurface[] = [];
|
|
||||||
for (const document of documents) {
|
|
||||||
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
|
|
||||||
for (const node of document.domNodes) {
|
|
||||||
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
|
|
||||||
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
|
|
||||||
const viewportRect = intersectViewport(node.rect, viewport);
|
|
||||||
const visibleRect = viewportRect
|
|
||||||
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
|
|
||||||
: null;
|
|
||||||
if (!visibleRect) continue;
|
|
||||||
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
|
|
||||||
surfaces.push({
|
|
||||||
renderingKind: "canvas",
|
|
||||||
frameId: document.frameId,
|
|
||||||
backendNodeId: node.backendNodeId,
|
|
||||||
parentBackendNodeId: node.parentBackendNodeId,
|
|
||||||
visibleRect,
|
|
||||||
paintOrder: node.paintOrder,
|
|
||||||
...(label ? { label } : {}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return surfaces;
|
|
||||||
}
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
export { clusterRenderedSurfaces } from "./cluster";
|
|
||||||
export { discoverRenderedSurfaces } from "./discovery";
|
|
||||||
export { projectRenderedSurfaces } from "./project";
|
|
||||||
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
|
|
||||||
import type { CapturedFrameDocument } from "../frame-capture";
|
|
||||||
import type { SemanticAxNode } from "../semantic-graph";
|
|
||||||
import type { RenderedSurfaceGroup } from "./types";
|
|
||||||
|
|
||||||
function frameBackendKey(frameId: string, backendNodeId: number): string {
|
|
||||||
return `${frameId}\u0000${backendNodeId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ProjectionIndex {
|
|
||||||
sceneByBackend: ReadonlyMap<string, number>;
|
|
||||||
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
|
|
||||||
domByFrame: ReadonlyMap<
|
|
||||||
string,
|
|
||||||
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
|
|
||||||
>;
|
|
||||||
rootSceneIdByFrame: ReadonlyMap<string, number>;
|
|
||||||
}
|
|
||||||
|
|
||||||
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
|
|
||||||
const document = index.documentByFrame.get(group.frameId);
|
|
||||||
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
|
|
||||||
let parentBackendNodeId = group.representative.parentBackendNodeId;
|
|
||||||
let guard = 0;
|
|
||||||
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
|
|
||||||
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
|
|
||||||
if (sceneId !== undefined) return sceneId;
|
|
||||||
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
|
|
||||||
guard += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
|
|
||||||
const owner = index.sceneByBackend.get(
|
|
||||||
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
|
|
||||||
);
|
|
||||||
if (owner !== undefined) return owner;
|
|
||||||
}
|
|
||||||
|
|
||||||
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildProjectionIndex(
|
|
||||||
scene: VomScene,
|
|
||||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
|
||||||
): ProjectionIndex {
|
|
||||||
const rootSceneIdByFrame = new Map<string, number>();
|
|
||||||
for (const node of scene.nodes) {
|
|
||||||
if (
|
|
||||||
node.frameId &&
|
|
||||||
!rootSceneIdByFrame.has(node.frameId) &&
|
|
||||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
|
|
||||||
) {
|
|
||||||
rootSceneIdByFrame.set(node.frameId, node.id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
sceneByBackend: new Map(
|
|
||||||
scene.nodes.flatMap((node) =>
|
|
||||||
node.backendNodeId === undefined
|
|
||||||
? []
|
|
||||||
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
|
|
||||||
),
|
|
||||||
),
|
|
||||||
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
|
|
||||||
domByFrame: new Map(
|
|
||||||
documents.map((document) => [
|
|
||||||
document.frameId,
|
|
||||||
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
rootSceneIdByFrame,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectRenderedSurfaces(
|
|
||||||
scene: VomScene,
|
|
||||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
|
||||||
groups: RenderedSurfaceGroup[],
|
|
||||||
): VomScene {
|
|
||||||
const index = buildProjectionIndex(scene, documents);
|
|
||||||
const visualSurfaces: VomVisualSurface[] = [];
|
|
||||||
|
|
||||||
for (const group of groups) {
|
|
||||||
const parentId = nearestSceneParent(index, group);
|
|
||||||
visualSurfaces.push({
|
|
||||||
parentId,
|
|
||||||
backendNodeId: group.representative.backendNodeId,
|
|
||||||
frameId: group.frameId,
|
|
||||||
renderingKind: "canvas",
|
|
||||||
visibleRect: group.representative.visibleRect,
|
|
||||||
...(group.label ? { label: group.label } : {}),
|
|
||||||
memberCount: group.members.length,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
...scene,
|
|
||||||
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
import type { Rect } from "@browser-skill/vom";
|
|
||||||
|
|
||||||
export interface RenderedSurface {
|
|
||||||
renderingKind: "canvas";
|
|
||||||
frameId: string;
|
|
||||||
backendNodeId: number;
|
|
||||||
parentBackendNodeId: number | null;
|
|
||||||
visibleRect: Rect;
|
|
||||||
paintOrder: number;
|
|
||||||
label?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RenderedSurfaceGroup {
|
|
||||||
frameId: string;
|
|
||||||
members: RenderedSurface[];
|
|
||||||
representative: RenderedSurface;
|
|
||||||
label?: string;
|
|
||||||
}
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
import type { Viewport } from "@browser-skill/vom";
|
|
||||||
import type { FrameDocument } from "../frame-document";
|
|
||||||
import {
|
|
||||||
frameAxKey,
|
|
||||||
frameBackendKey,
|
|
||||||
type SemanticAxNode,
|
|
||||||
type SemanticFrame,
|
|
||||||
type SemanticGraph,
|
|
||||||
type SemanticGraphNode,
|
|
||||||
type SemanticNodeId,
|
|
||||||
} from "./types";
|
|
||||||
|
|
||||||
export interface BuildSemanticGraphInput {
|
|
||||||
documents: FrameDocument<SemanticAxNode>[];
|
|
||||||
viewport: Viewport;
|
|
||||||
rootFrameId?: string;
|
|
||||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
|
||||||
}
|
|
||||||
|
|
||||||
function domNodeId(frameId: string, backendNodeId: number): SemanticNodeId {
|
|
||||||
return JSON.stringify([frameId, "dom", backendNodeId]);
|
|
||||||
}
|
|
||||||
|
|
||||||
function axNodeId(frameId: string, nodeId: string): SemanticNodeId {
|
|
||||||
return JSON.stringify([frameId, "ax", nodeId]);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGraph {
|
|
||||||
const rootFrameId =
|
|
||||||
input.rootFrameId ??
|
|
||||||
input.documents.find((document) => !document.parentFrameId)?.frameId ??
|
|
||||||
input.documents[0]?.frameId ??
|
|
||||||
"root";
|
|
||||||
const frames = new Map<string, SemanticFrame>();
|
|
||||||
const nodes = new Map<SemanticNodeId, SemanticGraphNode>();
|
|
||||||
const nodeIdsByFrame = new Map<string, SemanticNodeId[]>();
|
|
||||||
const nodeByFrameBackend = new Map<string, SemanticNodeId>();
|
|
||||||
const nodeByFrameAx = new Map<string, SemanticNodeId>();
|
|
||||||
|
|
||||||
for (const [frameOrder, document] of input.documents.entries()) {
|
|
||||||
frames.set(document.frameId, {
|
|
||||||
frameId: document.frameId,
|
|
||||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
|
||||||
...(document.ownerBackendNodeId !== undefined
|
|
||||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
|
||||||
: {}),
|
|
||||||
contextScopeId: document.contextScopeId,
|
|
||||||
target: document.target,
|
|
||||||
...(document.url ? { url: document.url } : {}),
|
|
||||||
order: frameOrder,
|
|
||||||
});
|
|
||||||
|
|
||||||
const frameNodeIds: SemanticNodeId[] = [];
|
|
||||||
nodeIdsByFrame.set(document.frameId, frameNodeIds);
|
|
||||||
for (const [index, dom] of document.domNodes.entries()) {
|
|
||||||
const id = domNodeId(document.frameId, dom.backendNodeId);
|
|
||||||
const node: SemanticGraphNode = {
|
|
||||||
id,
|
|
||||||
frameId: document.frameId,
|
|
||||||
backendNodeId: dom.backendNodeId,
|
|
||||||
dom,
|
|
||||||
sourceOrder: index,
|
|
||||||
excluded:
|
|
||||||
document.frameId === rootFrameId &&
|
|
||||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
|
|
||||||
};
|
|
||||||
nodes.set(id, node);
|
|
||||||
frameNodeIds.push(id);
|
|
||||||
nodeByFrameBackend.set(frameBackendKey(document.frameId, dom.backendNodeId), id);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [index, ax] of document.axNodes.entries()) {
|
|
||||||
const joinedId =
|
|
||||||
typeof ax.backendDOMNodeId === "number"
|
|
||||||
? nodeByFrameBackend.get(frameBackendKey(document.frameId, ax.backendDOMNodeId))
|
|
||||||
: undefined;
|
|
||||||
const id = joinedId ?? axNodeId(document.frameId, ax.nodeId);
|
|
||||||
const existing = nodes.get(id);
|
|
||||||
if (existing) {
|
|
||||||
existing.ax = ax;
|
|
||||||
existing.axNodeId = ax.nodeId;
|
|
||||||
} else {
|
|
||||||
nodes.set(id, {
|
|
||||||
id,
|
|
||||||
frameId: document.frameId,
|
|
||||||
...(typeof ax.backendDOMNodeId === "number"
|
|
||||||
? { backendNodeId: ax.backendDOMNodeId }
|
|
||||||
: {}),
|
|
||||||
axNodeId: ax.nodeId,
|
|
||||||
ax,
|
|
||||||
sourceOrder: document.domNodes.length + index,
|
|
||||||
excluded:
|
|
||||||
document.frameId === rootFrameId &&
|
|
||||||
typeof ax.backendDOMNodeId === "number" &&
|
|
||||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
|
|
||||||
});
|
|
||||||
frameNodeIds.push(id);
|
|
||||||
}
|
|
||||||
nodeByFrameAx.set(frameAxKey(document.frameId, ax.nodeId), id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const document of input.documents) {
|
|
||||||
for (const dom of document.domNodes) {
|
|
||||||
const id = nodeByFrameBackend.get(frameBackendKey(document.frameId, dom.backendNodeId));
|
|
||||||
const node = id ? nodes.get(id) : undefined;
|
|
||||||
if (!node || dom.parentBackendNodeId === null) continue;
|
|
||||||
node.domParentId = nodeByFrameBackend.get(
|
|
||||||
frameBackendKey(document.frameId, dom.parentBackendNodeId),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
for (const ax of document.axNodes) {
|
|
||||||
const id = nodeByFrameAx.get(frameAxKey(document.frameId, ax.nodeId));
|
|
||||||
const node = id ? nodes.get(id) : undefined;
|
|
||||||
if (!node || !ax.parentId) continue;
|
|
||||||
node.axParentId = nodeByFrameAx.get(frameAxKey(document.frameId, ax.parentId));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
|
||||||
const excludedQueue: SemanticNodeId[] = [];
|
|
||||||
for (const node of nodes.values()) {
|
|
||||||
if (node.axParentId) {
|
|
||||||
const children = axChildren.get(node.axParentId) ?? [];
|
|
||||||
children.push(node.id);
|
|
||||||
axChildren.set(node.axParentId, children);
|
|
||||||
}
|
|
||||||
if (node.excluded) excludedQueue.push(node.id);
|
|
||||||
}
|
|
||||||
for (let index = 0; index < excludedQueue.length; index += 1) {
|
|
||||||
for (const childId of axChildren.get(excludedQueue[index]) ?? []) {
|
|
||||||
const child = nodes.get(childId);
|
|
||||||
if (!child || child.excluded) continue;
|
|
||||||
child.excluded = true;
|
|
||||||
excludedQueue.push(childId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const frame of frames.values()) {
|
|
||||||
if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) continue;
|
|
||||||
frame.ownerNodeId = nodeByFrameBackend.get(
|
|
||||||
frameBackendKey(frame.parentFrameId, frame.ownerBackendNodeId),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
viewport: input.viewport,
|
|
||||||
rootFrameId,
|
|
||||||
frames,
|
|
||||||
nodes,
|
|
||||||
nodeIdsByFrame,
|
|
||||||
nodeByFrameBackend,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
import type { Viewport, VomScene } from "@browser-skill/vom";
|
|
||||||
import type { FrameDocument } from "../frame-document";
|
|
||||||
import { buildSemanticGraph as buildGraph } from "./build";
|
|
||||||
import { projectSemanticGraph as projectGraph } from "./project";
|
|
||||||
import { resolveSemanticGraph as resolveGraph } from "./resolve";
|
|
||||||
import { normalizeSemanticStructure as normalizeStructure } from "./structure";
|
|
||||||
import type {
|
|
||||||
ResolvedSemanticGraph,
|
|
||||||
SemanticAxNode,
|
|
||||||
SemanticGraph,
|
|
||||||
StructuredSemanticGraph,
|
|
||||||
} from "./types";
|
|
||||||
|
|
||||||
export type { SemanticAxNode } from "./types";
|
|
||||||
|
|
||||||
export interface SemanticVomInput {
|
|
||||||
documents: FrameDocument<SemanticAxNode>[];
|
|
||||||
viewport: Viewport;
|
|
||||||
rootFrameId?: string;
|
|
||||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph {
|
|
||||||
return buildGraph(input);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveSemanticGraph(
|
|
||||||
graph: SemanticGraph,
|
|
||||||
options: {
|
|
||||||
supplementalNames?: ReadonlyMap<string, string>;
|
|
||||||
identifierFallback?: boolean;
|
|
||||||
} = {},
|
|
||||||
): ResolvedSemanticGraph {
|
|
||||||
return resolveGraph(graph, options);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
|
|
||||||
return normalizeStructure(graph);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
|
|
||||||
return projectGraph(graph);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildSemanticVomScene(
|
|
||||||
input: SemanticVomInput & {
|
|
||||||
supplementalNames?: ReadonlyMap<string, string>;
|
|
||||||
identifierFallback?: boolean;
|
|
||||||
},
|
|
||||||
): VomScene {
|
|
||||||
return projectSemanticGraph(
|
|
||||||
normalizeSemanticStructure(
|
|
||||||
resolveSemanticGraph(buildSemanticGraph(input), {
|
|
||||||
supplementalNames: input.supplementalNames,
|
|
||||||
identifierFallback: input.identifierFallback,
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const;
|
|
||||||
const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]);
|
|
||||||
const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]);
|
|
||||||
|
|
||||||
function identifierTokens(value: string): string[] {
|
|
||||||
return value
|
|
||||||
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
||||||
.split(/[^a-zA-Z0-9]+/)
|
|
||||||
.map((token) => token.toLowerCase())
|
|
||||||
.filter(Boolean);
|
|
||||||
}
|
|
||||||
|
|
||||||
function readableIdentifier(value: string | undefined): string | undefined {
|
|
||||||
const raw = value?.trim();
|
|
||||||
if (!raw || raw.length < 3 || raw.length > 80) return undefined;
|
|
||||||
if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined;
|
|
||||||
|
|
||||||
const tokens = identifierTokens(raw);
|
|
||||||
if (GENERATED_PREFIXES.has(tokens[0])) return undefined;
|
|
||||||
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift();
|
|
||||||
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
|
|
||||||
if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined;
|
|
||||||
return tokens
|
|
||||||
.map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token))
|
|
||||||
.join(" ");
|
|
||||||
}
|
|
||||||
|
|
||||||
export function stableIdentifierName(attrs: Readonly<Record<string, string>>): string | undefined {
|
|
||||||
for (const name of IDENTIFIER_ATTRS) {
|
|
||||||
const label = readableIdentifier(attrs[name]);
|
|
||||||
if (label) return label;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
import type { VomNode, VomScene } from "@browser-skill/vom";
|
|
||||||
import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
|
|
||||||
|
|
||||||
function retained(node: StructuredSemanticNode): boolean {
|
|
||||||
return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
|
|
||||||
}
|
|
||||||
|
|
||||||
function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
|
|
||||||
const connected = new Set<string>([graph.rootFrameId]);
|
|
||||||
const childrenByParent = new Map<string, string[]>();
|
|
||||||
for (const frame of graph.frames.values()) {
|
|
||||||
if (!frame.parentFrameId) continue;
|
|
||||||
const children = childrenByParent.get(frame.parentFrameId) ?? [];
|
|
||||||
children.push(frame.frameId);
|
|
||||||
childrenByParent.set(frame.parentFrameId, children);
|
|
||||||
}
|
|
||||||
const queue = [graph.rootFrameId];
|
|
||||||
for (let index = 0; index < queue.length; index += 1) {
|
|
||||||
for (const frameId of childrenByParent.get(queue[index]) ?? []) {
|
|
||||||
if (connected.has(frameId)) continue;
|
|
||||||
const frame = graph.frames.get(frameId);
|
|
||||||
if (!frame?.ownerNodeId) continue;
|
|
||||||
const owner = graph.nodes.get(frame.ownerNodeId);
|
|
||||||
if (!owner || !retained(owner)) continue;
|
|
||||||
connected.add(frame.frameId);
|
|
||||||
queue.push(frame.frameId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return connected;
|
|
||||||
}
|
|
||||||
|
|
||||||
function domAncestors(
|
|
||||||
graph: StructuredSemanticGraph,
|
|
||||||
node: StructuredSemanticNode,
|
|
||||||
numericId: ReadonlyMap<SemanticNodeId, number>,
|
|
||||||
): number[] {
|
|
||||||
const ancestors: number[] = [];
|
|
||||||
const seen = new Set<SemanticNodeId>();
|
|
||||||
let current = node.domParentId;
|
|
||||||
while (current && !seen.has(current)) {
|
|
||||||
seen.add(current);
|
|
||||||
const id = numericId.get(current);
|
|
||||||
if (id !== undefined) ancestors.push(id);
|
|
||||||
current = graph.nodes.get(current)?.domParentId;
|
|
||||||
}
|
|
||||||
return ancestors;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
|
|
||||||
const connectedFrames = connectedFrameIds(graph);
|
|
||||||
const orderedNodes = [...graph.frames.values()]
|
|
||||||
.filter((frame) => connectedFrames.has(frame.frameId))
|
|
||||||
.sort((a, b) => a.order - b.order)
|
|
||||||
.flatMap((frame) =>
|
|
||||||
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
|
||||||
.map((id) => graph.nodes.get(id))
|
|
||||||
.filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
|
|
||||||
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
|
||||||
);
|
|
||||||
|
|
||||||
const numericId = new Map<SemanticNodeId, number>();
|
|
||||||
orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
|
|
||||||
const nodes: VomNode[] = orderedNodes.map((node) => ({
|
|
||||||
...node.vom,
|
|
||||||
id: numericId.get(node.id) as number,
|
|
||||||
parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
|
|
||||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
|
||||||
frameId: node.frameId,
|
|
||||||
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
|
||||||
referenceable: node.referenceable,
|
|
||||||
...(node.dom
|
|
||||||
? {
|
|
||||||
domParentId:
|
|
||||||
node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
|
|
||||||
domAncestorIds: domAncestors(graph, node, numericId),
|
|
||||||
}
|
|
||||||
: {}),
|
|
||||||
}));
|
|
||||||
|
|
||||||
return {
|
|
||||||
viewport: graph.viewport,
|
|
||||||
nodes,
|
|
||||||
rootFrameId: graph.rootFrameId,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,521 +0,0 @@
|
|||||||
import type { VomNode } from "@browser-skill/vom";
|
|
||||||
import { stableIdentifierName } from "./name-evidence";
|
|
||||||
import type {
|
|
||||||
ResolvedSemanticGraph,
|
|
||||||
ResolvedSemanticNode,
|
|
||||||
SemanticAxNode,
|
|
||||||
SemanticGraph,
|
|
||||||
SemanticGraphNode,
|
|
||||||
SemanticNodeId,
|
|
||||||
} from "./types";
|
|
||||||
import { frameBackendKey } from "./types";
|
|
||||||
|
|
||||||
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
|
||||||
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
|
|
||||||
const TEXT_AX_ROLES = new Set(["statictext", "inlinetextbox", "labeltext", "text"]);
|
|
||||||
const INTERACTIVE_ROLES = new Set([
|
|
||||||
"button",
|
|
||||||
"checkbox",
|
|
||||||
"combobox",
|
|
||||||
"link",
|
|
||||||
"listbox",
|
|
||||||
"menuitem",
|
|
||||||
"menuitemcheckbox",
|
|
||||||
"menuitemradio",
|
|
||||||
"option",
|
|
||||||
"radio",
|
|
||||||
"searchbox",
|
|
||||||
"slider",
|
|
||||||
"spinbutton",
|
|
||||||
"switch",
|
|
||||||
"tab",
|
|
||||||
"textbox",
|
|
||||||
"treeitem",
|
|
||||||
]);
|
|
||||||
const SENSITIVE_INPUT_TYPES = new Set([
|
|
||||||
"password",
|
|
||||||
"credit-card",
|
|
||||||
"one-time-code",
|
|
||||||
"current-password",
|
|
||||||
"new-password",
|
|
||||||
]);
|
|
||||||
|
|
||||||
function clean(value: string | undefined): string | undefined {
|
|
||||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
|
||||||
return normalized || undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function axValue(field?: { value?: string | number | boolean }): string | undefined {
|
|
||||||
return field?.value === undefined ? undefined : clean(String(field.value));
|
|
||||||
}
|
|
||||||
|
|
||||||
function axProperty(node: SemanticAxNode | undefined, name: string): string | undefined {
|
|
||||||
return axValue(node?.properties?.find((property) => property.name === name)?.value);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeRole(value: string | undefined): string | undefined {
|
|
||||||
const role = clean(value)?.toLowerCase();
|
|
||||||
return role || undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
|
|
||||||
if (tag === "button") return "button";
|
|
||||||
if (tag === "textarea") return "textbox";
|
|
||||||
if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
|
|
||||||
if (tag === "a" && attrs.href) return "link";
|
|
||||||
if (tag === "iframe") return "Iframe";
|
|
||||||
if (tag === "dialog") return "dialog";
|
|
||||||
if (tag === "img") return "img";
|
|
||||||
if (tag === "table") return "table";
|
|
||||||
if (tag === "tr") return "row";
|
|
||||||
if (tag === "th") return "columnheader";
|
|
||||||
if (tag === "td") return "cell";
|
|
||||||
if (tag === "ul" || tag === "ol") return "list";
|
|
||||||
if (tag === "li") return "listitem";
|
|
||||||
if (/^h[1-6]$/.test(tag)) return "heading";
|
|
||||||
if (tag === "nav") return "navigation";
|
|
||||||
if (tag === "main") return "main";
|
|
||||||
if (tag === "form") return "form";
|
|
||||||
if (tag === "input") {
|
|
||||||
const type = (attrs.type ?? "text").toLowerCase();
|
|
||||||
if (type === "hidden") return undefined;
|
|
||||||
if (type === "checkbox") return "checkbox";
|
|
||||||
if (type === "radio") return "radio";
|
|
||||||
if (type === "range") return "slider";
|
|
||||||
if (type === "number") return "spinbutton";
|
|
||||||
if (type === "search") return "searchbox";
|
|
||||||
if (["button", "submit", "reset", "image"].includes(type)) return "button";
|
|
||||||
return "textbox";
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function domRole(tag: string, attrs: Record<string, string>): string | undefined {
|
|
||||||
return normalizeRole(attrs.role) ?? nativeRole(tag, attrs);
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolvedRole(node: SemanticGraphNode): {
|
|
||||||
value?: string;
|
|
||||||
source: ResolvedSemanticNode["roleSource"];
|
|
||||||
} {
|
|
||||||
const rawAxRole = clean(axValue(node.ax?.role));
|
|
||||||
const axRole = normalizeRole(rawAxRole);
|
|
||||||
const attrs = node.dom?.attrs ?? {};
|
|
||||||
const explicitDomRole = normalizeRole(attrs.role);
|
|
||||||
const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs);
|
|
||||||
const usableAxRole = node.ax?.ignored !== true;
|
|
||||||
if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) {
|
|
||||||
return { value: rawAxRole, source: "ax" };
|
|
||||||
}
|
|
||||||
if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" };
|
|
||||||
if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" };
|
|
||||||
if (rawAxRole) {
|
|
||||||
return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" };
|
|
||||||
}
|
|
||||||
return { source: "none" };
|
|
||||||
}
|
|
||||||
|
|
||||||
function sensitive(node: SemanticGraphNode): boolean {
|
|
||||||
const attrs = node.dom?.attrs ?? {};
|
|
||||||
const type = (attrs.type ?? "").toLowerCase();
|
|
||||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
|
||||||
return (
|
|
||||||
type === "password" ||
|
|
||||||
autocomplete.startsWith("cc-") ||
|
|
||||||
SENSITIVE_INPUT_TYPES.has(autocomplete) ||
|
|
||||||
SENSITIVE_INPUT_TYPES.has(axProperty(node.ax, "inputType") ?? "")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function inputState(
|
|
||||||
node: SemanticGraphNode,
|
|
||||||
role: string | undefined,
|
|
||||||
value: string | undefined,
|
|
||||||
isSensitive: boolean,
|
|
||||||
): VomNode["inputState"] {
|
|
||||||
if (node.dom?.formState) return node.dom.formState;
|
|
||||||
const tag = node.dom?.tag.toLowerCase() ?? "";
|
|
||||||
if (!FORM_TAGS.has(tag) && !["textbox", "searchbox"].includes(role ?? "")) return undefined;
|
|
||||||
if (isSensitive) return value ? "filled" : "empty";
|
|
||||||
if (node.dom?.formValue !== undefined) {
|
|
||||||
if (node.dom.formValue === "") return "empty";
|
|
||||||
return node.dom.formValue === (node.dom.formDefaultValue ?? "") ? "default" : "filled";
|
|
||||||
}
|
|
||||||
return value ? "filled" : "empty";
|
|
||||||
}
|
|
||||||
|
|
||||||
function externalHrefHost(
|
|
||||||
href: string | undefined,
|
|
||||||
pageUrl: string | undefined,
|
|
||||||
): string | undefined {
|
|
||||||
if (!href || !pageUrl) return undefined;
|
|
||||||
try {
|
|
||||||
const page = new URL(pageUrl);
|
|
||||||
const target = new URL(href, page);
|
|
||||||
return target.origin === page.origin ? undefined : target.hostname;
|
|
||||||
} catch {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ResolveIndexes {
|
|
||||||
domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
|
||||||
axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
|
||||||
domIdIndex: Map<string, SemanticNodeId>;
|
|
||||||
labelForIndex: Map<string, SemanticNodeId[]>;
|
|
||||||
domText: (nodeId: SemanticNodeId) => string | undefined;
|
|
||||||
axText: (nodeId: SemanticNodeId) => string | undefined;
|
|
||||||
hasNativeDescendant: (nodeId: SemanticNodeId) => boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildIndexes(graph: SemanticGraph): ResolveIndexes {
|
|
||||||
const domChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
|
||||||
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
|
||||||
const domIdIndex = new Map<string, SemanticNodeId>();
|
|
||||||
const labelForIndex = new Map<string, SemanticNodeId[]>();
|
|
||||||
for (const node of graph.nodes.values()) {
|
|
||||||
if (node.domParentId) {
|
|
||||||
const children = domChildren.get(node.domParentId) ?? [];
|
|
||||||
children.push(node.id);
|
|
||||||
domChildren.set(node.domParentId, children);
|
|
||||||
}
|
|
||||||
if (node.axParentId) {
|
|
||||||
const children = axChildren.get(node.axParentId) ?? [];
|
|
||||||
children.push(node.id);
|
|
||||||
axChildren.set(node.axParentId, children);
|
|
||||||
}
|
|
||||||
const id = clean(node.dom?.attrs.id);
|
|
||||||
if (id) domIdIndex.set(`${node.frameId}\u0000${id}`, node.id);
|
|
||||||
const targetId = clean(node.dom?.attrs.for);
|
|
||||||
if (node.dom?.tag.toLowerCase() === "label" && targetId) {
|
|
||||||
const key = `${node.frameId}\u0000${targetId}`;
|
|
||||||
const labels = labelForIndex.get(key) ?? [];
|
|
||||||
labels.push(node.id);
|
|
||||||
labelForIndex.set(key, labels);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [parentId, children] of axChildren) {
|
|
||||||
const order = graph.nodes.get(parentId)?.ax?.childIds ?? [];
|
|
||||||
const orderByAxId = new Map(order.map((nodeId, index) => [nodeId, index]));
|
|
||||||
children.sort((a, b) => {
|
|
||||||
const aIndex = orderByAxId.get(graph.nodes.get(a)?.axNodeId ?? "") ?? -1;
|
|
||||||
const bIndex = orderByAxId.get(graph.nodes.get(b)?.axNodeId ?? "") ?? -1;
|
|
||||||
return (
|
|
||||||
(aIndex < 0 ? Number.MAX_SAFE_INTEGER : aIndex) -
|
|
||||||
(bIndex < 0 ? Number.MAX_SAFE_INTEGER : bIndex)
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const textMemo = new Map<SemanticNodeId, string | undefined>();
|
|
||||||
const resolvingText = new Set<SemanticNodeId>();
|
|
||||||
const domText = (nodeId: SemanticNodeId): string | undefined => {
|
|
||||||
if (textMemo.has(nodeId)) return textMemo.get(nodeId);
|
|
||||||
if (resolvingText.has(nodeId)) return undefined;
|
|
||||||
resolvingText.add(nodeId);
|
|
||||||
const node = graph.nodes.get(nodeId);
|
|
||||||
const parts: string[] = [];
|
|
||||||
const own = clean(node?.dom?.textContent);
|
|
||||||
if (own) parts.push(own);
|
|
||||||
for (const childId of domChildren.get(nodeId) ?? []) {
|
|
||||||
const child = graph.nodes.get(childId);
|
|
||||||
const childRole = child
|
|
||||||
? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
|
||||||
: undefined;
|
|
||||||
if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
|
|
||||||
const childText = domText(childId);
|
|
||||||
if (childText) parts.push(childText);
|
|
||||||
if (parts.join(" ").length >= 240) break;
|
|
||||||
}
|
|
||||||
resolvingText.delete(nodeId);
|
|
||||||
const result = clean(parts.join(" ").slice(0, 240));
|
|
||||||
textMemo.set(nodeId, result);
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
|
|
||||||
const axTextMemo = new Map<SemanticNodeId, string | undefined>();
|
|
||||||
const resolvingAxText = new Set<SemanticNodeId>();
|
|
||||||
const axText = (nodeId: SemanticNodeId): string | undefined => {
|
|
||||||
if (axTextMemo.has(nodeId)) return axTextMemo.get(nodeId);
|
|
||||||
if (resolvingAxText.has(nodeId)) return undefined;
|
|
||||||
resolvingAxText.add(nodeId);
|
|
||||||
const complete = (value: string | undefined): string | undefined => {
|
|
||||||
resolvingAxText.delete(nodeId);
|
|
||||||
axTextMemo.set(nodeId, value);
|
|
||||||
return value;
|
|
||||||
};
|
|
||||||
const node = graph.nodes.get(nodeId);
|
|
||||||
const role = normalizeRole(axValue(node?.ax?.role)) ?? "";
|
|
||||||
if (TEXT_AX_ROLES.has(role)) {
|
|
||||||
const own = clean(axValue(node?.ax?.name) ?? axValue(node?.ax?.value));
|
|
||||||
if (own) return complete(own);
|
|
||||||
}
|
|
||||||
const parts: string[] = [];
|
|
||||||
for (const childId of axChildren.get(nodeId) ?? []) {
|
|
||||||
const child = graph.nodes.get(childId);
|
|
||||||
const childRole = normalizeRole(axValue(child?.ax?.role)) ?? "";
|
|
||||||
if (INTERACTIVE_ROLES.has(childRole)) continue;
|
|
||||||
const childText = axText(childId);
|
|
||||||
if (childText) parts.push(childText);
|
|
||||||
if (parts.join(" ").length >= 240) break;
|
|
||||||
}
|
|
||||||
return complete(clean(parts.join(" ").slice(0, 240)));
|
|
||||||
};
|
|
||||||
|
|
||||||
const nativeMemo = new Map<SemanticNodeId, boolean>();
|
|
||||||
const hasNativeDescendant = (nodeId: SemanticNodeId): boolean => {
|
|
||||||
const cached = nativeMemo.get(nodeId);
|
|
||||||
if (cached !== undefined) return cached;
|
|
||||||
nativeMemo.set(nodeId, false);
|
|
||||||
for (const childId of domChildren.get(nodeId) ?? []) {
|
|
||||||
const child = graph.nodes.get(childId);
|
|
||||||
const childRole =
|
|
||||||
(child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ??
|
|
||||||
(child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined);
|
|
||||||
if (
|
|
||||||
(child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
|
|
||||||
INTERACTIVE_ROLES.has(childRole ?? "") ||
|
|
||||||
hasNativeDescendant(childId)
|
|
||||||
) {
|
|
||||||
nativeMemo.set(nodeId, true);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
return {
|
|
||||||
domChildren,
|
|
||||||
axChildren,
|
|
||||||
domIdIndex,
|
|
||||||
labelForIndex,
|
|
||||||
domText,
|
|
||||||
axText,
|
|
||||||
hasNativeDescendant,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function ariaLabelledText(node: SemanticGraphNode, indexes: ResolveIndexes): string | undefined {
|
|
||||||
const ids = (node.dom?.attrs["aria-labelledby"] ?? "").split(/\s+/).filter(Boolean);
|
|
||||||
const parts = ids.flatMap((id) => {
|
|
||||||
const target = indexes.domIdIndex.get(`${node.frameId}\u0000${id}`);
|
|
||||||
return target ? (indexes.domText(target) ?? []) : [];
|
|
||||||
});
|
|
||||||
return clean(parts.join(" "));
|
|
||||||
}
|
|
||||||
|
|
||||||
function nativeLabelText(
|
|
||||||
node: SemanticGraphNode,
|
|
||||||
graph: SemanticGraph,
|
|
||||||
indexes: ResolveIndexes,
|
|
||||||
): string | undefined {
|
|
||||||
const id = clean(node.dom?.attrs.id);
|
|
||||||
if (id) {
|
|
||||||
const labels = indexes.labelForIndex.get(`${node.frameId}\u0000${id}`) ?? [];
|
|
||||||
const explicit = clean(
|
|
||||||
labels
|
|
||||||
.map((labelId) => indexes.domText(labelId))
|
|
||||||
.filter(Boolean)
|
|
||||||
.join(" "),
|
|
||||||
);
|
|
||||||
if (explicit) return explicit;
|
|
||||||
}
|
|
||||||
let parentId = node.domParentId;
|
|
||||||
let guard = 0;
|
|
||||||
while (parentId && guard <= graph.nodes.size) {
|
|
||||||
const parent = graph.nodes.get(parentId);
|
|
||||||
if (!parent) break;
|
|
||||||
if (parent.dom?.tag.toLowerCase() === "label") return indexes.domText(parentId);
|
|
||||||
parentId = parent.domParentId;
|
|
||||||
guard += 1;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function precedingAxText(
|
|
||||||
node: SemanticGraphNode,
|
|
||||||
graph: SemanticGraph,
|
|
||||||
indexes: ResolveIndexes,
|
|
||||||
): string | undefined {
|
|
||||||
if (!node.axParentId) return undefined;
|
|
||||||
const siblings = indexes.axChildren.get(node.axParentId) ?? [];
|
|
||||||
const index = siblings.indexOf(node.id);
|
|
||||||
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
|
||||||
const sibling = graph.nodes.get(siblings[index - offset]);
|
|
||||||
const role = normalizeRole(axValue(sibling?.ax?.role)) ?? "";
|
|
||||||
if (INTERACTIVE_ROLES.has(role)) break;
|
|
||||||
if (!TEXT_AX_ROLES.has(role) && role !== "generic") continue;
|
|
||||||
const text = clean(axValue(sibling?.ax?.name) ?? axValue(sibling?.ax?.value))?.replace(
|
|
||||||
/[::]\s*$/,
|
|
||||||
"",
|
|
||||||
);
|
|
||||||
if (text && text.length <= 80) return text;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function precedingDomText(
|
|
||||||
node: SemanticGraphNode,
|
|
||||||
graph: SemanticGraph,
|
|
||||||
indexes: ResolveIndexes,
|
|
||||||
): string | undefined {
|
|
||||||
if (!node.domParentId) return undefined;
|
|
||||||
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
|
|
||||||
const index = siblings.indexOf(node.id);
|
|
||||||
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
|
||||||
const sibling = graph.nodes.get(siblings[index - offset]);
|
|
||||||
const siblingRole = sibling
|
|
||||||
? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
|
|
||||||
: undefined;
|
|
||||||
if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
|
|
||||||
const text = indexes.domText(siblings[index - offset])?.replace(/[::]\s*$/, "");
|
|
||||||
if (text && text.length <= 80) return text;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolvedName(
|
|
||||||
node: SemanticGraphNode,
|
|
||||||
role: string | undefined,
|
|
||||||
graph: SemanticGraph,
|
|
||||||
indexes: ResolveIndexes,
|
|
||||||
supplementalNames: ReadonlyMap<string, string>,
|
|
||||||
identifierFallback: boolean,
|
|
||||||
): string | undefined {
|
|
||||||
const attrs = node.dom?.attrs ?? {};
|
|
||||||
const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
|
||||||
const descendantText = indexes.domText(node.id);
|
|
||||||
const interactive =
|
|
||||||
INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
|
|
||||||
NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
|
||||||
const candidates = [
|
|
||||||
node.ax?.ignored === true ? undefined : axValue(node.ax?.name),
|
|
||||||
ariaLabelledText(node, indexes),
|
|
||||||
clean(attrs["aria-label"]),
|
|
||||||
nativeLabelText(node, graph, indexes),
|
|
||||||
clean(attrs.title),
|
|
||||||
clean(attrs.alt),
|
|
||||||
node.ax?.ignored === true ? undefined : indexes.axText(node.id),
|
|
||||||
descendantText,
|
|
||||||
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
|
||||||
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
|
||||||
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
|
||||||
formControl || !interactive || node.backendNodeId === undefined
|
|
||||||
? undefined
|
|
||||||
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
|
|
||||||
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
|
|
||||||
];
|
|
||||||
let name = candidates.find((candidate) => candidate !== undefined);
|
|
||||||
const hasPopup = axProperty(node.ax, "hasPopup");
|
|
||||||
if (name && hasPopup && hasPopup !== "false") {
|
|
||||||
name =
|
|
||||||
axProperty(node.ax, "expanded") === "true" ? `${name} [expanded]` : `${name} [has-submenu]`;
|
|
||||||
}
|
|
||||||
if (!name && role?.toLowerCase() === "iframe") return clean(attrs.id);
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
function insideNative(node: SemanticGraphNode, graph: SemanticGraph): boolean {
|
|
||||||
let parentId = node.domParentId;
|
|
||||||
let guard = 0;
|
|
||||||
while (parentId && guard <= graph.nodes.size) {
|
|
||||||
const parent = graph.nodes.get(parentId);
|
|
||||||
if (!parent) break;
|
|
||||||
if (parent.dom && NATIVE_CONTROL_TAGS.has(parent.dom.tag.toLowerCase())) return true;
|
|
||||||
parentId = parent.domParentId;
|
|
||||||
guard += 1;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function nearbyText(
|
|
||||||
node: SemanticGraphNode,
|
|
||||||
graph: SemanticGraph,
|
|
||||||
indexes: ResolveIndexes,
|
|
||||||
): string | undefined {
|
|
||||||
if (!node.domParentId) return undefined;
|
|
||||||
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
|
|
||||||
const index = siblings.indexOf(node.id);
|
|
||||||
if (index < 0) return undefined;
|
|
||||||
const texts = siblings
|
|
||||||
.slice(Math.max(0, index - 3), index + 4)
|
|
||||||
.filter((id) => id !== node.id)
|
|
||||||
.flatMap((id) => indexes.domText(id) ?? [])
|
|
||||||
.slice(0, 3);
|
|
||||||
return clean(texts.join(" "));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveSemanticGraph(
|
|
||||||
graph: SemanticGraph,
|
|
||||||
options: {
|
|
||||||
supplementalNames?: ReadonlyMap<string, string>;
|
|
||||||
identifierFallback?: boolean;
|
|
||||||
} = {},
|
|
||||||
): ResolvedSemanticGraph {
|
|
||||||
const indexes = buildIndexes(graph);
|
|
||||||
const supplementalNames = options.supplementalNames ?? new Map<string, string>();
|
|
||||||
const identifierFallback = options.identifierFallback ?? true;
|
|
||||||
const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
|
|
||||||
for (const node of graph.nodes.values()) {
|
|
||||||
const roleResolution = resolvedRole(node);
|
|
||||||
const role = roleResolution.value;
|
|
||||||
const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback);
|
|
||||||
const isSensitive = sensitive(node);
|
|
||||||
const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
|
|
||||||
const value = isSensitive ? undefined : sourceValue;
|
|
||||||
const attrs = { ...(node.dom?.attrs ?? {}) };
|
|
||||||
if (isSensitive) delete attrs.value;
|
|
||||||
const frame = graph.frames.get(node.frameId);
|
|
||||||
const tag = node.dom?.tag.toLowerCase() ?? "";
|
|
||||||
const vom: ResolvedSemanticNode["vom"] = {
|
|
||||||
tag,
|
|
||||||
rect: node.dom?.rect ?? null,
|
|
||||||
paintOrder: node.dom?.paintOrder ?? 0,
|
|
||||||
position: node.dom?.position || "static",
|
|
||||||
pointerEvents: node.dom?.pointerEvents || "auto",
|
|
||||||
...(node.dom?.cursor ? { cursor: node.dom.cursor } : {}),
|
|
||||||
...(role ? { role } : {}),
|
|
||||||
...(name ? { name } : {}),
|
|
||||||
...(value !== undefined ? { value } : {}),
|
|
||||||
...(clean(node.dom?.formPlaceholder ?? attrs.placeholder)
|
|
||||||
? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) }
|
|
||||||
: {}),
|
|
||||||
attrs,
|
|
||||||
...(clean(node.dom?.textContent) ? { text: clean(node.dom?.textContent) } : {}),
|
|
||||||
...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}),
|
|
||||||
inputState: inputState(node, role, sourceValue, isSensitive),
|
|
||||||
sensitive: isSensitive,
|
|
||||||
modal:
|
|
||||||
["dialog", "alertdialog"].includes(role?.toLowerCase() ?? "") ||
|
|
||||||
tag === "dialog" ||
|
|
||||||
(attrs["aria-modal"] ?? "").toLowerCase() === "true",
|
|
||||||
disabled:
|
|
||||||
Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
|
|
||||||
(attrs["aria-disabled"] ?? "").toLowerCase() === "true",
|
|
||||||
inert: Object.prototype.hasOwnProperty.call(attrs, "inert"),
|
|
||||||
hasNativeDescendant: indexes.hasNativeDescendant(node.id),
|
|
||||||
insideNative: insideNative(node, graph),
|
|
||||||
...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
|
|
||||||
};
|
|
||||||
const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected");
|
|
||||||
const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded");
|
|
||||||
const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls");
|
|
||||||
if (selected === "true" && !vom.attrs?.["aria-selected"]) {
|
|
||||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
|
|
||||||
}
|
|
||||||
if (expanded === "true" && !vom.attrs?.["aria-expanded"]) {
|
|
||||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-expanded": "true" };
|
|
||||||
}
|
|
||||||
if (controls && !vom.attrs?.["aria-controls"]) {
|
|
||||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-controls": controls };
|
|
||||||
}
|
|
||||||
nodes.set(node.id, {
|
|
||||||
...node,
|
|
||||||
vom,
|
|
||||||
referenceable: node.backendNodeId !== undefined,
|
|
||||||
roleSource: roleResolution.source,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return { ...graph, nodes };
|
|
||||||
}
|
|
||||||
@@ -1,456 +0,0 @@
|
|||||||
import {
|
|
||||||
applyVomInteractionRecovery,
|
|
||||||
isVomReferenceNode,
|
|
||||||
isVomStructuralRole,
|
|
||||||
type VomNode,
|
|
||||||
} from "@browser-skill/vom";
|
|
||||||
import type {
|
|
||||||
ResolvedSemanticGraph,
|
|
||||||
ResolvedSemanticNode,
|
|
||||||
SemanticNodeId,
|
|
||||||
StructureDecision,
|
|
||||||
StructuredSemanticGraph,
|
|
||||||
StructuredSemanticNode,
|
|
||||||
} from "./types";
|
|
||||||
|
|
||||||
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
|
|
||||||
const ROOT_ROLES = new Set(["rootwebarea", "webarea"]);
|
|
||||||
const CONTENT_BOUNDARY_ROLES = new Set([
|
|
||||||
"article",
|
|
||||||
"dialog",
|
|
||||||
"document",
|
|
||||||
"feed",
|
|
||||||
"form",
|
|
||||||
"grid",
|
|
||||||
"heading",
|
|
||||||
"iframe",
|
|
||||||
"list",
|
|
||||||
"main",
|
|
||||||
"navigation",
|
|
||||||
"paragraph",
|
|
||||||
"region",
|
|
||||||
"section",
|
|
||||||
"table",
|
|
||||||
"tree",
|
|
||||||
]);
|
|
||||||
|
|
||||||
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
|
|
||||||
const attrs = node.vom.attrs ?? {};
|
|
||||||
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
|
|
||||||
attrs[name]?.trim(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function explicitStructural(node: ResolvedSemanticNode): boolean {
|
|
||||||
return (
|
|
||||||
isVomStructuralRole(node.vom.role) &&
|
|
||||||
node.roleSource !== "none" &&
|
|
||||||
node.roleSource !== "ax-ignored"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean {
|
|
||||||
return node.ax !== undefined && node.ax.ignored !== true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isRenderedDomFallback(node: ResolvedSemanticNode): boolean {
|
|
||||||
if (!node.dom) return false;
|
|
||||||
const attrs = node.dom.attrs;
|
|
||||||
if (
|
|
||||||
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
|
||||||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
|
||||||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
|
||||||
) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null);
|
|
||||||
}
|
|
||||||
|
|
||||||
function initialParentId(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
node: ResolvedSemanticNode,
|
|
||||||
): SemanticNodeId | undefined {
|
|
||||||
let parentId = node.axParentId ?? node.domParentId;
|
|
||||||
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
|
|
||||||
const axParent = graph.nodes.get(node.axParentId);
|
|
||||||
const axRole = axParent?.vom.role?.toLowerCase() ?? "";
|
|
||||||
const meaningfulAxParent =
|
|
||||||
axParent !== undefined &&
|
|
||||||
!ROOT_ROLES.has(axRole) &&
|
|
||||||
(explicitStructural(axParent) ||
|
|
||||||
!TRANSPARENT_ROLES.has(axRole) ||
|
|
||||||
Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent)));
|
|
||||||
parentId = meaningfulAxParent ? node.axParentId : node.domParentId;
|
|
||||||
}
|
|
||||||
if (parentId) return parentId;
|
|
||||||
if (node.frameId === graph.rootFrameId) return undefined;
|
|
||||||
return graph.frames.get(node.frameId)?.ownerNodeId;
|
|
||||||
}
|
|
||||||
|
|
||||||
function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] {
|
|
||||||
return [...graph.frames.values()]
|
|
||||||
.sort((a, b) => a.order - b.order)
|
|
||||||
.flatMap((frame) =>
|
|
||||||
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
|
||||||
.map((id) => graph.nodes.get(id))
|
|
||||||
.filter((node): node is ResolvedSemanticNode => node !== undefined)
|
|
||||||
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function recoveredNodes(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
nodes: ResolvedSemanticNode[],
|
|
||||||
): Map<SemanticNodeId, VomNode> {
|
|
||||||
const numericId = new Map(nodes.map((node, index) => [node.id, index + 1]));
|
|
||||||
const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id]));
|
|
||||||
const provisional = nodes.map((node) => ({
|
|
||||||
...node.vom,
|
|
||||||
id: numericId.get(node.id) as number,
|
|
||||||
parentId: (() => {
|
|
||||||
const parent = initialParentId(graph, node);
|
|
||||||
return parent ? (numericId.get(parent) ?? null) : null;
|
|
||||||
})(),
|
|
||||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
|
||||||
frameId: node.frameId,
|
|
||||||
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
|
||||||
referenceable: node.referenceable,
|
|
||||||
}));
|
|
||||||
return new Map(
|
|
||||||
applyVomInteractionRecovery(provisional).flatMap((node) => {
|
|
||||||
const id = semanticId.get(node.id);
|
|
||||||
return id ? [[id, node] as const] : [];
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function redundantSourceNodes(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
nodes: ResolvedSemanticNode[],
|
|
||||||
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
|
|
||||||
): Set<SemanticNodeId> {
|
|
||||||
const byKey = new Map<string, ResolvedSemanticNode[]>();
|
|
||||||
for (const node of nodes) {
|
|
||||||
const output = recovered.get(node.id) as VomNode;
|
|
||||||
const name = output.name?.trim().toLowerCase() ?? "";
|
|
||||||
if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue;
|
|
||||||
const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`;
|
|
||||||
const matches = byKey.get(key) ?? [];
|
|
||||||
matches.push(node);
|
|
||||||
byKey.set(key, matches);
|
|
||||||
}
|
|
||||||
const redundant = new Set<SemanticNodeId>();
|
|
||||||
for (const matches of byKey.values()) {
|
|
||||||
const axOnly = matches.filter((node) => node.ax && !node.dom);
|
|
||||||
const domOnly = matches.filter((node) => node.dom && !node.ax);
|
|
||||||
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
|
|
||||||
redundant.add(domOnly[0].id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return redundant;
|
|
||||||
}
|
|
||||||
|
|
||||||
function baseDecision(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
node: ResolvedSemanticNode,
|
|
||||||
output: VomNode,
|
|
||||||
frameOwnerIds: ReadonlySet<SemanticNodeId>,
|
|
||||||
redundant: ReadonlySet<SemanticNodeId>,
|
|
||||||
): StructureDecision {
|
|
||||||
if (node.excluded) return { kind: "excluded", reason: "excluded" };
|
|
||||||
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
|
|
||||||
if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) {
|
|
||||||
return { kind: "excluded", reason: "non-rendered-dom-fallback" };
|
|
||||||
}
|
|
||||||
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
|
|
||||||
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
|
|
||||||
return { kind: "transparent", reason: "child-document-root" };
|
|
||||||
}
|
|
||||||
if (node.ax?.ignored === true && !node.dom) {
|
|
||||||
return { kind: "transparent", reason: "ax-ignored" };
|
|
||||||
}
|
|
||||||
const role = output.role?.toLowerCase() ?? "";
|
|
||||||
if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" };
|
|
||||||
if (
|
|
||||||
output.name ||
|
|
||||||
output.text ||
|
|
||||||
output.value ||
|
|
||||||
output.placeholder ||
|
|
||||||
hasMeaningfulRelations(node) ||
|
|
||||||
(output.rect && ["fixed", "sticky"].includes(output.position))
|
|
||||||
) {
|
|
||||||
return { kind: "keep", reason: "semantic" };
|
|
||||||
}
|
|
||||||
return { kind: "transparent", reason: "wrapper" };
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildChildren(
|
|
||||||
nodes: ResolvedSemanticNode[],
|
|
||||||
relation: "axParentId" | "domParentId",
|
|
||||||
): Map<SemanticNodeId, SemanticNodeId[]> {
|
|
||||||
const children = new Map<SemanticNodeId, SemanticNodeId[]>();
|
|
||||||
for (const node of nodes) {
|
|
||||||
const parentId = node[relation];
|
|
||||||
if (!parentId) continue;
|
|
||||||
const siblings = children.get(parentId) ?? [];
|
|
||||||
siblings.push(node.id);
|
|
||||||
children.set(parentId, siblings);
|
|
||||||
}
|
|
||||||
return children;
|
|
||||||
}
|
|
||||||
|
|
||||||
function explicitDomAncestors(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
nodes: ResolvedSemanticNode[],
|
|
||||||
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
|
||||||
): Set<SemanticNodeId> {
|
|
||||||
const inside = new Set<SemanticNodeId>();
|
|
||||||
const roots = nodes.filter((node) => !node.domParentId);
|
|
||||||
const queue = roots.map((node) => node.id);
|
|
||||||
for (let index = 0; index < queue.length; index += 1) {
|
|
||||||
const parentId = queue[index];
|
|
||||||
const parent = graph.nodes.get(parentId);
|
|
||||||
const parentIsBoundary =
|
|
||||||
parent !== undefined &&
|
|
||||||
explicitStructural(parent) &&
|
|
||||||
!ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? "");
|
|
||||||
for (const childId of domChildren.get(parentId) ?? []) {
|
|
||||||
if (inside.has(parentId) || parentIsBoundary) inside.add(childId);
|
|
||||||
queue.push(childId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return inside;
|
|
||||||
}
|
|
||||||
|
|
||||||
function reliableGroupCandidates(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
nodes: ResolvedSemanticNode[],
|
|
||||||
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
|
|
||||||
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
|
|
||||||
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
|
||||||
): Set<SemanticNodeId> {
|
|
||||||
const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren);
|
|
||||||
const remainingChildren = new Map(
|
|
||||||
nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]),
|
|
||||||
);
|
|
||||||
const summaries = new Map<SemanticNodeId, { blocked: boolean; interactions: number }>();
|
|
||||||
const queue = nodes
|
|
||||||
.filter((node) => (remainingChildren.get(node.id) ?? 0) === 0)
|
|
||||||
.map((node) => node.id);
|
|
||||||
for (let index = 0; index < queue.length; index += 1) {
|
|
||||||
const nodeId = queue[index];
|
|
||||||
const node = graph.nodes.get(nodeId);
|
|
||||||
const output = recovered.get(nodeId);
|
|
||||||
if (!node || !output) continue;
|
|
||||||
const role = output.role?.toLowerCase() ?? "";
|
|
||||||
const excluded = decisions.get(nodeId)?.kind === "excluded";
|
|
||||||
const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role);
|
|
||||||
const summary = excluded
|
|
||||||
? { blocked: true, interactions: 0 }
|
|
||||||
: isVomReferenceNode(output)
|
|
||||||
? { blocked: false, interactions: 1 }
|
|
||||||
: blocked
|
|
||||||
? { blocked: true, interactions: 0 }
|
|
||||||
: (domChildren.get(nodeId) ?? []).reduce(
|
|
||||||
(combined, childId) => {
|
|
||||||
const child = summaries.get(childId);
|
|
||||||
return {
|
|
||||||
blocked: combined.blocked || !child || child.blocked,
|
|
||||||
interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)),
|
|
||||||
};
|
|
||||||
},
|
|
||||||
{ blocked: false, interactions: 0 },
|
|
||||||
);
|
|
||||||
summaries.set(nodeId, summary);
|
|
||||||
const parentId = node.domParentId;
|
|
||||||
if (!parentId) continue;
|
|
||||||
const remaining = (remainingChildren.get(parentId) ?? 1) - 1;
|
|
||||||
remainingChildren.set(parentId, remaining);
|
|
||||||
if (remaining === 0) queue.push(parentId);
|
|
||||||
}
|
|
||||||
|
|
||||||
const candidates = new Set<SemanticNodeId>();
|
|
||||||
for (const node of nodes) {
|
|
||||||
const output = recovered.get(node.id) as VomNode;
|
|
||||||
const role = output.role?.toLowerCase() ?? "";
|
|
||||||
if (
|
|
||||||
!node.dom ||
|
|
||||||
node.dom.tag.toLowerCase() !== "div" ||
|
|
||||||
!TRANSPARENT_ROLES.has(role) ||
|
|
||||||
insideExplicitStructure.has(node.id) ||
|
|
||||||
decisions.get(node.id)?.kind === "excluded" ||
|
|
||||||
node.dom.attrs.hidden !== undefined ||
|
|
||||||
(node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" ||
|
|
||||||
node.dom.attrs.inert !== undefined
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id));
|
|
||||||
const blocked = branches.some((summary) => !summary || summary.blocked);
|
|
||||||
const contributingBranches = branches.filter(
|
|
||||||
(summary) => (summary?.interactions ?? 0) > 0,
|
|
||||||
).length;
|
|
||||||
const interactions = branches.reduce(
|
|
||||||
(count, summary) => Math.min(2, count + (summary?.interactions ?? 0)),
|
|
||||||
0,
|
|
||||||
);
|
|
||||||
if (!blocked && contributingBranches >= 2 && interactions >= 2) {
|
|
||||||
candidates.add(node.id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return candidates;
|
|
||||||
}
|
|
||||||
|
|
||||||
function leafCandidates(
|
|
||||||
nodes: ResolvedSemanticNode[],
|
|
||||||
candidates: ReadonlySet<SemanticNodeId>,
|
|
||||||
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
|
||||||
): Set<SemanticNodeId> {
|
|
||||||
const parentById = new Map(
|
|
||||||
nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])),
|
|
||||||
);
|
|
||||||
const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]));
|
|
||||||
const containsCandidate = new Set<SemanticNodeId>();
|
|
||||||
const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id);
|
|
||||||
const selected = new Set<SemanticNodeId>();
|
|
||||||
for (let index = 0; index < queue.length; index += 1) {
|
|
||||||
const nodeId = queue[index];
|
|
||||||
if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId);
|
|
||||||
const parentId = parentById.get(nodeId);
|
|
||||||
if (!parentId) continue;
|
|
||||||
if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId);
|
|
||||||
const remaining = (childCount.get(parentId) ?? 1) - 1;
|
|
||||||
childCount.set(parentId, remaining);
|
|
||||||
if (remaining === 0) queue.push(parentId);
|
|
||||||
}
|
|
||||||
return selected;
|
|
||||||
}
|
|
||||||
|
|
||||||
function nearestKept(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
startId: SemanticNodeId | undefined,
|
|
||||||
relation: "axParentId" | "domParentId",
|
|
||||||
kept: ReadonlySet<SemanticNodeId>,
|
|
||||||
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
|
||||||
): SemanticNodeId | undefined {
|
|
||||||
let currentId = startId;
|
|
||||||
const path: SemanticNodeId[] = [];
|
|
||||||
const seen = new Set<SemanticNodeId>();
|
|
||||||
let result: SemanticNodeId | undefined;
|
|
||||||
while (currentId && !seen.has(currentId)) {
|
|
||||||
if (kept.has(currentId)) {
|
|
||||||
result = currentId;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (memo.has(currentId)) {
|
|
||||||
result = memo.get(currentId);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
seen.add(currentId);
|
|
||||||
path.push(currentId);
|
|
||||||
currentId = graph.nodes.get(currentId)?.[relation];
|
|
||||||
}
|
|
||||||
for (const id of path) memo.set(id, result);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
function finalParentId(
|
|
||||||
graph: ResolvedSemanticGraph,
|
|
||||||
node: ResolvedSemanticNode,
|
|
||||||
kept: ReadonlySet<SemanticNodeId>,
|
|
||||||
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
|
|
||||||
axParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
|
||||||
domParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
|
||||||
): SemanticNodeId | undefined {
|
|
||||||
const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo);
|
|
||||||
const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo);
|
|
||||||
const axNode = axParent ? graph.nodes.get(axParent) : undefined;
|
|
||||||
const domNode = domParent ? graph.nodes.get(domParent) : undefined;
|
|
||||||
const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? "");
|
|
||||||
const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false;
|
|
||||||
const domIsStructural =
|
|
||||||
domNode !== undefined &&
|
|
||||||
(explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group");
|
|
||||||
if (axIsExplicit) return axParent;
|
|
||||||
if (domIsStructural) return domParent;
|
|
||||||
if (axParent && !axIsRoot) return axParent;
|
|
||||||
if (domParent) return domParent;
|
|
||||||
if (axParent) return axParent;
|
|
||||||
if (node.frameId === graph.rootFrameId) return undefined;
|
|
||||||
return graph.frames.get(node.frameId)?.ownerNodeId;
|
|
||||||
}
|
|
||||||
|
|
||||||
function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] {
|
|
||||||
const {
|
|
||||||
id: _id,
|
|
||||||
parentId: _parentId,
|
|
||||||
backendNodeId: _backendNodeId,
|
|
||||||
frameId: _frameId,
|
|
||||||
contextScopeId: _contextScopeId,
|
|
||||||
referenceable: _referenceable,
|
|
||||||
...vom
|
|
||||||
} = node;
|
|
||||||
if (!inferredGroup) return vom;
|
|
||||||
const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom;
|
|
||||||
return { ...groupVom, role: "group" };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
|
|
||||||
const nodes = orderedNodes(graph);
|
|
||||||
const recovered = recoveredNodes(graph, nodes);
|
|
||||||
const redundant = redundantSourceNodes(graph, nodes, recovered);
|
|
||||||
const frameOwnerIds = new Set(
|
|
||||||
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
|
|
||||||
);
|
|
||||||
const decisions = new Map<SemanticNodeId, StructureDecision>();
|
|
||||||
for (const node of nodes) {
|
|
||||||
decisions.set(
|
|
||||||
node.id,
|
|
||||||
baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const domChildren = buildChildren(nodes, "domParentId");
|
|
||||||
const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren);
|
|
||||||
for (const nodeId of leafCandidates(nodes, candidates, domChildren)) {
|
|
||||||
decisions.set(nodeId, {
|
|
||||||
kind: "inferred-group",
|
|
||||||
reason: "independent-interaction-branches",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const kept = new Set(
|
|
||||||
nodes
|
|
||||||
.filter((node) => {
|
|
||||||
const kind = decisions.get(node.id)?.kind;
|
|
||||||
return kind === "keep" || kind === "inferred-group";
|
|
||||||
})
|
|
||||||
.map((node) => node.id),
|
|
||||||
);
|
|
||||||
const axParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
|
||||||
const domParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
|
||||||
const structuredNodes = new Map<SemanticNodeId, StructuredSemanticNode>();
|
|
||||||
for (const node of nodes) {
|
|
||||||
const structure = decisions.get(node.id) as StructureDecision;
|
|
||||||
const inferredGroup = structure.kind === "inferred-group";
|
|
||||||
structuredNodes.set(node.id, {
|
|
||||||
...node,
|
|
||||||
vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup),
|
|
||||||
structure,
|
|
||||||
...(kept.has(node.id)
|
|
||||||
? {
|
|
||||||
semanticParentId: finalParentId(
|
|
||||||
graph,
|
|
||||||
node,
|
|
||||||
kept,
|
|
||||||
decisions,
|
|
||||||
axParentMemo,
|
|
||||||
domParentMemo,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
: {}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return { ...graph, nodes: structuredNodes };
|
|
||||||
}
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
import type { Viewport, VomNode } from "@browser-skill/vom";
|
|
||||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
|
||||||
import type { CapturedNode } from "../capture";
|
|
||||||
|
|
||||||
export interface SemanticAxNode {
|
|
||||||
nodeId: string;
|
|
||||||
frameId?: string;
|
|
||||||
parentId?: string;
|
|
||||||
backendDOMNodeId?: number;
|
|
||||||
ignored?: boolean;
|
|
||||||
role?: { type: string; value?: string };
|
|
||||||
name?: { type: string; value?: string };
|
|
||||||
description?: { value?: string };
|
|
||||||
value?: { value?: string | number | boolean };
|
|
||||||
properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
|
|
||||||
childIds?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export type SemanticNodeId = string;
|
|
||||||
|
|
||||||
export interface SemanticFrame {
|
|
||||||
frameId: string;
|
|
||||||
parentFrameId?: string;
|
|
||||||
ownerBackendNodeId?: number;
|
|
||||||
ownerNodeId?: SemanticNodeId;
|
|
||||||
contextScopeId: string;
|
|
||||||
target: CdpTarget;
|
|
||||||
url?: string;
|
|
||||||
order: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SemanticGraphNode {
|
|
||||||
id: SemanticNodeId;
|
|
||||||
frameId: string;
|
|
||||||
backendNodeId?: number;
|
|
||||||
axNodeId?: string;
|
|
||||||
ax?: SemanticAxNode;
|
|
||||||
dom?: CapturedNode;
|
|
||||||
axParentId?: SemanticNodeId;
|
|
||||||
domParentId?: SemanticNodeId;
|
|
||||||
sourceOrder: number;
|
|
||||||
excluded: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SemanticGraph {
|
|
||||||
viewport: Viewport;
|
|
||||||
rootFrameId: string;
|
|
||||||
frames: Map<string, SemanticFrame>;
|
|
||||||
nodes: Map<SemanticNodeId, SemanticGraphNode>;
|
|
||||||
nodeIdsByFrame: Map<string, SemanticNodeId[]>;
|
|
||||||
nodeByFrameBackend: Map<string, SemanticNodeId>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ResolvedSemanticNode extends SemanticGraphNode {
|
|
||||||
vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
|
|
||||||
referenceable: boolean;
|
|
||||||
roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none";
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
|
|
||||||
nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type StructureDecision =
|
|
||||||
| { kind: "keep"; reason: "semantic" | "frame-owner" }
|
|
||||||
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
|
|
||||||
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
|
|
||||||
| {
|
|
||||||
kind: "excluded";
|
|
||||||
reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback";
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface StructuredSemanticNode extends ResolvedSemanticNode {
|
|
||||||
structure: StructureDecision;
|
|
||||||
semanticParentId?: SemanticNodeId;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface StructuredSemanticGraph extends Omit<ResolvedSemanticGraph, "nodes"> {
|
|
||||||
nodes: Map<SemanticNodeId, StructuredSemanticNode>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function frameBackendKey(frameId: string, backendNodeId: number): string {
|
|
||||||
return `${frameId}\u0000${backendNodeId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function frameAxKey(frameId: string, axNodeId: string): string {
|
|
||||||
return `${frameId}\u0000${axNodeId}`;
|
|
||||||
}
|
|
||||||
@@ -23,12 +23,9 @@ export type ErrorCode =
|
|||||||
export type RpcErrorReason =
|
export type RpcErrorReason =
|
||||||
| "agent_window_scope"
|
| "agent_window_scope"
|
||||||
| "element_not_visible"
|
| "element_not_visible"
|
||||||
| "ref_capability_denied"
|
|
||||||
| "ref_not_found"
|
| "ref_not_found"
|
||||||
| "selector_not_found"
|
| "selector_not_found"
|
||||||
| "target_not_fillable"
|
| "target_not_fillable"
|
||||||
| "focus_target_unresolved"
|
|
||||||
| "input_not_applied"
|
|
||||||
| "target_not_select"
|
| "target_not_select"
|
||||||
| "option_not_found"
|
| "option_not_found"
|
||||||
| "single_select_value_count"
|
| "single_select_value_count"
|
||||||
@@ -36,11 +33,6 @@ export type RpcErrorReason =
|
|||||||
| "restricted_tab_url"
|
| "restricted_tab_url"
|
||||||
| "borrow_conflict"
|
| "borrow_conflict"
|
||||||
| "screenshot_capture_failed"
|
| "screenshot_capture_failed"
|
||||||
| "surface_capture_not_found"
|
|
||||||
| "surface_capture_expired"
|
|
||||||
| "surface_capture_consumed"
|
|
||||||
| "surface_capture_stale"
|
|
||||||
| "surface_coordinate_invalid"
|
|
||||||
| "cleanup_failed";
|
| "cleanup_failed";
|
||||||
|
|
||||||
export interface RpcErrorData {
|
export interface RpcErrorData {
|
||||||
@@ -306,17 +298,9 @@ export interface ScreenshotResult {
|
|||||||
height: number;
|
height: number;
|
||||||
format: string;
|
format: string;
|
||||||
tab_id: number;
|
tab_id: number;
|
||||||
capture?: SurfaceCaptureInfo;
|
|
||||||
dialogs?: JavaScriptDialogInfo[];
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SurfaceCaptureInfo {
|
|
||||||
id: string;
|
|
||||||
surface_ref: string;
|
|
||||||
coordinate_space: "capture-image-pixel";
|
|
||||||
expires_at: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SnapshotParams {
|
export interface SnapshotParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
tab_id?: number;
|
tab_id?: number;
|
||||||
@@ -430,9 +414,6 @@ export interface ClickParams {
|
|||||||
click_count?: number;
|
click_count?: number;
|
||||||
modifiers?: KeyModifier[];
|
modifiers?: KeyModifier[];
|
||||||
timeout_ms?: number;
|
timeout_ms?: number;
|
||||||
capture_id?: string;
|
|
||||||
image_x?: number;
|
|
||||||
image_y?: number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClickResult {
|
export interface ClickResult {
|
||||||
@@ -441,9 +422,6 @@ export interface ClickResult {
|
|||||||
used_selector?: string;
|
used_selector?: string;
|
||||||
x: number;
|
x: number;
|
||||||
y: number;
|
y: number;
|
||||||
capture_id?: string;
|
|
||||||
image_x?: number;
|
|
||||||
image_y?: number;
|
|
||||||
dialogs?: JavaScriptDialogInfo[];
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,136 +0,0 @@
|
|||||||
# Rendered Surface browser fixtures
|
|
||||||
|
|
||||||
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
|
|
||||||
|
|
||||||
## Serve
|
|
||||||
|
|
||||||
From the repository root:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python3 -m http.server 4173 --bind 127.0.0.1 \
|
|
||||||
--directory apps/extension/test-fixtures/rendered-surfaces
|
|
||||||
```
|
|
||||||
|
|
||||||
Use an Agent Window large enough to keep the primary matrix visible:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk session start --width 1440 --height 1100
|
|
||||||
bsk navigate http://127.0.0.1:4173/index.html --session <id>
|
|
||||||
bsk observe --session <id>
|
|
||||||
```
|
|
||||||
|
|
||||||
Always stop the session after the run.
|
|
||||||
|
|
||||||
## Pages
|
|
||||||
|
|
||||||
### `index.html`
|
|
||||||
|
|
||||||
Expected individual Surface labels:
|
|
||||||
|
|
||||||
- `fixture-basic-large`
|
|
||||||
- `fixture-tiny-8px`
|
|
||||||
- `fixture-pointer-events-none`
|
|
||||||
- `fixture-partial-ax`
|
|
||||||
- `fixture-contained-outer`
|
|
||||||
- `fixture-contained-inner`
|
|
||||||
- `fixture-different-parent-a`
|
|
||||||
- `fixture-different-parent-b`
|
|
||||||
|
|
||||||
Expected one grouped Surface:
|
|
||||||
|
|
||||||
- `fixture-layered-group`, with `layers=2`
|
|
||||||
|
|
||||||
Expected absent:
|
|
||||||
|
|
||||||
- `fixture-display-none`
|
|
||||||
- `fixture-visibility-hidden`
|
|
||||||
- `fixture-opacity-zero`
|
|
||||||
- `fixture-parent-opacity-zero`
|
|
||||||
- `fixture-hidden-attribute`
|
|
||||||
- `fixture-inert-parent`
|
|
||||||
- `fixture-aria-hidden-parent`
|
|
||||||
- `fixture-zero-size`
|
|
||||||
- `fixture-offscreen-right`
|
|
||||||
- `fixture-fully-overflow-clipped`
|
|
||||||
|
|
||||||
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
|
|
||||||
|
|
||||||
### `unlabeled.html`
|
|
||||||
|
|
||||||
This page contains eight Canvas elements but must render seven generic Surface markers because
|
|
||||||
the two coincident layers form one group.
|
|
||||||
|
|
||||||
- Every marker uses the stable fallback label `canvas visual surface`.
|
|
||||||
- Exactly one marker has `layers=2`.
|
|
||||||
- The two contained canvases remain separate markers.
|
|
||||||
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
|
|
||||||
- Blank `aria-label` and `title` values behave as no label.
|
|
||||||
- Fallback AX content does not suppress the corresponding Surface.
|
|
||||||
|
|
||||||
### `frames.html`
|
|
||||||
|
|
||||||
Expected Surface labels:
|
|
||||||
|
|
||||||
- `fixture-same-origin-frame`
|
|
||||||
- `fixture-cross-origin-frame`
|
|
||||||
- `fixture-nested-frame`
|
|
||||||
|
|
||||||
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
|
|
||||||
|
|
||||||
Each frame Canvas also exposes a `pointer status` live region. A successful
|
|
||||||
screenshot-bound point click must report
|
|
||||||
`down button=0 buttons=1 hoverReady=true; click received` and turn the Canvas
|
|
||||||
yellow. The hover flag is set on the animation frame after `pointermove`, so the
|
|
||||||
fixture verifies both child-document delivery and the scheduling boundary
|
|
||||||
required before `pointerdown`, rather than merely validating projected
|
|
||||||
coordinates or a successful CDP response.
|
|
||||||
|
|
||||||
### `viewport.html`
|
|
||||||
|
|
||||||
Run with a `900x700` Agent Window.
|
|
||||||
|
|
||||||
- `fixture-partial-viewport` is partly visible and must be represented.
|
|
||||||
- `fixture-below-fold` is absent initially.
|
|
||||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
|
||||||
|
|
||||||
### `input-frames.html`
|
|
||||||
|
|
||||||
This page exercises generic frame-aware input routing. It contains a top-level
|
|
||||||
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
|
||||||
frame includes an input, textarea, contenteditable editor, and a controlled
|
|
||||||
input that resets its value on the next animation frame.
|
|
||||||
|
|
||||||
- `fill` must route focus, text insertion, DOM events, and readback through the
|
|
||||||
ref's owning CDP target.
|
|
||||||
- `press --ref` must focus and dispatch every key event through that same target.
|
|
||||||
- Untargeted `press` must follow the actual focused target or refuse when focus
|
|
||||||
ownership cannot be determined uniquely.
|
|
||||||
- The rejecting input must return `input_not_applied`, never a successful
|
|
||||||
requested-value length.
|
|
||||||
|
|
||||||
### `canvas-input.html`
|
|
||||||
|
|
||||||
This is the PR3 end-to-end fixture. Its cross-origin frame contains a
|
|
||||||
visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
|
|
||||||
focuses a standard textbox; ordinary `fill` must then write and read back its
|
|
||||||
real value through the textbox's owning target. The fixture deliberately does
|
|
||||||
not provide a Canvas-specific fill action.
|
|
||||||
|
|
||||||
### `dynamic.html`
|
|
||||||
|
|
||||||
Use the normal DOM buttons and re-observe after each action:
|
|
||||||
|
|
||||||
- Initial state: `fixture-dynamic` absent.
|
|
||||||
- After **Insert canvas**: present.
|
|
||||||
- After **Resize to zero**: absent.
|
|
||||||
- After **Restore size**: present with a fresh ref.
|
|
||||||
- After **Remove canvas**: absent.
|
|
||||||
|
|
||||||
## Invariants
|
|
||||||
|
|
||||||
- `snapshot` must not emit any of these Surface markers.
|
|
||||||
- `observe` must emit only currently visible Canvas surfaces.
|
|
||||||
- Existing DOM/AX controls remain readable alongside Surface markers.
|
|
||||||
- Missing labels never prevent discovery; naming and discovery are independent.
|
|
||||||
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
|
|
||||||
- No fixture URL, id, label, or layout constant may appear in production discovery code.
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Canvas to standard input fixture</title>
|
|
||||||
<style>
|
|
||||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
|
||||||
iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Canvas to standard input fixture</h1>
|
|
||||||
<p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
|
|
||||||
<iframe
|
|
||||||
title="cross-origin Canvas editor"
|
|
||||||
src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
|
|
||||||
></iframe>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Rendered Surface dynamic matrix</title>
|
|
||||||
<style>
|
|
||||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
|
||||||
body { padding: 16px; }
|
|
||||||
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
|
|
||||||
button { padding: 8px 12px; }
|
|
||||||
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Dynamic Canvas lifecycle</h1>
|
|
||||||
<div class="controls">
|
|
||||||
<button id="insert" type="button">Insert canvas</button>
|
|
||||||
<button id="zero" type="button">Resize to zero</button>
|
|
||||||
<button id="restore" type="button">Restore size</button>
|
|
||||||
<button id="remove" type="button">Remove canvas</button>
|
|
||||||
</div>
|
|
||||||
<div id="mount"></div>
|
|
||||||
<script>
|
|
||||||
const mount = document.querySelector("#mount");
|
|
||||||
const getCanvas = () => mount.querySelector("canvas");
|
|
||||||
document.querySelector("#insert").addEventListener("click", () => {
|
|
||||||
if (getCanvas()) return;
|
|
||||||
const canvas = document.createElement("canvas");
|
|
||||||
canvas.width = 260;
|
|
||||||
canvas.height = 120;
|
|
||||||
canvas.setAttribute("aria-label", "fixture-dynamic");
|
|
||||||
mount.append(canvas);
|
|
||||||
});
|
|
||||||
document.querySelector("#zero").addEventListener("click", () => {
|
|
||||||
const canvas = getCanvas();
|
|
||||||
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
|
|
||||||
});
|
|
||||||
document.querySelector("#restore").addEventListener("click", () => {
|
|
||||||
const canvas = getCanvas();
|
|
||||||
if (canvas) canvas.removeAttribute("style");
|
|
||||||
});
|
|
||||||
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<style>
|
|
||||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
|
||||||
body { padding: 12px; background: #fdfefe; }
|
|
||||||
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
|
|
||||||
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
|
|
||||||
#pointer-status { margin: 8px 0 0; min-height: 20px; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<canvas id="frame-canvas" width="220" height="90"></canvas>
|
|
||||||
<script>
|
|
||||||
const params = new URLSearchParams(location.search);
|
|
||||||
const canvas = document.querySelector("#frame-canvas");
|
|
||||||
if (params.get("canvas") === "0") canvas.remove();
|
|
||||||
else {
|
|
||||||
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
|
||||||
if (params.get("editor") === "1") {
|
|
||||||
const context = canvas.getContext("2d");
|
|
||||||
context.font = "14px system-ui, sans-serif";
|
|
||||||
context.textBaseline = "middle";
|
|
||||||
context.strokeStyle = "#1e8449";
|
|
||||||
context.fillStyle = "#145a32";
|
|
||||||
for (let row = 0; row < 3; row += 1) {
|
|
||||||
const top = row * 30;
|
|
||||||
context.strokeRect(0.5, top + 0.5, 219, 29);
|
|
||||||
context.fillText(`Row ${row + 1}`, 12, top + 15);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (params.get("interactive") === "1") {
|
|
||||||
const status = document.createElement("p");
|
|
||||||
status.id = "pointer-status";
|
|
||||||
status.setAttribute("aria-live", "polite");
|
|
||||||
status.textContent = "pointer status: idle";
|
|
||||||
canvas.insertAdjacentElement("afterend", status);
|
|
||||||
let hoverReady = false;
|
|
||||||
canvas.addEventListener("pointermove", () => {
|
|
||||||
hoverReady = false;
|
|
||||||
requestAnimationFrame(() => {
|
|
||||||
hoverReady = true;
|
|
||||||
});
|
|
||||||
});
|
|
||||||
canvas.addEventListener("pointerdown", (event) => {
|
|
||||||
status.textContent = `pointer status: down button=${event.button} buttons=${event.buttons} hoverReady=${hoverReady}`;
|
|
||||||
if (event.buttons === 1 && hoverReady) canvas.style.background = "#f9e79f";
|
|
||||||
});
|
|
||||||
canvas.addEventListener("click", () => {
|
|
||||||
status.textContent += "; click received";
|
|
||||||
});
|
|
||||||
if (params.get("editor") === "1") {
|
|
||||||
canvas.addEventListener("click", (event) => {
|
|
||||||
const rect = canvas.getBoundingClientRect();
|
|
||||||
const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
|
|
||||||
const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
|
|
||||||
status.textContent += `; selected row=${row + 1}`;
|
|
||||||
document.querySelector("#canvas-cell-editor")?.remove();
|
|
||||||
if (row !== 2) return;
|
|
||||||
const editor = document.createElement("input");
|
|
||||||
editor.id = "canvas-cell-editor";
|
|
||||||
editor.setAttribute("aria-label", "selected canvas cell editor");
|
|
||||||
editor.placeholder = "Type into selected row 3";
|
|
||||||
canvas.insertAdjacentElement("afterend", editor);
|
|
||||||
editor.focus();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (params.get("nested") === "1") {
|
|
||||||
const iframe = document.createElement("iframe");
|
|
||||||
iframe.title = "nested fixture frame";
|
|
||||||
iframe.src = "frame.html?label=fixture-nested-frame&interactive=1";
|
|
||||||
document.body.append(iframe);
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Rendered Surface frame matrix</title>
|
|
||||||
<style>
|
|
||||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
|
||||||
body { padding: 16px; }
|
|
||||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
|
||||||
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Frame matrix</h1>
|
|
||||||
<div class="frames">
|
|
||||||
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame&interactive=1"></iframe>
|
|
||||||
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame&interactive=1"></iframe>
|
|
||||||
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Rendered Surface discovery matrix</title>
|
|
||||||
<style>
|
|
||||||
* { box-sizing: border-box; }
|
|
||||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
|
||||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
|
||||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
|
||||||
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
|
|
||||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
|
||||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
|
||||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
|
||||||
.stack { position: relative; width: 180px; height: 90px; }
|
|
||||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
|
||||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
|
||||||
.contained { position: relative; width: 190px; height: 100px; }
|
|
||||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
|
||||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
|
||||||
.same-place { position: relative; width: 180px; height: 90px; }
|
|
||||||
.same-place > div { position: absolute; inset: 0; }
|
|
||||||
.same-place canvas { width: 180px; height: 90px; }
|
|
||||||
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
|
|
||||||
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
|
|
||||||
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
|
|
||||||
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
|
|
||||||
.tiny { width: 8px; height: 8px; border-width: 1px; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Rendered Surface discovery matrix</h1>
|
|
||||||
<main class="matrix">
|
|
||||||
<section class="case">
|
|
||||||
<h2>Visible named canvas</h2>
|
|
||||||
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Tiny canvas remains addressable</h2>
|
|
||||||
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Visible but pointer-events none</h2>
|
|
||||||
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Partial AX/fallback content</h2>
|
|
||||||
<canvas aria-label="fixture-partial-ax" width="180" height="80">
|
|
||||||
<button type="button">Fallback button must not suppress the surface</button>
|
|
||||||
</canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Two real rendering layers</h2>
|
|
||||||
<div class="stack">
|
|
||||||
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
|
|
||||||
<canvas width="180" height="90"></canvas>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Contained canvases are independent</h2>
|
|
||||||
<div class="contained">
|
|
||||||
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
|
|
||||||
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Same geometry, different parents</h2>
|
|
||||||
<div class="same-place">
|
|
||||||
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
|
|
||||||
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Partially overflow-clipped</h2>
|
|
||||||
<div class="clip-box">
|
|
||||||
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Fully overflow-clipped</h2>
|
|
||||||
<div class="clip-box">
|
|
||||||
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>CSS-hidden variants</h2>
|
|
||||||
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
|
|
||||||
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
|
|
||||||
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Hidden ancestor variants</h2>
|
|
||||||
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
|
|
||||||
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
|
|
||||||
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
|
|
||||||
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Zero-size and offscreen</h2>
|
|
||||||
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
|
|
||||||
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
|
|
||||||
</section>
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<style>
|
|
||||||
body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
|
|
||||||
label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
|
|
||||||
input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
|
|
||||||
iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<label>Frame input <input aria-label="frame input" /></label>
|
|
||||||
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
|
||||||
<div contenteditable="true" aria-label="frame editor"></div>
|
|
||||||
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
|
||||||
<script>
|
|
||||||
const params = new URLSearchParams(location.search);
|
|
||||||
const rejecting = document.querySelector("#rejecting");
|
|
||||||
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
|
||||||
if (params.get("nested") === "1") {
|
|
||||||
const iframe = document.createElement("iframe");
|
|
||||||
iframe.title = "nested input frame";
|
|
||||||
iframe.src = "input-frame.html";
|
|
||||||
document.body.append(iframe);
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Frame input routing matrix</title>
|
|
||||||
<style>
|
|
||||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
|
||||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
|
||||||
iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
|
|
||||||
input { padding: 6px; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Frame input routing matrix</h1>
|
|
||||||
<label>Top input <input aria-label="top input" /></label>
|
|
||||||
<div class="frames">
|
|
||||||
<iframe title="same-origin inputs" src="input-frame.html"></iframe>
|
|
||||||
<iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
|
|
||||||
<iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Unlabeled Rendered Surface matrix</title>
|
|
||||||
<style>
|
|
||||||
* { box-sizing: border-box; }
|
|
||||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
|
||||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
|
||||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
|
||||||
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
|
|
||||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
|
||||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
|
||||||
.case p { margin: 0 0 8px; }
|
|
||||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
|
||||||
.stack { position: relative; width: 180px; height: 90px; }
|
|
||||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
|
||||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
|
||||||
.contained { position: relative; width: 190px; height: 100px; }
|
|
||||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
|
||||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Unlabeled Rendered Surface matrix</h1>
|
|
||||||
<main class="matrix">
|
|
||||||
<section class="case">
|
|
||||||
<h2>Only engineering attributes</h2>
|
|
||||||
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Blank explicit names</h2>
|
|
||||||
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Nearby prose is not a Canvas name</h2>
|
|
||||||
<p>must-not-be-inferred-as-a-surface-label</p>
|
|
||||||
<canvas width="180" height="80"></canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Fallback AX content does not name or suppress the Canvas</h2>
|
|
||||||
<canvas width="180" height="80">
|
|
||||||
<button type="button">Fallback control remains separate</button>
|
|
||||||
</canvas>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Unnamed rendering layers still cluster</h2>
|
|
||||||
<div class="stack">
|
|
||||||
<canvas width="180" height="90"></canvas>
|
|
||||||
<canvas width="180" height="90"></canvas>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="case">
|
|
||||||
<h2>Unnamed contained canvases stay independent</h2>
|
|
||||||
<div class="contained">
|
|
||||||
<canvas width="190" height="100"></canvas>
|
|
||||||
<canvas width="70" height="40"></canvas>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Rendered Surface viewport matrix</title>
|
|
||||||
<style>
|
|
||||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
|
||||||
body { min-height: 1800px; padding: 16px; }
|
|
||||||
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
|
|
||||||
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
|
|
||||||
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Viewport matrix</h1>
|
|
||||||
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
|
|
||||||
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
|
|
||||||
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
||||||
@@ -29,7 +29,7 @@ Drive the user's **real Chromium browser** (with their logins and cookies) throu
|
|||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||||
|
|
||||||
## Mandatory workflow
|
## Mandatory workflow
|
||||||
@@ -80,16 +80,6 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
|||||||
|
|
||||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||||
|
|
||||||
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
|
||||||
|
|
||||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
|
||||||
```
|
|
||||||
|
|
||||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||||
@@ -184,7 +174,7 @@ bsk emulate --session <id> --off
|
|||||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
|
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||||
|
|
||||||
### Console & network debugging (read-only; require `--session`)
|
### Console & network debugging (read-only; require `--session`)
|
||||||
|
|
||||||
@@ -210,7 +200,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
|||||||
|
|
||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||||
@@ -309,7 +299,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
|||||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||||
|
|
||||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||||
|
|
||||||
|
|||||||
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `bsk doctor` check: every connected browser should use the same
|
/// `bsk doctor` check: connected browsers should speak a protocol the
|
||||||
/// protocol version as the daemon. Minor protocol drift is accepted by
|
/// daemon accepts. A different protocol string is still a live
|
||||||
/// the daemon but flagged here so the user can see who needs updating
|
/// connection — report Ok so agents keep working. The detail is an
|
||||||
/// (M10.4).
|
/// upgrade reminder, not a blocker.
|
||||||
///
|
///
|
||||||
/// Review M2 (round-1 minor): when no browsers are connected, the
|
/// Review M2 (round-1 minor): when no browsers are connected, the
|
||||||
/// check has nothing to compare against, so it now reports
|
/// check has nothing to compare against, so it now reports
|
||||||
@@ -396,13 +396,12 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(", ");
|
.join(", ");
|
||||||
CheckResult::fail(
|
CheckResult::ok(
|
||||||
name,
|
name,
|
||||||
format!(
|
format!(
|
||||||
"{} browser(s) have protocol minor drift from the daemon: {stale}",
|
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
|
||||||
status.version_skew_browsers.len()
|
status.version_skew_browsers.len()
|
||||||
),
|
),
|
||||||
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -581,7 +580,7 @@ mod m2_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn browsers_check_reports_fail_when_skew_present() {
|
fn browsers_check_reports_ok_when_skew_present() {
|
||||||
let status = fake_status(
|
let status = fake_status(
|
||||||
vec![BrowserStatusEntry {
|
vec![BrowserStatusEntry {
|
||||||
instance_id: "alpha".into(),
|
instance_id: "alpha".into(),
|
||||||
@@ -605,9 +604,17 @@ mod m2_tests {
|
|||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
assert_eq!(check.status, CheckStatus::Fail);
|
assert_eq!(check.status, CheckStatus::Ok);
|
||||||
assert!(check.detail.contains("alpha"));
|
assert!(check.detail.contains("alpha"));
|
||||||
assert!(check.hint.is_some());
|
assert!(
|
||||||
|
check
|
||||||
|
.detail
|
||||||
|
.contains("still usable — continue, and upgrade soon")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
check.hint.is_none(),
|
||||||
|
"a protocol-version note must not hint to stop"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -635,7 +642,7 @@ mod m2_tests {
|
|||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
assert_eq!(check.status, CheckStatus::Fail);
|
assert_eq!(check.status, CheckStatus::Ok);
|
||||||
assert!(
|
assert!(
|
||||||
check
|
check
|
||||||
.detail
|
.detail
|
||||||
|
|||||||
@@ -136,18 +136,6 @@ pub struct ClickArgs {
|
|||||||
|
|
||||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||||
pub timeout: u32,
|
pub timeout: u32,
|
||||||
|
|
||||||
/// Capture id returned by `bsk screenshot --ref <surface>`.
|
|
||||||
#[arg(long)]
|
|
||||||
pub capture: Option<String>,
|
|
||||||
|
|
||||||
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
|
|
||||||
#[arg(long = "image-x", allow_hyphen_values = true)]
|
|
||||||
pub image_x: Option<f64>,
|
|
||||||
|
|
||||||
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
|
|
||||||
#[arg(long = "image-y", allow_hyphen_values = true)]
|
|
||||||
pub image_y: Option<f64>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||||
@@ -159,14 +147,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
|||||||
)?;
|
)?;
|
||||||
let modifiers = parse_modifiers(&args.modifiers)
|
let modifiers = parse_modifiers(&args.modifiers)
|
||||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||||
let point_arg_count = usize::from(args.capture.is_some())
|
|
||||||
+ usize::from(args.image_x.is_some())
|
|
||||||
+ usize::from(args.image_y.is_some());
|
|
||||||
if point_arg_count != 0 && point_arg_count != 3 {
|
|
||||||
return Err(CliError::Local(anyhow::anyhow!(
|
|
||||||
"--capture, --image-x, and --image-y must be given together"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let params = ClickParams {
|
let params = ClickParams {
|
||||||
session_id: args.session.clone(),
|
session_id: args.session.clone(),
|
||||||
ref_,
|
ref_,
|
||||||
@@ -180,9 +160,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
|||||||
Some(modifiers)
|
Some(modifiers)
|
||||||
},
|
},
|
||||||
timeout_ms: Some(args.timeout),
|
timeout_ms: Some(args.timeout),
|
||||||
capture_id: args.capture.clone(),
|
|
||||||
image_x: args.image_x,
|
|
||||||
image_y: args.image_y,
|
|
||||||
};
|
};
|
||||||
let reply: ClickResult = call(
|
let reply: ClickResult = call(
|
||||||
info.sock_path,
|
info.sock_path,
|
||||||
@@ -206,14 +183,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
|||||||
"click ok tab={} target={target} at=({}, {})",
|
"click ok tab={} target={target} at=({}, {})",
|
||||||
reply.tab_id, reply.x, reply.y
|
reply.tab_id, reply.x, reply.y
|
||||||
);
|
);
|
||||||
if let Some(capture_id) = &reply.capture_id {
|
|
||||||
println!(
|
|
||||||
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
|
|
||||||
capture_id,
|
|
||||||
reply.image_x.unwrap_or_default(),
|
|
||||||
reply.image_y.unwrap_or_default()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
print_dialog_summaries(&reply.dialogs);
|
print_dialog_summaries(&reply.dialogs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,23 +40,15 @@ use bsk_protocol::ErrorCode;
|
|||||||
pub mod reason {
|
pub mod reason {
|
||||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||||
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
|
|
||||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||||
pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
|
|
||||||
pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
|
|
||||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||||
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
||||||
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
||||||
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
||||||
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
|
|
||||||
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
|
|
||||||
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
|
|
||||||
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
|
|
||||||
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
|
|
||||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||||
}
|
}
|
||||||
@@ -236,27 +228,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
|||||||
),
|
),
|
||||||
exit_code: base.exit_code,
|
exit_code: base.exit_code,
|
||||||
},
|
},
|
||||||
(ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
|
|
||||||
summary: "focused frame target could not be determined",
|
|
||||||
hint: Some(
|
|
||||||
"pass an explicit --ref or --selector so the key event can be routed safely",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
|
|
||||||
summary: "the page did not retain the requested input",
|
|
||||||
hint: Some(
|
|
||||||
"refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
|
||||||
summary: "snapshot ref does not support this operation",
|
|
||||||
hint: Some(
|
|
||||||
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
|
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
|
||||||
summary: "target element is not a <select>",
|
summary: "target element is not a <select>",
|
||||||
hint: Some(
|
hint: Some(
|
||||||
@@ -299,41 +270,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
|||||||
),
|
),
|
||||||
exit_code: base.exit_code,
|
exit_code: base.exit_code,
|
||||||
},
|
},
|
||||||
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
|
|
||||||
summary: "surface capture transaction was not found",
|
|
||||||
hint: Some(
|
|
||||||
"take a fresh surface screenshot and use its capture id with coordinates from that image",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
|
|
||||||
summary: "surface capture transaction has expired",
|
|
||||||
hint: Some(
|
|
||||||
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
|
|
||||||
summary: "surface capture transaction was already used",
|
|
||||||
hint: Some(
|
|
||||||
"each capture id is single-use; take a fresh surface screenshot before another point action",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
|
|
||||||
summary: "page state changed since the surface screenshot",
|
|
||||||
hint: Some(
|
|
||||||
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
|
|
||||||
summary: "surface image coordinate is invalid",
|
|
||||||
hint: Some(
|
|
||||||
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(
|
(
|
||||||
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
|
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
|
||||||
reason::RECORD_START_PAGE_UNREACHABLE,
|
reason::RECORD_START_PAGE_UNREACHABLE,
|
||||||
@@ -441,18 +377,6 @@ mod tests {
|
|||||||
assert!(!info.summary.contains("sandbox"));
|
assert!(!info.summary.contains("sandbox"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn ref_capability_denied_explains_visual_surface_refs() {
|
|
||||||
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
|
|
||||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
|
||||||
assert_eq!(info.summary, "snapshot ref does not support this operation");
|
|
||||||
assert!(
|
|
||||||
info.hint.unwrap().contains("capture id"),
|
|
||||||
"expected visual-surface-specific hint"
|
|
||||||
);
|
|
||||||
assert_eq!(info.exit_code, 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn session_busy_overrides_timeout_copy() {
|
fn session_busy_overrides_timeout_copy() {
|
||||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||||
@@ -495,25 +419,6 @@ mod tests {
|
|||||||
assert_eq!(info.exit_code, 3);
|
assert_eq!(info.exit_code, 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn surface_capture_stale_requests_a_fresh_screenshot() {
|
|
||||||
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
|
|
||||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
|
||||||
assert_eq!(
|
|
||||||
info.summary,
|
|
||||||
"page state changed since the surface screenshot"
|
|
||||||
);
|
|
||||||
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
|
|
||||||
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
|
|
||||||
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
|
|
||||||
assert_eq!(info.summary, "surface image coordinate is invalid");
|
|
||||||
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ref_not_found_overrides_not_found_copy() {
|
fn ref_not_found_overrides_not_found_copy() {
|
||||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||||
@@ -548,20 +453,4 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(info.hint.unwrap().contains("exactly one"));
|
assert!(info.hint.unwrap().contains("exactly one"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn focus_target_unresolved_overrides_cdp_copy() {
|
|
||||||
let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
|
|
||||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
|
||||||
assert_eq!(info.summary, "focused frame target could not be determined");
|
|
||||||
assert!(info.hint.unwrap().contains("--ref"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn input_not_applied_overrides_cdp_copy() {
|
|
||||||
let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
|
|
||||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
|
||||||
assert_eq!(info.summary, "the page did not retain the requested input");
|
|
||||||
assert!(info.hint.unwrap().contains("rejecting"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
|||||||
.map_err(CliError::Local)?;
|
.map_err(CliError::Local)?;
|
||||||
match format {
|
match format {
|
||||||
Format::Json => {
|
Format::Json => {
|
||||||
let mut json = serde_json::json!({
|
let json = serde_json::json!({
|
||||||
"tab_id": reply.tab_id,
|
"tab_id": reply.tab_id,
|
||||||
"width": reply.width,
|
"width": reply.width,
|
||||||
"height": reply.height,
|
"height": reply.height,
|
||||||
@@ -67,10 +67,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
|||||||
"path": out_path.to_string_lossy(),
|
"path": out_path.to_string_lossy(),
|
||||||
"byte_size": bytes.len(),
|
"byte_size": bytes.len(),
|
||||||
});
|
});
|
||||||
if let Some(capture) = &reply.capture {
|
|
||||||
json["capture"] = serde_json::to_value(capture)
|
|
||||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
|
||||||
}
|
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
serde_json::to_string_pretty(&json)
|
serde_json::to_string_pretty(&json)
|
||||||
@@ -79,12 +75,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
|||||||
}
|
}
|
||||||
Format::Human => {
|
Format::Human => {
|
||||||
println!("{}", out_path.display());
|
println!("{}", out_path.display());
|
||||||
if let Some(capture) = &reply.capture {
|
|
||||||
println!(
|
|
||||||
"capture={} surface={} coordinate_space={} expires_at={}",
|
|
||||||
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
|
|
||||||
);
|
|
||||||
}
|
|
||||||
print_dialog_summaries(&reply.dialogs);
|
print_dialog_summaries(&reply.dialogs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
|
|||||||
let bold = "\x1b[1m";
|
let bold = "\x1b[1m";
|
||||||
let reset = "\x1b[0m";
|
let reset = "\x1b[0m";
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
|
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}",
|
||||||
s.version_skew_browsers.len()
|
s.version_skew_browsers.len()
|
||||||
);
|
);
|
||||||
for skew in &s.version_skew_browsers {
|
for skew in &s.version_skew_browsers {
|
||||||
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
|
|||||||
skew.label.clone()
|
skew.label.clone()
|
||||||
};
|
};
|
||||||
eprintln!(
|
eprintln!(
|
||||||
" {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
|
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade",
|
||||||
skew.instance_id,
|
skew.instance_id,
|
||||||
label,
|
label,
|
||||||
display_protocol(&skew.client_protocol_version),
|
display_protocol(&skew.client_protocol_version),
|
||||||
|
|||||||
@@ -188,29 +188,6 @@ fn parses_click_count_alias() {
|
|||||||
assert_eq!(args.click_count, 2);
|
assert_eq!(args.click_count, 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn parses_surface_capture_image_coordinates() {
|
|
||||||
let cli = parse(&[
|
|
||||||
"bsk",
|
|
||||||
"click",
|
|
||||||
"@e1",
|
|
||||||
"--session",
|
|
||||||
"s1",
|
|
||||||
"--capture",
|
|
||||||
"sc_test",
|
|
||||||
"--image-x",
|
|
||||||
"160.5",
|
|
||||||
"--image-y",
|
|
||||||
"-1",
|
|
||||||
]);
|
|
||||||
let Command::Click(args) = cli.command else {
|
|
||||||
panic!("expected click command");
|
|
||||||
};
|
|
||||||
assert_eq!(args.capture.as_deref(), Some("sc_test"));
|
|
||||||
assert_eq!(args.image_x, Some(160.5));
|
|
||||||
assert_eq!(args.image_y, Some(-1.0));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parses_hover_with_settle() {
|
fn parses_hover_with_settle() {
|
||||||
let cli = parse(&[
|
let cli = parse(&[
|
||||||
|
|||||||
@@ -238,7 +238,6 @@ async fn screenshot_returns_image_base64_with_dimensions() {
|
|||||||
height: 600,
|
height: 600,
|
||||||
format: "png".into(),
|
format: "png".into(),
|
||||||
tab_id: 7,
|
tab_id: 7,
|
||||||
capture: None,
|
|
||||||
dialogs: vec![],
|
dialogs: vec![],
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
@@ -287,7 +286,6 @@ async fn screenshot_forwards_ref_to_extension() {
|
|||||||
height: 60,
|
height: 60,
|
||||||
format: "png".into(),
|
format: "png".into(),
|
||||||
tab_id: 7,
|
tab_id: 7,
|
||||||
capture: None,
|
|
||||||
dialogs: vec![],
|
dialogs: vec![],
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
|
|||||||
@@ -354,9 +354,6 @@ async fn click_round_trips_ref_and_modifiers() {
|
|||||||
used_selector: None,
|
used_selector: None,
|
||||||
x: 12.5,
|
x: 12.5,
|
||||||
y: 34.0,
|
y: 34.0,
|
||||||
capture_id: None,
|
|
||||||
image_x: None,
|
|
||||||
image_y: None,
|
|
||||||
dialogs: vec![],
|
dialogs: vec![],
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
@@ -376,9 +373,6 @@ async fn click_round_trips_ref_and_modifiers() {
|
|||||||
click_count: Some(1),
|
click_count: Some(1),
|
||||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||||
timeout_ms: Some(5_000),
|
timeout_ms: Some(5_000),
|
||||||
capture_id: None,
|
|
||||||
image_x: None,
|
|
||||||
image_y: None,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -616,9 +610,6 @@ async fn m7_tools_propagate_extension_errors() {
|
|||||||
click_count: None,
|
click_count: None,
|
||||||
modifiers: None,
|
modifiers: None,
|
||||||
timeout_ms: Some(5_000),
|
timeout_ms: Some(5_000),
|
||||||
capture_id: None,
|
|
||||||
image_x: None,
|
|
||||||
image_y: None,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -16,13 +16,6 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"capture_id": {
|
|
||||||
"description": "Short-lived capture id returned by a visual Surface screenshot.",
|
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"click_count": {
|
"click_count": {
|
||||||
"description": "Number of consecutive mouse presses (double-click = 2).",
|
"description": "Number of consecutive mouse presses (double-click = 2).",
|
||||||
"type": [
|
"type": [
|
||||||
@@ -32,22 +25,6 @@
|
|||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
"minimum": 1.0
|
"minimum": 1.0
|
||||||
},
|
},
|
||||||
"image_x": {
|
|
||||||
"description": "X coordinate in the capture image's pixel coordinate space.",
|
|
||||||
"type": [
|
|
||||||
"number",
|
|
||||||
"null"
|
|
||||||
],
|
|
||||||
"format": "double"
|
|
||||||
},
|
|
||||||
"image_y": {
|
|
||||||
"description": "Y coordinate in the capture image's pixel coordinate space.",
|
|
||||||
"type": [
|
|
||||||
"number",
|
|
||||||
"null"
|
|
||||||
],
|
|
||||||
"format": "double"
|
|
||||||
},
|
|
||||||
"modifiers": {
|
"modifiers": {
|
||||||
"type": [
|
"type": [
|
||||||
"array",
|
"array",
|
||||||
|
|||||||
@@ -8,12 +8,6 @@
|
|||||||
"y"
|
"y"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"capture_id": {
|
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"dialogs": {
|
"dialogs": {
|
||||||
"description": "Native JS dialogs observed and auto-handled during this call.",
|
"description": "Native JS dialogs observed and auto-handled during this call.",
|
||||||
"type": "array",
|
"type": "array",
|
||||||
@@ -21,20 +15,6 @@
|
|||||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"image_x": {
|
|
||||||
"type": [
|
|
||||||
"number",
|
|
||||||
"null"
|
|
||||||
],
|
|
||||||
"format": "double"
|
|
||||||
},
|
|
||||||
"image_y": {
|
|
||||||
"type": [
|
|
||||||
"number",
|
|
||||||
"null"
|
|
||||||
],
|
|
||||||
"format": "double"
|
|
||||||
},
|
|
||||||
"tab_id": {
|
"tab_id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "int64"
|
"format": "int64"
|
||||||
|
|||||||
@@ -8,16 +8,6 @@
|
|||||||
"tab_id"
|
"tab_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"capture": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/SurfaceCaptureInfo"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"dialogs": {
|
"dialogs": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
@@ -119,31 +109,6 @@
|
|||||||
"prompt",
|
"prompt",
|
||||||
"beforeunload"
|
"beforeunload"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
"SurfaceCaptureInfo": {
|
|
||||||
"type": "object",
|
|
||||||
"required": [
|
|
||||||
"coordinate_space",
|
|
||||||
"expires_at",
|
|
||||||
"id",
|
|
||||||
"surface_ref"
|
|
||||||
],
|
|
||||||
"properties": {
|
|
||||||
"coordinate_space": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"expires_at": {
|
|
||||||
"type": "integer",
|
|
||||||
"format": "uint64",
|
|
||||||
"minimum": 0.0
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"surface_ref": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -67,15 +67,6 @@ pub struct ClickParams {
|
|||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
#[schemars(range(min = 1))]
|
#[schemars(range(min = 1))]
|
||||||
pub timeout_ms: Option<u32>,
|
pub timeout_ms: Option<u32>,
|
||||||
/// Short-lived capture id returned by a visual Surface screenshot.
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub capture_id: Option<String>,
|
|
||||||
/// X coordinate in the capture image's pixel coordinate space.
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub image_x: Option<f64>,
|
|
||||||
/// Y coordinate in the capture image's pixel coordinate space.
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub image_y: Option<f64>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
@@ -89,12 +80,6 @@ pub struct ClickResult {
|
|||||||
/// agents can correlate with a follow-up `tool.screenshot`.
|
/// agents can correlate with a follow-up `tool.screenshot`.
|
||||||
pub x: f64,
|
pub x: f64,
|
||||||
pub y: f64,
|
pub y: f64,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub capture_id: Option<String>,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub image_x: Option<f64>,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub image_y: Option<f64>,
|
|
||||||
/// Native JS dialogs observed and auto-handled during this call.
|
/// Native JS dialogs observed and auto-handled during this call.
|
||||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
@@ -303,9 +288,6 @@ mod tests {
|
|||||||
click_count: Some(1),
|
click_count: Some(1),
|
||||||
modifiers: Some(vec![KeyModifier::Ctrl]),
|
modifiers: Some(vec![KeyModifier::Ctrl]),
|
||||||
timeout_ms: Some(5_000),
|
timeout_ms: Some(5_000),
|
||||||
capture_id: Some("sc_test".into()),
|
|
||||||
image_x: Some(160.0),
|
|
||||||
image_y: Some(90.0),
|
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&p).unwrap();
|
let v = serde_json::to_value(&p).unwrap();
|
||||||
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||||
|
|||||||
@@ -208,20 +208,10 @@ pub struct ScreenshotResult {
|
|||||||
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
|
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
|
||||||
pub format: String,
|
pub format: String,
|
||||||
pub tab_id: i64,
|
pub tab_id: i64,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub capture: Option<SurfaceCaptureInfo>,
|
|
||||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
|
||||||
pub struct SurfaceCaptureInfo {
|
|
||||||
pub id: String,
|
|
||||||
pub surface_ref: String,
|
|
||||||
pub coordinate_space: String,
|
|
||||||
pub expires_at: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -303,12 +293,6 @@ mod tests {
|
|||||||
height: 600,
|
height: 600,
|
||||||
format: "png".into(),
|
format: "png".into(),
|
||||||
tab_id: 7,
|
tab_id: 7,
|
||||||
capture: Some(SurfaceCaptureInfo {
|
|
||||||
id: "sc_test".into(),
|
|
||||||
surface_ref: "@e3".into(),
|
|
||||||
coordinate_space: "capture-image-pixel".into(),
|
|
||||||
expires_at: 1_700_000_030_000,
|
|
||||||
}),
|
|
||||||
dialogs: vec![],
|
dialogs: vec![],
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&r).unwrap();
|
let v = serde_json::to_value(&r).unwrap();
|
||||||
|
|||||||
@@ -107,8 +107,6 @@
|
|||||||
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
|
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
|
||||||
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
|
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
|
||||||
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
||||||
"@deepseek-ai/dsh-scope": "^0.1.0-rc.6",
|
|
||||||
"@deepseek-ai/dsh-skill": "^0.1.0-rc.6",
|
|
||||||
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
||||||
"@deepseek-ai/schemastery": "^3.18.1",
|
"@deepseek-ai/schemastery": "^3.18.1",
|
||||||
"@tailwindcss/cli": "^4.3.3",
|
"@tailwindcss/cli": "^4.3.3",
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import { registerObservationRoutes } from "./observation-http";
|
|||||||
import { KeyedExecutor } from "./queue";
|
import { KeyedExecutor } from "./queue";
|
||||||
import { type BskRunner, createBskRunner } from "./runner";
|
import { type BskRunner, createBskRunner } from "./runner";
|
||||||
import { SessionRegistry } from "./sessions";
|
import { SessionRegistry } from "./sessions";
|
||||||
import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
|
import { registerBskSkill } from "./skill";
|
||||||
import { type PluginConfig, registerTools } from "./tools";
|
import { type PluginConfig, registerTools } from "./tools";
|
||||||
|
|
||||||
export const name = "dsh-plugin-browserskill";
|
export const name = "dsh-plugin-browserskill";
|
||||||
@@ -95,11 +95,6 @@ export function apply(
|
|||||||
// ONLY model-visible advertisement — the tool suite reveals itself on a
|
// ONLY model-visible advertisement — the tool suite reveals itself on a
|
||||||
// successful skill invocation (or a session whose history already has one).
|
// successful skill invocation (or a session whose history already has one).
|
||||||
const unregisterSkill = registerBskSkill(ctx);
|
const unregisterSkill = registerBskSkill(ctx);
|
||||||
// A same-name CLI skill left in ~/.agents is discovered in the nearer preset
|
|
||||||
// layer and shadows the global registration above. Re-register through every
|
|
||||||
// exact agent context at startup so DSH always loads the browser_* protocol
|
|
||||||
// instructions; the shared CLI skill remains untouched for other agents.
|
|
||||||
const disarmAgentSkill = armAgentScopedBskSkill(ctx);
|
|
||||||
const removeSuite = resolved.lazyTools
|
const removeSuite = resolved.lazyTools
|
||||||
? armLazyTools(ctx, () =>
|
? armLazyTools(ctx, () =>
|
||||||
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
|
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
|
||||||
@@ -140,15 +135,17 @@ export function apply(
|
|||||||
ctx.effect(() => {
|
ctx.effect(() => {
|
||||||
return () => {
|
return () => {
|
||||||
removeSuite();
|
removeSuite();
|
||||||
disarmAgentSkill();
|
|
||||||
unregisterSkill();
|
unregisterSkill();
|
||||||
removeRoutes();
|
removeRoutes();
|
||||||
disarmArchiveCleanup();
|
disarmArchiveCleanup();
|
||||||
|
observation.dispose();
|
||||||
runner.killAll();
|
runner.killAll();
|
||||||
const stops = registry
|
const stops = registry
|
||||||
.ownedIds()
|
.ownedIds()
|
||||||
.map((sessionId) => observation.stopSession(sessionId).catch(() => false));
|
.map((sessionId) =>
|
||||||
return Promise.all(stops).then(() => observation.dispose());
|
runner.run(["session", "stop", sessionId], { timeoutMs: 15_000 }).catch(() => {}),
|
||||||
|
);
|
||||||
|
return Promise.all(stops).then(() => {});
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import { tmpdir } from "node:os";
|
|||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import type { Context } from "@deepseek-ai/cordis";
|
import type { Context } from "@deepseek-ai/cordis";
|
||||||
import type { KeyedExecutor } from "./queue";
|
import type { KeyedExecutor } from "./queue";
|
||||||
import { BskError, type BskRunner, parseBskJson, runWithSessionBusyRetry } from "./runner";
|
import type { BskRunner } from "./runner";
|
||||||
import type { SessionRegistry } from "./sessions";
|
import type { SessionRegistry } from "./sessions";
|
||||||
|
|
||||||
/** One owned session's live observation record (wire-stable shape). */
|
/** One owned session's live observation record (wire-stable shape). */
|
||||||
@@ -90,10 +90,6 @@ export class ObservationService {
|
|||||||
private readonly listeners = new Set<(event: ObservationEvent) => void>();
|
private readonly listeners = new Set<(event: ObservationEvent) => void>();
|
||||||
private readonly captureTimers = new Map<string, unknown>();
|
private readonly captureTimers = new Map<string, unknown>();
|
||||||
private readonly captureInFlight = new Set<string>();
|
private readonly captureInFlight = new Set<string>();
|
||||||
/** Captures intentionally cancelled to make way for foreground work. */
|
|
||||||
private readonly capturePreempted = new Set<string>();
|
|
||||||
/** Number of queued/running model-facing calls that currently outrank thumbnails. */
|
|
||||||
private readonly foregroundDepth = new Map<string, number>();
|
|
||||||
private readonly captureFailures = new Map<string, number>();
|
private readonly captureFailures = new Map<string, number>();
|
||||||
private readonly lastActivity = new Map<string, number>();
|
private readonly lastActivity = new Map<string, number>();
|
||||||
/** Ephemeral frame id → PNG bytes. Only the live ring members are present. */
|
/** Ephemeral frame id → PNG bytes. Only the live ring members are present. */
|
||||||
@@ -179,8 +175,6 @@ export class ObservationService {
|
|||||||
/** Drop a session (called from browser_session_stop). */
|
/** Drop a session (called from browser_session_stop). */
|
||||||
removeSession(sessionId: string): void {
|
removeSession(sessionId: string): void {
|
||||||
this.cancelCapture(sessionId);
|
this.cancelCapture(sessionId);
|
||||||
this.foregroundDepth.delete(sessionId);
|
|
||||||
this.capturePreempted.delete(sessionId);
|
|
||||||
this.captureFailures.delete(sessionId);
|
this.captureFailures.delete(sessionId);
|
||||||
this.lastActivity.delete(sessionId);
|
this.lastActivity.delete(sessionId);
|
||||||
this.dropSessionFrames(sessionId);
|
this.dropSessionFrames(sessionId);
|
||||||
@@ -192,37 +186,6 @@ export class ObservationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Give model-facing work priority over the best-effort thumbnail lane.
|
|
||||||
* The first lease cancels a pending timer and gracefully interrupts an
|
|
||||||
* active bsk screenshot; the last release schedules one fresh frame.
|
|
||||||
*/
|
|
||||||
acquireForeground(sessionId: string): () => void {
|
|
||||||
if (!this.deps.options.enabled || this.disposed || !this.observations.has(sessionId)) {
|
|
||||||
return () => {};
|
|
||||||
}
|
|
||||||
const depth = this.foregroundDepth.get(sessionId) ?? 0;
|
|
||||||
this.foregroundDepth.set(sessionId, depth + 1);
|
|
||||||
if (depth === 0) {
|
|
||||||
this.cancelCapture(sessionId);
|
|
||||||
if (this.deps.runner.killFor(`observation:${sessionId}`) > 0) {
|
|
||||||
this.capturePreempted.add(sessionId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let released = false;
|
|
||||||
return () => {
|
|
||||||
if (released) return;
|
|
||||||
released = true;
|
|
||||||
const remaining = (this.foregroundDepth.get(sessionId) ?? 1) - 1;
|
|
||||||
if (remaining > 0) {
|
|
||||||
this.foregroundDepth.set(sessionId, remaining);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.foregroundDepth.delete(sessionId);
|
|
||||||
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId, 0);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Mark an action starting on a session (tool entry instrumentation). */
|
/** Mark an action starting on a session (tool entry instrumentation). */
|
||||||
beginAction(sessionId: string, action: string): void {
|
beginAction(sessionId: string, action: string): void {
|
||||||
if (!this.deps.options.enabled || this.disposed) return;
|
if (!this.deps.options.enabled || this.disposed) return;
|
||||||
@@ -250,7 +213,7 @@ export class ObservationService {
|
|||||||
if (error !== undefined) next.lastError = error;
|
if (error !== undefined) next.lastError = error;
|
||||||
else delete next.lastError;
|
else delete next.lastError;
|
||||||
this.put(next);
|
this.put(next);
|
||||||
if (!this.foregroundDepth.has(sessionId)) this.scheduleCapture(sessionId, 0);
|
this.scheduleCapture(sessionId, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Record the settled page URL (navigate success / start with url). */
|
/** Record the settled page URL (navigate success / start with url). */
|
||||||
@@ -279,48 +242,29 @@ export class ObservationService {
|
|||||||
* button — same end state as `browser_session_stop`). Never waits behind a
|
* button — same end state as `browser_session_stop`). Never waits behind a
|
||||||
* hung in-flight command: tool children are killed first so the session's
|
* hung in-flight command: tool children are killed first so the session's
|
||||||
* keyed queue drains immediately, and no further captures queue up. A
|
* keyed queue drains immediately, and no further captures queue up. A
|
||||||
* session the daemon already forgot stops idempotently — the goal state
|
* session the daemon already forgot (dead strip entries) stops
|
||||||
* (entry gone) is identical.
|
* idempotently — the goal state (entry gone) is identical.
|
||||||
* @returns false only for a foreign session; bsk failures reject so callers
|
* @returns whether the session ended up stopped/removed.
|
||||||
* can preserve the structured error instead of silently leaving a ghost.
|
|
||||||
*/
|
*/
|
||||||
async stopSession(sessionId: string, signal?: AbortSignal): Promise<boolean> {
|
async stopSession(sessionId: string): Promise<boolean> {
|
||||||
if (!this.deps.registry.isOwned(sessionId)) return false;
|
if (!this.deps.registry.isOwned(sessionId)) return false;
|
||||||
const releaseForeground = this.acquireForeground(sessionId);
|
this.cancelCapture(sessionId);
|
||||||
let actionError: string | undefined;
|
this.deps.runner.killFor(sessionId);
|
||||||
this.beginAction(sessionId, "stopping");
|
const result = await this.deps.queue.run(sessionId, () =>
|
||||||
try {
|
this.deps.runner.run(["session", "stop", sessionId], { timeoutMs: 30_000 }),
|
||||||
this.deps.runner.killFor(sessionId);
|
);
|
||||||
const result = await this.deps.queue.run(
|
if (result.code !== 0) {
|
||||||
sessionId,
|
let code: string | undefined;
|
||||||
() =>
|
|
||||||
runWithSessionBusyRetry(
|
|
||||||
() =>
|
|
||||||
this.deps.runner.run(["session", "stop", sessionId], {
|
|
||||||
signal,
|
|
||||||
timeoutMs: 30_000,
|
|
||||||
tag: sessionId,
|
|
||||||
}),
|
|
||||||
signal,
|
|
||||||
),
|
|
||||||
signal,
|
|
||||||
);
|
|
||||||
if (result.aborted) throw abortError();
|
|
||||||
try {
|
try {
|
||||||
parseBskJson(result, "session stop");
|
code = (JSON.parse(result.stdout) as { code?: string }).code;
|
||||||
} catch (error) {
|
} catch {
|
||||||
if (!isSessionNotFoundError(error)) throw error;
|
code = undefined;
|
||||||
}
|
}
|
||||||
this.deps.registry.remove(sessionId);
|
if (code !== "session_not_found") return false;
|
||||||
this.removeSession(sessionId);
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
|
||||||
actionError = error instanceof Error ? error.message.split("\n")[0] : String(error);
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
this.endAction(sessionId, actionError);
|
|
||||||
releaseForeground();
|
|
||||||
}
|
}
|
||||||
|
this.deps.registry.remove(sessionId);
|
||||||
|
this.removeSession(sessionId);
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -343,8 +287,6 @@ export class ObservationService {
|
|||||||
for (const sessionId of [...this.captureTimers.keys()]) this.cancelCapture(sessionId);
|
for (const sessionId of [...this.captureTimers.keys()]) this.cancelCapture(sessionId);
|
||||||
this.captureTimers.clear();
|
this.captureTimers.clear();
|
||||||
this.captureInFlight.clear();
|
this.captureInFlight.clear();
|
||||||
this.capturePreempted.clear();
|
|
||||||
this.foregroundDepth.clear();
|
|
||||||
this.captureFailures.clear();
|
this.captureFailures.clear();
|
||||||
this.lastActivity.clear();
|
this.lastActivity.clear();
|
||||||
this.frames.clear();
|
this.frames.clear();
|
||||||
@@ -371,7 +313,6 @@ export class ObservationService {
|
|||||||
/** Schedule the next capture for a session; `delayMs` 0 means "as soon as the event loop allows". */
|
/** Schedule the next capture for a session; `delayMs` 0 means "as soon as the event loop allows". */
|
||||||
private scheduleCapture(sessionId: string, delayMs?: number): void {
|
private scheduleCapture(sessionId: string, delayMs?: number): void {
|
||||||
if (!this.deps.options.enabled || this.disposed) return;
|
if (!this.deps.options.enabled || this.disposed) return;
|
||||||
if (this.foregroundDepth.has(sessionId)) return;
|
|
||||||
const entry = this.observations.get(sessionId);
|
const entry = this.observations.get(sessionId);
|
||||||
if (entry === undefined || entry.dead === true) return;
|
if (entry === undefined || entry.dead === true) return;
|
||||||
this.cancelCapture(sessionId);
|
this.cancelCapture(sessionId);
|
||||||
@@ -408,15 +349,12 @@ export class ObservationService {
|
|||||||
const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
|
const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
|
||||||
let writtenPath = outPath;
|
let writtenPath = outPath;
|
||||||
try {
|
try {
|
||||||
const result = await this.deps.queue.run(sessionId, () => {
|
const result = await this.deps.queue.run(sessionId, () =>
|
||||||
// A foreground lease may have arrived while this capture was queued.
|
this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
|
||||||
if (this.foregroundDepth.has(sessionId)) return Promise.resolve(undefined);
|
|
||||||
return this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
|
|
||||||
timeoutMs: 15_000,
|
timeoutMs: 15_000,
|
||||||
tag: `observation:${sessionId}`,
|
tag: `observation:${sessionId}`,
|
||||||
});
|
}),
|
||||||
});
|
);
|
||||||
if (result === undefined) return;
|
|
||||||
if (result.code !== 0) {
|
if (result.code !== 0) {
|
||||||
let code: string | undefined;
|
let code: string | undefined;
|
||||||
try {
|
try {
|
||||||
@@ -424,12 +362,8 @@ export class ObservationService {
|
|||||||
} catch {
|
} catch {
|
||||||
code = undefined;
|
code = undefined;
|
||||||
}
|
}
|
||||||
if (isSessionNotFoundCode(code)) {
|
if (code === "session_not_found") {
|
||||||
// Observation entries are owned sessions. If the daemon has already
|
this.markDead(sessionId);
|
||||||
// forgotten one (external close / restart), reconcile both stores so
|
|
||||||
// the PiP cannot retain a ghost strip or keep retrying screenshots.
|
|
||||||
this.deps.registry.remove(sessionId);
|
|
||||||
this.removeSession(sessionId);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
throw new Error(`screenshot exited ${result.code}`);
|
throw new Error(`screenshot exited ${result.code}`);
|
||||||
@@ -444,17 +378,12 @@ export class ObservationService {
|
|||||||
this.setAvailable(true);
|
this.setAvailable(true);
|
||||||
this.publishFrame(sessionId, entry, data);
|
this.publishFrame(sessionId, entry, data);
|
||||||
} catch {
|
} catch {
|
||||||
// Foreground preemption is healthy scheduling, not browser/daemon
|
// Silent by design: keep the previous frame, count toward backoff.
|
||||||
// unavailability; do not let normal tool traffic trip capture backoff.
|
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
|
||||||
if (!this.capturePreempted.delete(sessionId)) {
|
this.globalFailures += 1;
|
||||||
// Silent by design: keep the previous frame, count toward backoff.
|
if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
|
||||||
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
|
|
||||||
this.globalFailures += 1;
|
|
||||||
if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
await unlink(writtenPath).catch(() => {});
|
await unlink(writtenPath).catch(() => {});
|
||||||
this.capturePreempted.delete(sessionId);
|
|
||||||
this.captureInFlight.delete(sessionId);
|
this.captureInFlight.delete(sessionId);
|
||||||
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
|
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
|
||||||
}
|
}
|
||||||
@@ -485,20 +414,14 @@ export class ObservationService {
|
|||||||
this.hashes.delete(sessionId);
|
this.hashes.delete(sessionId);
|
||||||
this.seqs.delete(sessionId);
|
this.seqs.delete(sessionId);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
function isSessionNotFoundCode(code: string | undefined): boolean {
|
/** Mark a session dead: grey it out, stop asking for frames, keep the entry for the strip. */
|
||||||
return code === "not_found" || code === "session_not_found";
|
private markDead(sessionId: string): void {
|
||||||
}
|
const entry = this.observations.get(sessionId);
|
||||||
|
if (entry === undefined || entry.dead === true) return;
|
||||||
function isSessionNotFoundError(error: unknown): boolean {
|
this.cancelCapture(sessionId);
|
||||||
return error instanceof BskError && isSessionNotFoundCode(error.code);
|
this.put({ ...entry, dead: true, action: "idle" });
|
||||||
}
|
}
|
||||||
|
|
||||||
function abortError(): Error {
|
|
||||||
const error = new Error("tool call aborted");
|
|
||||||
error.name = "AbortError";
|
|
||||||
return error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Map a bsk command label onto its observation action verb. */
|
/** Map a bsk command label onto its observation action verb. */
|
||||||
|
|||||||
@@ -64,17 +64,14 @@ export interface BskRunner {
|
|||||||
killFor(tag: string): number;
|
killFor(tag: string): number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Business RPCs translate SIGINT into the daemon's cancel(rpc_id) protocol.
|
const KILL_GRACE_MS = 2000;
|
||||||
// Give that bounded reconciliation path time to settle before the hard kill.
|
|
||||||
const KILL_GRACE_MS = 3000;
|
|
||||||
const SESSION_BUSY_RETRY_DELAY_MS = 100;
|
|
||||||
|
|
||||||
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
|
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
|
||||||
const live = new Map<ChildProcess, string | undefined>();
|
const live = new Map<ChildProcess, string | undefined>();
|
||||||
|
|
||||||
function killChild(child: ChildProcess): void {
|
function killChild(child: ChildProcess): void {
|
||||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||||
child.kill("SIGINT");
|
child.kill("SIGTERM");
|
||||||
const force = setTimeout(() => {
|
const force = setTimeout(() => {
|
||||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||||
}, KILL_GRACE_MS);
|
}, KILL_GRACE_MS);
|
||||||
@@ -166,60 +163,6 @@ export function isCommandNotFound(error: unknown): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** True only for the daemon's transient per-session reconciliation window. */
|
|
||||||
export function isSessionBusyResult(result: BskRunResult): boolean {
|
|
||||||
if (result.code === 0) return false;
|
|
||||||
try {
|
|
||||||
const body = JSON.parse(result.stdout) as BskErrorBody;
|
|
||||||
const reason =
|
|
||||||
typeof body.data === "object" && body.data !== null && "reason" in body.data
|
|
||||||
? (body.data as { reason?: unknown }).reason
|
|
||||||
: undefined;
|
|
||||||
return reason === "session_busy";
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Retry exactly once after the tiny daemon-settlement race that can follow a
|
|
||||||
* graceful SIGINT cancellation. Other errors and persistent busy states stay
|
|
||||||
* visible to callers.
|
|
||||||
*/
|
|
||||||
export async function runWithSessionBusyRetry(
|
|
||||||
run: () => Promise<BskRunResult>,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
): Promise<BskRunResult> {
|
|
||||||
const first = await run();
|
|
||||||
if (!isSessionBusyResult(first)) return first;
|
|
||||||
await abortableDelay(SESSION_BUSY_RETRY_DELAY_MS, signal);
|
|
||||||
return run();
|
|
||||||
}
|
|
||||||
|
|
||||||
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
|
||||||
if (signal?.aborted) return Promise.reject(abortError());
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const timer = setTimeout(done, ms);
|
|
||||||
timer.unref();
|
|
||||||
const onAbort = () => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
reject(abortError());
|
|
||||||
};
|
|
||||||
function done() {
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
resolve();
|
|
||||||
}
|
|
||||||
signal?.addEventListener("abort", onAbort, { once: true });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function abortError(): Error {
|
|
||||||
const error = new Error("tool call aborted");
|
|
||||||
error.name = "AbortError";
|
|
||||||
return error;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Install guidance shown when the bsk CLI cannot be spawned. */
|
/** Install guidance shown when the bsk CLI cannot be spawned. */
|
||||||
export function bskInstallMessage(bskPath: string): string {
|
export function bskInstallMessage(bskPath: string): string {
|
||||||
return (
|
return (
|
||||||
@@ -239,7 +182,7 @@ export function parseBskJson(result: BskRunResult, commandLabel: string): unknow
|
|||||||
throw new BskError(`bsk ${commandLabel} timed out`, { timedOut: true });
|
throw new BskError(`bsk ${commandLabel} timed out`, { timedOut: true });
|
||||||
}
|
}
|
||||||
const body = result.stdout.trim();
|
const body = result.stdout.trim();
|
||||||
// Killed by our own interrupt (SIGINT from killFor), not by the abort path:
|
// Killed by our own interrupt (SIGTERM from killFor), not by the abort path:
|
||||||
// say so instead of doubling the generic label into the message.
|
// say so instead of doubling the generic label into the message.
|
||||||
if (result.code === null && !result.aborted && !result.timedOut) {
|
if (result.code === null && !result.aborted && !result.timedOut) {
|
||||||
throw new BskError(`bsk ${commandLabel} was interrupted (process killed)`);
|
throw new BskError(`bsk ${commandLabel} was interrupted (process killed)`);
|
||||||
|
|||||||
@@ -9,7 +9,6 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import type { Context } from "@deepseek-ai/cordis";
|
import type { Context } from "@deepseek-ai/cordis";
|
||||||
import type { Agent } from "@deepseek-ai/dsh-agent";
|
|
||||||
import {
|
import {
|
||||||
BSK_SKILL_DESCRIPTION,
|
BSK_SKILL_DESCRIPTION,
|
||||||
BSK_SKILL_MARKDOWN,
|
BSK_SKILL_MARKDOWN,
|
||||||
@@ -26,11 +25,6 @@ interface SkillsLike {
|
|||||||
}): () => void;
|
}): () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Structural view of the live-agent registry (absent in bare compositions). */
|
|
||||||
interface AgentsLike {
|
|
||||||
list(): Agent[];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Publish the browser skill as an embedded runtime skill. Returns the unregistration
|
* Publish the browser skill as an embedded runtime skill. Returns the unregistration
|
||||||
* disposer. A composition without the `skills` service (or with dsh-tool-skill
|
* disposer. A composition without the `skills` service (or with dsh-tool-skill
|
||||||
@@ -49,54 +43,3 @@ export function registerBskSkill(ctx: Context): () => void {
|
|||||||
source: "bundled",
|
source: "bundled",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Install the DSH-specific browser skill into every exact agent scope.
|
|
||||||
*
|
|
||||||
* DSH merges skill layers nearest-first (`agent -> preset -> global`). A legacy
|
|
||||||
* CLI skill installed at `~/.agents/skills/browser-skill` is discovered by the
|
|
||||||
* preset filesystem provider and therefore shadows a plugin-level registration.
|
|
||||||
* Registering the embedded skill through `agent.ctx` makes the DSH protocol
|
|
||||||
* contract authoritative for that agent without touching the shared CLI skill.
|
|
||||||
*
|
|
||||||
* New agents are handled at `agent/session-start`, the first supported startup
|
|
||||||
* injection point and still before the first prompt assembly. Existing agents
|
|
||||||
* are registered immediately so plugin reloads take effect without recreating
|
|
||||||
* the conversation. Returns a disposer for all plugin-owned registrations.
|
|
||||||
*/
|
|
||||||
export function armAgentScopedBskSkill(ctx: Context): () => void {
|
|
||||||
const registrations = new Map<Agent, () => void>();
|
|
||||||
let active = true;
|
|
||||||
|
|
||||||
const registerForAgent = (agent: Agent): void => {
|
|
||||||
if (!active || registrations.has(agent)) return;
|
|
||||||
registrations.set(agent, registerBskSkill(agent.ctx));
|
|
||||||
};
|
|
||||||
|
|
||||||
let stopSessionStart = () => {};
|
|
||||||
let stopDisposed = () => {};
|
|
||||||
if (typeof ctx.on === "function") {
|
|
||||||
stopSessionStart = ctx.on("agent/session-start", ({ agent }) => {
|
|
||||||
registerForAgent(agent);
|
|
||||||
});
|
|
||||||
stopDisposed = ctx.on("agent/disposed", ({ agent }) => {
|
|
||||||
// Agent-scoped effects have already unwound at this lifecycle edge. Drop
|
|
||||||
// the retained disposer without invoking it against a disposed context.
|
|
||||||
registrations.delete(agent);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const agents = ctx.get("agents") as AgentsLike | undefined | null;
|
|
||||||
if (agents != null && typeof agents.list === "function") {
|
|
||||||
for (const agent of agents.list()) registerForAgent(agent);
|
|
||||||
}
|
|
||||||
|
|
||||||
return () => {
|
|
||||||
if (!active) return;
|
|
||||||
active = false;
|
|
||||||
stopDisposed();
|
|
||||||
stopSessionStart();
|
|
||||||
for (const unregister of [...registrations.values()].reverse()) unregister();
|
|
||||||
registrations.clear();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -22,13 +22,7 @@ import { trySaveScreenshot } from "./image";
|
|||||||
import { actionForLabel, type ObservationService } from "./observation";
|
import { actionForLabel, type ObservationService } from "./observation";
|
||||||
import { registerPhaseOneTools } from "./phase-one-tools";
|
import { registerPhaseOneTools } from "./phase-one-tools";
|
||||||
import type { KeyedExecutor } from "./queue";
|
import type { KeyedExecutor } from "./queue";
|
||||||
import {
|
import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
|
||||||
type BskRunner,
|
|
||||||
bskInstallMessage,
|
|
||||||
isCommandNotFound,
|
|
||||||
parseBskJson,
|
|
||||||
runWithSessionBusyRetry,
|
|
||||||
} from "./runner";
|
|
||||||
import type { SessionRegistry } from "./sessions";
|
import type { SessionRegistry } from "./sessions";
|
||||||
import { SESSION_PARAM } from "./tool-params";
|
import { SESSION_PARAM } from "./tool-params";
|
||||||
|
|
||||||
@@ -85,8 +79,6 @@ async function runBsk(
|
|||||||
observeSession?: string,
|
observeSession?: string,
|
||||||
runnerTimeoutMs?: number,
|
runnerTimeoutMs?: number,
|
||||||
): Promise<unknown> {
|
): Promise<unknown> {
|
||||||
const releaseForeground =
|
|
||||||
observeSession !== undefined ? deps.observation.acquireForeground(observeSession) : undefined;
|
|
||||||
// beginAction must fire only when the task actually starts inside the
|
// beginAction must fire only when the task actually starts inside the
|
||||||
// per-session queue — marking it while still queued would overwrite the
|
// per-session queue — marking it while still queued would overwrite the
|
||||||
// in-flight action's label (and flash it idle on a queued abort).
|
// in-flight action's label (and flash it idle on a queued abort).
|
||||||
@@ -100,15 +92,11 @@ async function runBsk(
|
|||||||
began = true;
|
began = true;
|
||||||
deps.observation.beginAction(observeSession, actionForLabel(label));
|
deps.observation.beginAction(observeSession, actionForLabel(label));
|
||||||
}
|
}
|
||||||
return runWithSessionBusyRetry(
|
return deps.runner.run(args, {
|
||||||
() =>
|
signal: exec.signal,
|
||||||
deps.runner.run(args, {
|
timeoutMs: runnerTimeoutMs ?? deps.config.defaultTimeoutMs,
|
||||||
signal: exec.signal,
|
...(observeSession !== undefined ? { tag: observeSession } : {}),
|
||||||
timeoutMs: runnerTimeoutMs ?? deps.config.defaultTimeoutMs,
|
});
|
||||||
...(observeSession !== undefined ? { tag: observeSession } : {}),
|
|
||||||
}),
|
|
||||||
exec.signal,
|
|
||||||
);
|
|
||||||
};
|
};
|
||||||
result =
|
result =
|
||||||
observeSession !== undefined
|
observeSession !== undefined
|
||||||
@@ -133,7 +121,6 @@ async function runBsk(
|
|||||||
if (observeSession !== undefined && began) {
|
if (observeSession !== undefined && began) {
|
||||||
deps.observation.endAction(observeSession, actionError);
|
deps.observation.endAction(observeSession, actionError);
|
||||||
}
|
}
|
||||||
releaseForeground?.();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,15 +267,22 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// A half-initialized session must not leak: stop it before surfacing.
|
// A half-initialized session must not leak: stop it before surfacing.
|
||||||
// Reuse the same capture-preempting, idempotent path as explicit and
|
// The stop funnels through the session's queue (an in-flight command
|
||||||
// overlay stops; local ownership is dropped even when daemon cleanup
|
// — e.g. an observation capture — would otherwise make the daemon
|
||||||
// itself fails, because this start never becomes usable to the model.
|
// refuse it), with one retry for good measure.
|
||||||
try {
|
for (let attempt = 0; attempt < 2; attempt++) {
|
||||||
await deps.observation.stopSession(reply.session_id);
|
try {
|
||||||
} catch {
|
const stopped = await deps.queue.run(reply.session_id, () =>
|
||||||
registry.remove(reply.session_id);
|
deps.runner.run(["session", "stop", reply.session_id], { timeoutMs: 30_000 }),
|
||||||
deps.observation.removeSession(reply.session_id);
|
);
|
||||||
|
if (stopped.code === 0) break;
|
||||||
|
} catch {
|
||||||
|
// retry once, then give up quietly (registry/observation are
|
||||||
|
// cleaned below either way; the daemon reaps orphans).
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
registry.remove(reply.session_id);
|
||||||
|
deps.observation.removeSession(reply.session_id);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
@@ -332,15 +326,9 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
},
|
},
|
||||||
async execute(args, exec) {
|
async execute(args, exec) {
|
||||||
const sessionId = registry.resolveForStop(args.session);
|
const sessionId = registry.resolveForStop(args.session);
|
||||||
try {
|
await runBsk(deps, exec, ["session", "stop", sessionId], "session stop", sessionId);
|
||||||
const stopped = await deps.observation.stopSession(sessionId, exec.signal);
|
registry.remove(sessionId);
|
||||||
if (!stopped) throw new Error(`browser session ${sessionId} is not owned by this plugin`);
|
deps.observation.removeSession(sessionId);
|
||||||
} catch (error) {
|
|
||||||
if (isCommandNotFound(error)) {
|
|
||||||
throw new Error(bskInstallMessage(deps.config.bskPath));
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
return { stopped: sessionId };
|
return { stopped: sessionId };
|
||||||
},
|
},
|
||||||
presentCall: (args) => ({
|
presentCall: (args) => ({
|
||||||
@@ -497,9 +485,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||||
"probes) with @eN refs. Visual-only surfaces require image understanding and their refs are " +
|
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||||
"screenshot-only; keep using any DOM/AX semantics rendered alongside them. " +
|
|
||||||
"Read-only: never submits input.";
|
|
||||||
register(
|
register(
|
||||||
defineTool({
|
defineTool({
|
||||||
name,
|
name,
|
||||||
@@ -571,8 +557,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
name: "browser_click",
|
name: "browser_click",
|
||||||
description:
|
description:
|
||||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||||
"last browser_snapshot/browser_observe, or a CSS selector. A visual Surface requires the " +
|
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||||
"captureId and image coordinates returned by browser_screenshot.",
|
|
||||||
parameters: {
|
parameters: {
|
||||||
target: {
|
target: {
|
||||||
type: "string",
|
type: "string",
|
||||||
@@ -589,23 +574,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
type: "integer",
|
type: "integer",
|
||||||
description: "Number of consecutive presses (double-click = 2).",
|
description: "Number of consecutive presses (double-click = 2).",
|
||||||
},
|
},
|
||||||
modifiers: {
|
|
||||||
type: "array",
|
|
||||||
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
|
|
||||||
description: "Keyboard modifiers held during the click.",
|
|
||||||
},
|
|
||||||
captureId: {
|
|
||||||
type: "string",
|
|
||||||
description: "Short-lived Surface capture id returned by browser_screenshot.",
|
|
||||||
},
|
|
||||||
imageX: {
|
|
||||||
type: "number",
|
|
||||||
description: "X coordinate in capture-image pixels; requires captureId and imageY.",
|
|
||||||
},
|
|
||||||
imageY: {
|
|
||||||
type: "number",
|
|
||||||
description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
output: {
|
output: {
|
||||||
schema: {
|
schema: {
|
||||||
@@ -616,7 +584,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
tabId: { type: "integer", required: true },
|
tabId: { type: "integer", required: true },
|
||||||
x: { type: "number", required: true },
|
x: { type: "number", required: true },
|
||||||
y: { type: "number", required: true },
|
y: { type: "number", required: true },
|
||||||
captureId: { type: "string" },
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
render: (_args, value) => [
|
render: (_args, value) => [
|
||||||
@@ -628,44 +595,17 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
},
|
},
|
||||||
async execute(args, exec) {
|
async execute(args, exec) {
|
||||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||||
const pointArgCount =
|
|
||||||
Number(args.captureId !== undefined) +
|
|
||||||
Number(args.imageX !== undefined) +
|
|
||||||
Number(args.imageY !== undefined);
|
|
||||||
if (pointArgCount !== 0 && pointArgCount !== 3) {
|
|
||||||
throw new Error("captureId, imageX, and imageY must be given together");
|
|
||||||
}
|
|
||||||
const sessionId = registry.resolve(args.session, "browser_click");
|
const sessionId = registry.resolve(args.session, "browser_click");
|
||||||
const cmdArgs = ["click", "--session", sessionId];
|
const cmdArgs = ["click", "--session", sessionId];
|
||||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
|
||||||
cmdArgs.push("--modifiers", args.modifiers.join(","));
|
|
||||||
}
|
|
||||||
if (args.captureId !== undefined) {
|
|
||||||
cmdArgs.push(
|
|
||||||
"--capture",
|
|
||||||
args.captureId,
|
|
||||||
"--image-x",
|
|
||||||
String(args.imageX),
|
|
||||||
"--image-y",
|
|
||||||
String(args.imageY),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
cmdArgs.push(args.target);
|
cmdArgs.push(args.target);
|
||||||
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
||||||
tab_id: number;
|
tab_id: number;
|
||||||
x: number;
|
x: number;
|
||||||
y: number;
|
y: number;
|
||||||
capture_id?: string;
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
session: sessionId,
|
|
||||||
tabId: reply.tab_id,
|
|
||||||
x: reply.x,
|
|
||||||
y: reply.y,
|
|
||||||
...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
|
|
||||||
};
|
};
|
||||||
|
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||||
},
|
},
|
||||||
presentCall: (args) => ({
|
presentCall: (args) => ({
|
||||||
card: "terminal",
|
card: "terminal",
|
||||||
@@ -808,17 +748,12 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
name: "browser_screenshot",
|
name: "browser_screenshot",
|
||||||
description:
|
description:
|
||||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||||
"This is the supported operation for visual surface refs emitted by browser_observe. Only use " +
|
|
||||||
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
|
|
||||||
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
|
|
||||||
"not guess content or coordinates. " +
|
|
||||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||||
parameters: {
|
parameters: {
|
||||||
session: SESSION_PARAM,
|
session: SESSION_PARAM,
|
||||||
ref: {
|
ref: {
|
||||||
type: "string",
|
type: "string",
|
||||||
description:
|
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||||
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
output: {
|
output: {
|
||||||
@@ -832,8 +767,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
width: { type: "integer", required: true },
|
width: { type: "integer", required: true },
|
||||||
height: { type: "integer", required: true },
|
height: { type: "integer", required: true },
|
||||||
byteSize: { type: "integer", required: true },
|
byteSize: { type: "integer", required: true },
|
||||||
captureId: { type: "string" },
|
|
||||||
captureExpiresAt: { type: "integer" },
|
|
||||||
image: {
|
image: {
|
||||||
type: "object",
|
type: "object",
|
||||||
additionalProperties: false,
|
additionalProperties: false,
|
||||||
@@ -853,11 +786,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
value.image !== undefined
|
value.image !== undefined
|
||||||
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
||||||
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
||||||
const captureText =
|
const blocks: ContentBlock[] = [{ type: "text", text }];
|
||||||
value.captureId !== undefined
|
|
||||||
? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
|
|
||||||
: "";
|
|
||||||
const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
|
|
||||||
if (value.image !== undefined) {
|
if (value.image !== undefined) {
|
||||||
blocks.push({
|
blocks.push({
|
||||||
type: "image",
|
type: "image",
|
||||||
@@ -896,7 +825,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
height: number;
|
height: number;
|
||||||
path: string;
|
path: string;
|
||||||
byte_size: number;
|
byte_size: number;
|
||||||
capture?: { id: string; expires_at: number };
|
|
||||||
};
|
};
|
||||||
writtenPath = reply.path;
|
writtenPath = reply.path;
|
||||||
const data = await readFile(reply.path);
|
const data = await readFile(reply.path);
|
||||||
@@ -910,9 +838,6 @@ export function registerTools(deps: ToolDeps): () => void {
|
|||||||
width: reply.width,
|
width: reply.width,
|
||||||
height: reply.height,
|
height: reply.height,
|
||||||
byteSize: reply.byte_size,
|
byteSize: reply.byte_size,
|
||||||
...(reply.capture !== undefined
|
|
||||||
? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
|
|
||||||
: {}),
|
|
||||||
...(ref !== undefined
|
...(ref !== undefined
|
||||||
? {
|
? {
|
||||||
image: {
|
image: {
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ function fakeRunner(
|
|||||||
if (opts.stopNotFound) {
|
if (opts.stopNotFound) {
|
||||||
return {
|
return {
|
||||||
code: 4,
|
code: 4,
|
||||||
stdout: JSON.stringify({ code: "not_found", message: "no such session" }),
|
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
|
||||||
stderr: "",
|
stderr: "",
|
||||||
timedOut: false,
|
timedOut: false,
|
||||||
aborted: false,
|
aborted: false,
|
||||||
@@ -93,7 +93,7 @@ function fakeRunner(
|
|||||||
if (opts.screenshotNotFound) {
|
if (opts.screenshotNotFound) {
|
||||||
return {
|
return {
|
||||||
code: 4,
|
code: 4,
|
||||||
stdout: JSON.stringify({ code: "not_found", message: "no such session" }),
|
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
|
||||||
stderr: "",
|
stderr: "",
|
||||||
timedOut: false,
|
timedOut: false,
|
||||||
aborted: false,
|
aborted: false,
|
||||||
@@ -224,57 +224,6 @@ describe("state machine", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("thumbnail cadence", () => {
|
describe("thumbnail cadence", () => {
|
||||||
it("gives foreground work priority and resumes with one fresh capture", async () => {
|
|
||||||
const scheduler = fakeScheduler();
|
|
||||||
let finishCapture: ((result: BskRunResult) => void) | undefined;
|
|
||||||
const killed: string[] = [];
|
|
||||||
const runner: FakeRunner = {
|
|
||||||
calls: [],
|
|
||||||
killed,
|
|
||||||
run(args) {
|
|
||||||
if (args[0] !== "screenshot") {
|
|
||||||
return Promise.resolve({
|
|
||||||
code: 0,
|
|
||||||
stdout: "{}",
|
|
||||||
stderr: "",
|
|
||||||
timedOut: false,
|
|
||||||
aborted: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
finishCapture = resolve;
|
|
||||||
});
|
|
||||||
},
|
|
||||||
killAll() {},
|
|
||||||
killFor(tag) {
|
|
||||||
killed.push(tag);
|
|
||||||
if (tag === "observation:s1" && finishCapture !== undefined) {
|
|
||||||
finishCapture({
|
|
||||||
code: 5,
|
|
||||||
stdout: JSON.stringify({ code: "cancelled", message: "cancelled" }),
|
|
||||||
stderr: "",
|
|
||||||
timedOut: false,
|
|
||||||
aborted: false,
|
|
||||||
});
|
|
||||||
finishCapture = undefined;
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const { service } = setup({ runner, scheduler });
|
|
||||||
service.addSession("s1");
|
|
||||||
scheduler.runNext();
|
|
||||||
await waitFor(() => finishCapture !== undefined);
|
|
||||||
|
|
||||||
const release = service.acquireForeground("s1");
|
|
||||||
expect(killed).toEqual(["observation:s1"]);
|
|
||||||
await waitFor(() => scheduler.pending().length === 0);
|
|
||||||
release();
|
|
||||||
expect(service.isAvailable()).toBe(true);
|
|
||||||
expect(scheduler.pending()).toEqual([0]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("captures immediately on add and after action end, then fast-cadences while active", async () => {
|
it("captures immediately on add and after action end, then fast-cadences while active", async () => {
|
||||||
const scheduler = fakeScheduler();
|
const scheduler = fakeScheduler();
|
||||||
const runner = fakeRunner();
|
const runner = fakeRunner();
|
||||||
@@ -402,7 +351,7 @@ describe("stopSession", () => {
|
|||||||
expect(registry.isOwned("s1")).toBe(true);
|
expect(registry.isOwned("s1")).toBe(true);
|
||||||
|
|
||||||
await expect(service.stopSession("s1")).resolves.toBe(true);
|
await expect(service.stopSession("s1")).resolves.toBe(true);
|
||||||
expect(runner.killed).toEqual(["observation:s1", "s1"]);
|
expect(runner.killed).toEqual(["s1"]);
|
||||||
const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
|
const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
|
||||||
expect(stop?.args).toEqual(["session", "stop", "s1"]);
|
expect(stop?.args).toEqual(["session", "stop", "s1"]);
|
||||||
expect(registry.isOwned("s1")).toBe(false);
|
expect(registry.isOwned("s1")).toBe(false);
|
||||||
@@ -431,13 +380,13 @@ describe("stopSession", () => {
|
|||||||
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
|
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects and keeps the entry when the stop itself fails", async () => {
|
it("returns false and keeps the entry when the stop itself fails", async () => {
|
||||||
const registry = new SessionRegistry(5);
|
const registry = new SessionRegistry(5);
|
||||||
own(registry, "s1");
|
own(registry, "s1");
|
||||||
const runner = fakeRunner({ stopFails: true });
|
const runner = fakeRunner({ stopFails: true });
|
||||||
const { service } = setup({ registry, runner });
|
const { service } = setup({ registry, runner });
|
||||||
service.addSession("s1");
|
service.addSession("s1");
|
||||||
await expect(service.stopSession("s1")).rejects.toThrow(/boom/);
|
await expect(service.stopSession("s1")).resolves.toBe(false);
|
||||||
expect(registry.isOwned("s1")).toBe(true);
|
expect(registry.isOwned("s1")).toBe(true);
|
||||||
expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
|
expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
|
||||||
});
|
});
|
||||||
@@ -641,7 +590,7 @@ describe("HTTP/SSE interface", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("availability and missing sessions", () => {
|
describe("availability and dead sessions", () => {
|
||||||
it("flips availability off after repeated global failures and back on success", async () => {
|
it("flips availability off after repeated global failures and back on success", async () => {
|
||||||
const scheduler = fakeScheduler();
|
const scheduler = fakeScheduler();
|
||||||
const runner = fakeRunner({ screenshotFails: true });
|
const runner = fakeRunner({ screenshotFails: true });
|
||||||
@@ -661,18 +610,23 @@ describe("availability and missing sessions", () => {
|
|||||||
service.dispose();
|
service.dispose();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("removes an owned ghost on not_found and stops asking for frames", async () => {
|
it("marks a session dead on session_not_found and stops asking for frames", async () => {
|
||||||
const scheduler = fakeScheduler();
|
const scheduler = fakeScheduler();
|
||||||
const runner = fakeRunner({ screenshotNotFound: true });
|
const runner = fakeRunner({ screenshotNotFound: true });
|
||||||
const registry = new SessionRegistry(5);
|
const { service, events } = setup({ runner, scheduler });
|
||||||
own(registry, "s1");
|
|
||||||
const { service, events } = setup({ runner, scheduler, registry });
|
|
||||||
service.addSession("s1");
|
service.addSession("s1");
|
||||||
scheduler.runNext();
|
scheduler.runNext();
|
||||||
await waitFor(() => events.some((e) => e.type === "remove"));
|
await waitFor(() => events.some((e) => e.type === "upsert" && e.session?.dead === true));
|
||||||
expect(service.getState()).toEqual([]);
|
expect(service.getState()[0].dead).toBe(true);
|
||||||
expect(registry.isOwned("s1")).toBe(false);
|
expect(service.getState()[0].action).toBe("idle");
|
||||||
|
// No further captures are scheduled for a dead session.
|
||||||
expect(scheduler.pending()).toEqual([]);
|
expect(scheduler.pending()).toEqual([]);
|
||||||
|
// Instrumentation is dropped for dead sessions.
|
||||||
|
service.beginAction("s1", "clicking");
|
||||||
|
expect(service.getState()[0].action).toBe("idle");
|
||||||
|
// A dead session leaves cleanly.
|
||||||
|
service.removeSession("s1");
|
||||||
|
expect(service.getState()).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clears lastError on the next action and keeps it on failure", () => {
|
it("clears lastError on the next action and keeps it on failure", () => {
|
||||||
|
|||||||
@@ -1,14 +1,7 @@
|
|||||||
import type { ChildProcess } from "node:child_process";
|
import type { ChildProcess } from "node:child_process";
|
||||||
import { EventEmitter } from "node:events";
|
import { EventEmitter } from "node:events";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import { type BskRunResult, createBskRunner, isCommandNotFound, parseBskJson } from "../src/runner";
|
||||||
type BskRunResult,
|
|
||||||
createBskRunner,
|
|
||||||
isCommandNotFound,
|
|
||||||
isSessionBusyResult,
|
|
||||||
parseBskJson,
|
|
||||||
runWithSessionBusyRetry,
|
|
||||||
} from "../src/runner";
|
|
||||||
|
|
||||||
/** Minimal fake ChildProcess driven by the test. */
|
/** Minimal fake ChildProcess driven by the test. */
|
||||||
class FakeChild extends EventEmitter {
|
class FakeChild extends EventEmitter {
|
||||||
@@ -70,7 +63,7 @@ describe("createBskRunner", () => {
|
|||||||
controller.abort();
|
controller.abort();
|
||||||
const result = await promise;
|
const result = await promise;
|
||||||
expect(result.aborted).toBe(true);
|
expect(result.aborted).toBe(true);
|
||||||
expect(child.killedWith).toContain("SIGINT");
|
expect(child.killedWith).toContain("SIGTERM");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("kills the child on timeout", async () => {
|
it("kills the child on timeout", async () => {
|
||||||
@@ -87,7 +80,7 @@ describe("createBskRunner", () => {
|
|||||||
const promise = runner.run(["session", "list"]);
|
const promise = runner.run(["session", "list"]);
|
||||||
runner.killAll();
|
runner.killAll();
|
||||||
const result = await promise;
|
const result = await promise;
|
||||||
expect(child.killedWith).toContain("SIGINT");
|
expect(child.killedWith).toContain("SIGTERM");
|
||||||
expect(result.code).toBeNull();
|
expect(result.code).toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -147,51 +140,3 @@ describe("isCommandNotFound", () => {
|
|||||||
expect(isCommandNotFound(new Error("other"))).toBe(false);
|
expect(isCommandNotFound(new Error("other"))).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("session busy reconciliation", () => {
|
|
||||||
const busy: BskRunResult = {
|
|
||||||
code: 4,
|
|
||||||
stdout: JSON.stringify({
|
|
||||||
code: "timeout",
|
|
||||||
message: "session already has an unfinished command",
|
|
||||||
data: { reason: "session_busy" },
|
|
||||||
}),
|
|
||||||
stderr: "",
|
|
||||||
timedOut: false,
|
|
||||||
aborted: false,
|
|
||||||
};
|
|
||||||
const ok: BskRunResult = {
|
|
||||||
code: 0,
|
|
||||||
stdout: "{}",
|
|
||||||
stderr: "",
|
|
||||||
timedOut: false,
|
|
||||||
aborted: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
it("recognizes only the structured session_busy reason", () => {
|
|
||||||
expect(isSessionBusyResult(busy)).toBe(true);
|
|
||||||
expect(isSessionBusyResult({ ...busy, stdout: '{"code":"timeout"}' })).toBe(false);
|
|
||||||
expect(isSessionBusyResult(ok)).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("retries one transient busy result and returns the settled response", async () => {
|
|
||||||
const replies = [busy, ok];
|
|
||||||
let calls = 0;
|
|
||||||
const result = await runWithSessionBusyRetry(async () => {
|
|
||||||
calls += 1;
|
|
||||||
return replies.shift() ?? ok;
|
|
||||||
});
|
|
||||||
expect(result).toBe(ok);
|
|
||||||
expect(calls).toBe(2);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not loop on a persistent busy result", async () => {
|
|
||||||
let calls = 0;
|
|
||||||
const result = await runWithSessionBusyRetry(async () => {
|
|
||||||
calls += 1;
|
|
||||||
return busy;
|
|
||||||
});
|
|
||||||
expect(result).toBe(busy);
|
|
||||||
expect(calls).toBe(2);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,11 +1,8 @@
|
|||||||
// Browser skill injection: registration wiring, catalog content, progressive-load
|
// Browser skill injection: registration wiring, catalog content, progressive-load
|
||||||
// weight, and silent degradation without the skill seam.
|
// weight, and silent degradation without the skill seam.
|
||||||
|
|
||||||
import { Context } from "@deepseek-ai/cordis";
|
|
||||||
import { createScope, scopeTarget } from "@deepseek-ai/dsh-scope";
|
|
||||||
import { SkillRegistry } from "@deepseek-ai/dsh-skill";
|
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { armAgentScopedBskSkill, registerBskSkill } from "../src/skill";
|
import { registerBskSkill } from "../src/skill";
|
||||||
|
|
||||||
function fakeCtx(skills?: unknown) {
|
function fakeCtx(skills?: unknown) {
|
||||||
return { get: (key: string) => (key === "skills" ? skills : undefined) } as never;
|
return { get: (key: string) => (key === "skills" ? skills : undefined) } as never;
|
||||||
@@ -103,88 +100,3 @@ describe("registerBskSkill", () => {
|
|||||||
expect(() => disposer()).not.toThrow();
|
expect(() => disposer()).not.toThrow();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("armAgentScopedBskSkill", () => {
|
|
||||||
it("overrides a nearer legacy CLI skill when the DSH agent starts", async () => {
|
|
||||||
const root = new Context();
|
|
||||||
const skillFiber = root.plugin(SkillRegistry);
|
|
||||||
await skillFiber;
|
|
||||||
|
|
||||||
// The plugin's original registration is global.
|
|
||||||
registerBskSkill(skillFiber.ctx);
|
|
||||||
|
|
||||||
// Model the standard preset's filesystem discovery of
|
|
||||||
// ~/.agents/skills/browser-skill. Its nearer scope wins before the fix.
|
|
||||||
const presetKey = { preset: "standard" };
|
|
||||||
const presetScope = createScope(skillFiber.ctx, presetKey);
|
|
||||||
presetScope.ctx.skills.register({
|
|
||||||
name: "browser-skill",
|
|
||||||
description: "Legacy BrowserSkill CLI instructions",
|
|
||||||
content: "Use bsk shell commands.",
|
|
||||||
source: "user-agents",
|
|
||||||
});
|
|
||||||
|
|
||||||
const agentKey = { agent: "test-agent" };
|
|
||||||
const agentScope = createScope(skillFiber.ctx, agentKey, { parent: presetKey });
|
|
||||||
const agent = {
|
|
||||||
id: "test-agent",
|
|
||||||
ctx: agentScope.ctx,
|
|
||||||
} as never;
|
|
||||||
const agents = { list: () => [] };
|
|
||||||
const pluginCtx = skillFiber.ctx.extend({ agents });
|
|
||||||
|
|
||||||
const before = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
|
|
||||||
expect(before?.content).toBe("Use bsk shell commands.");
|
|
||||||
expect(before?.source).toBe("user-agents");
|
|
||||||
|
|
||||||
const disarm = armAgentScopedBskSkill(pluginCtx);
|
|
||||||
pluginCtx.emit(scopeTarget(agent, agentKey), "agent/session-start", {
|
|
||||||
agent,
|
|
||||||
source: "startup",
|
|
||||||
});
|
|
||||||
|
|
||||||
const after = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
|
|
||||||
expect(after?.content).toContain("All browser operations must use the injected tools directly");
|
|
||||||
expect(after?.content).not.toMatch(/\bbsk\b/i);
|
|
||||||
expect(after?.source).toBe("bundled");
|
|
||||||
|
|
||||||
disarm();
|
|
||||||
await agentScope.dispose();
|
|
||||||
await presetScope.dispose();
|
|
||||||
await skillFiber.dispose();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("installs on existing agents for plugin reload and disposes registrations", () => {
|
|
||||||
const registered: Record<string, unknown>[] = [];
|
|
||||||
let disposed = 0;
|
|
||||||
const agent = {
|
|
||||||
ctx: fakeCtx({
|
|
||||||
register(skill: Record<string, unknown>) {
|
|
||||||
registered.push(skill);
|
|
||||||
return () => {
|
|
||||||
disposed += 1;
|
|
||||||
};
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
} as never;
|
|
||||||
const listeners = new Map<string, (payload: { agent: never }) => void>();
|
|
||||||
const ctx = {
|
|
||||||
get: (key: string) => (key === "agents" ? { list: () => [agent] } : undefined),
|
|
||||||
on(event: string, listener: (payload: { agent: never }) => void) {
|
|
||||||
listeners.set(event, listener);
|
|
||||||
return () => listeners.delete(event);
|
|
||||||
},
|
|
||||||
} as never;
|
|
||||||
|
|
||||||
const disarm = armAgentScopedBskSkill(ctx);
|
|
||||||
expect(registered).toHaveLength(1);
|
|
||||||
|
|
||||||
// A later lifecycle notification for the same agent must not duplicate it.
|
|
||||||
listeners.get("agent/session-start")?.({ agent });
|
|
||||||
expect(registered).toHaveLength(1);
|
|
||||||
|
|
||||||
disarm();
|
|
||||||
expect(disposed).toBe(1);
|
|
||||||
expect(listeners.size).toBe(0);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -377,12 +377,12 @@ describe("interaction tools", () => {
|
|||||||
emulate: { tab_id: 7, cleared: false },
|
emulate: { tab_id: 7, cleared: false },
|
||||||
};
|
};
|
||||||
|
|
||||||
it("browser_click maps target, button, click count, and modifiers", async () => {
|
it("browser_click maps target, button, and click count", async () => {
|
||||||
const { tools, calls } = setup(responses);
|
const { tools, calls } = setup(responses);
|
||||||
await startSession(tools);
|
await startSession(tools);
|
||||||
const click = tools.get("browser_click");
|
const click = tools.get("browser_click");
|
||||||
const value = (await click?.execute(
|
const value = (await click?.execute(
|
||||||
{ target: "@e1", button: "right", clickCount: 2, modifiers: ["ctrl", "shift"] },
|
{ target: "@e1", button: "right", clickCount: 2 },
|
||||||
makeExec(),
|
makeExec(),
|
||||||
)) as { session: string; x: number; y: number };
|
)) as { session: string; x: number; y: number };
|
||||||
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
|
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
|
||||||
@@ -394,43 +394,6 @@ describe("interaction tools", () => {
|
|||||||
"right",
|
"right",
|
||||||
"--click-count",
|
"--click-count",
|
||||||
"2",
|
"2",
|
||||||
"--modifiers",
|
|
||||||
"ctrl,shift",
|
|
||||||
"@e1",
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("browser_click forwards a complete Surface capture coordinate transaction", async () => {
|
|
||||||
const { tools, calls } = setup({
|
|
||||||
...responses,
|
|
||||||
click: {
|
|
||||||
tab_id: 7,
|
|
||||||
used_ref: "e1",
|
|
||||||
x: 110,
|
|
||||||
y: 70,
|
|
||||||
capture_id: "sc_test",
|
|
||||||
image_x: 200,
|
|
||||||
image_y: 100,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await startSession(tools);
|
|
||||||
const click = tools.get("browser_click");
|
|
||||||
const value = await click?.execute(
|
|
||||||
{ target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
|
|
||||||
makeExec(),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
|
|
||||||
expect(calls[1].args).toEqual([
|
|
||||||
"click",
|
|
||||||
"--session",
|
|
||||||
"s1",
|
|
||||||
"--capture",
|
|
||||||
"sc_test",
|
|
||||||
"--image-x",
|
|
||||||
"200",
|
|
||||||
"--image-y",
|
|
||||||
"100",
|
|
||||||
"@e1",
|
"@e1",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -851,32 +814,6 @@ describe("browser_screenshot", () => {
|
|||||||
expect(rendered?.every((block) => block.type === "text")).toBe(true);
|
expect(rendered?.every((block) => block.type === "text")).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns Surface capture metadata alongside the screenshot", async () => {
|
|
||||||
const path = pngFile();
|
|
||||||
const { tools } = setup({
|
|
||||||
"session start": START_REPLY("s1"),
|
|
||||||
screenshot: {
|
|
||||||
tab_id: 7,
|
|
||||||
width: 800,
|
|
||||||
height: 600,
|
|
||||||
format: "png",
|
|
||||||
path,
|
|
||||||
byte_size: 8,
|
|
||||||
capture: { id: "sc_test", expires_at: 1700000030000 },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await startSession(tools);
|
|
||||||
const screenshot = tools.get("browser_screenshot");
|
|
||||||
const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
|
|
||||||
|
|
||||||
expect(value).toMatchObject({
|
|
||||||
captureId: "sc_test",
|
|
||||||
captureExpiresAt: 1700000030000,
|
|
||||||
});
|
|
||||||
const rendered = screenshot?.output.render({ session: "s1" }, value as never);
|
|
||||||
expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("commits the image through the attachment store on an image-capable route", async () => {
|
it("commits the image through the attachment store on an image-capable route", async () => {
|
||||||
const path = pngFile();
|
const path = pngFile();
|
||||||
const services = {
|
const services = {
|
||||||
|
|||||||
@@ -6,27 +6,27 @@
|
|||||||
"disconnected": "Disconnected",
|
"disconnected": "Disconnected",
|
||||||
"connecting": "Connecting…",
|
"connecting": "Connecting…",
|
||||||
"connected": "Connected",
|
"connected": "Connected",
|
||||||
"version_skew": "Protocol mismatch",
|
"version_skew": "Connected",
|
||||||
"disabled": "Connection off"
|
"disabled": "Connection off"
|
||||||
},
|
},
|
||||||
"stateDetail": {
|
"stateDetail": {
|
||||||
"disconnected": "Open BrowserSkill to connect.",
|
"disconnected": "Open BrowserSkill to connect.",
|
||||||
"connecting": "Connecting to BrowserSkill…",
|
"connecting": "Connecting to BrowserSkill…",
|
||||||
"connected": "Ready for automation tasks.",
|
"connected": "Ready for automation tasks.",
|
||||||
"version_skew": "Some features may be unavailable."
|
"version_skew": "Protocol versions differ. Please upgrade."
|
||||||
},
|
},
|
||||||
"stateBadge": {
|
"stateBadge": {
|
||||||
"disconnected": "Standby",
|
"disconnected": "Standby",
|
||||||
"connecting": "Igniting",
|
"connecting": "Igniting",
|
||||||
"connected": "Ready",
|
"connected": "Ready",
|
||||||
"version_skew": "Action needed",
|
"version_skew": "Upgradable",
|
||||||
"disabled": "Off"
|
"disabled": "Off"
|
||||||
},
|
},
|
||||||
"connectionToggleTitle": "BrowserSkill connection",
|
"connectionToggleTitle": "BrowserSkill connection",
|
||||||
"controlHintsToggleTitle": "Control hints",
|
"controlHintsToggleTitle": "Control hints",
|
||||||
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
||||||
"controlHintsInfoLabel": "About control hints",
|
"controlHintsInfoLabel": "About control hints",
|
||||||
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
|
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, CLI protocol v{{cliProtocol}}. Protocol versions differ — please upgrade.",
|
||||||
"upgradeAvailable": "Upgradable",
|
"upgradeAvailable": "Upgradable",
|
||||||
"launcher": {
|
"launcher": {
|
||||||
"title": "Quick actions"
|
"title": "Quick actions"
|
||||||
|
|||||||
@@ -6,27 +6,27 @@
|
|||||||
"disconnected": "未连接",
|
"disconnected": "未连接",
|
||||||
"connecting": "连接中…",
|
"connecting": "连接中…",
|
||||||
"connected": "已连接",
|
"connected": "已连接",
|
||||||
"version_skew": "协议不一致",
|
"version_skew": "已连接",
|
||||||
"disabled": "连接已关闭"
|
"disabled": "连接已关闭"
|
||||||
},
|
},
|
||||||
"stateDetail": {
|
"stateDetail": {
|
||||||
"disconnected": "请先打开 BrowserSkill。",
|
"disconnected": "请先打开 BrowserSkill。",
|
||||||
"connecting": "正在连接 BrowserSkill…",
|
"connecting": "正在连接 BrowserSkill…",
|
||||||
"connected": "可接收自动化任务。",
|
"connected": "可接收自动化任务。",
|
||||||
"version_skew": "部分能力可能不可用。"
|
"version_skew": "协议版本不同,请及时升级。"
|
||||||
},
|
},
|
||||||
"stateBadge": {
|
"stateBadge": {
|
||||||
"disconnected": "Standby",
|
"disconnected": "Standby",
|
||||||
"connecting": "Igniting",
|
"connecting": "Igniting",
|
||||||
"connected": "Ready",
|
"connected": "Ready",
|
||||||
"version_skew": "Action needed",
|
"version_skew": "可升级",
|
||||||
"disabled": "Off"
|
"disabled": "Off"
|
||||||
},
|
},
|
||||||
"connectionToggleTitle": "BrowserSkill 连接",
|
"connectionToggleTitle": "BrowserSkill 连接",
|
||||||
"controlHintsToggleTitle": "控制提示",
|
"controlHintsToggleTitle": "控制提示",
|
||||||
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
||||||
"controlHintsInfoLabel": "控制提示说明",
|
"controlHintsInfoLabel": "控制提示说明",
|
||||||
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
|
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},CLI 协议 v{{cliProtocol}}。协议版本不同,请及时升级。",
|
||||||
"upgradeAvailable": "可升级",
|
"upgradeAvailable": "可升级",
|
||||||
"launcher": {
|
"launcher": {
|
||||||
"title": "快捷功能"
|
"title": "快捷功能"
|
||||||
|
|||||||
@@ -29,66 +29,6 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("renderVom single-layer page", () => {
|
describe("renderVom single-layer page", () => {
|
||||||
it("renders visual surfaces as screenshot-only refs", () => {
|
|
||||||
const input = scene([
|
|
||||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
|
||||||
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
|
|
||||||
]);
|
|
||||||
input.visualSurfaces = [
|
|
||||||
{
|
|
||||||
parentId: 2,
|
|
||||||
backendNodeId: 99,
|
|
||||||
frameId: "child",
|
|
||||||
renderingKind: "canvas",
|
|
||||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
|
||||||
label: "Data grid",
|
|
||||||
memberCount: 2,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
const out = renderVom(input);
|
|
||||||
|
|
||||||
expect(out.text).toContain(
|
|
||||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
|
||||||
);
|
|
||||||
expect(out.refs).toContainEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
ref: "e1",
|
|
||||||
backendNodeId: 99,
|
|
||||||
kind: "surface",
|
|
||||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("renders an unnamed visual surface with a stable generic label", () => {
|
|
||||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
|
||||||
input.visualSurfaces = [
|
|
||||||
{
|
|
||||||
parentId: 1,
|
|
||||||
backendNodeId: 99,
|
|
||||||
frameId: "main",
|
|
||||||
renderingKind: "canvas",
|
|
||||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
|
||||||
memberCount: 1,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const out = renderVom(input);
|
|
||||||
|
|
||||||
expect(out.text).toContain(
|
|
||||||
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
|
||||||
);
|
|
||||||
expect(out.refs).toContainEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
ref: "e1",
|
|
||||||
name: "canvas visual surface",
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not derive handle context across frame scopes", () => {
|
it("does not derive handle context across frame scopes", () => {
|
||||||
const out = renderVom(
|
const out = renderVom(
|
||||||
scene([
|
scene([
|
||||||
@@ -393,84 +333,6 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not treat implementation CSS classes as control names", () => {
|
|
||||||
const out = renderVom(
|
|
||||||
scene([
|
|
||||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
|
||||||
node({
|
|
||||||
id: 2,
|
|
||||||
parentId: 1,
|
|
||||||
role: "generic",
|
|
||||||
cursor: "pointer",
|
|
||||||
attrs: { onclick: "handleClick()" },
|
|
||||||
rect: { x: 20, y: 20, w: 40, h: 40 },
|
|
||||||
}),
|
|
||||||
node({
|
|
||||||
id: 3,
|
|
||||||
parentId: 2,
|
|
||||||
role: "img",
|
|
||||||
tag: "svg",
|
|
||||||
attrs: { class: "kocomz" },
|
|
||||||
}),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(out.text).not.toContain("kocomz");
|
|
||||||
expect(out.refs).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("recovers icon controls only from explicit semantic icon evidence", () => {
|
|
||||||
const out = renderVom(
|
|
||||||
scene([
|
|
||||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
|
||||||
node({
|
|
||||||
id: 2,
|
|
||||||
parentId: 1,
|
|
||||||
role: "generic",
|
|
||||||
cursor: "pointer",
|
|
||||||
attrs: { onclick: "handleFilter()" },
|
|
||||||
rect: { x: 20, y: 20, w: 40, h: 40 },
|
|
||||||
}),
|
|
||||||
node({
|
|
||||||
id: 3,
|
|
||||||
parentId: 2,
|
|
||||||
role: "img",
|
|
||||||
tag: "svg",
|
|
||||||
attrs: { "aria-label": "Filter rows", class: "kocomz" },
|
|
||||||
}),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Filter rows"');
|
|
||||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps a semantic image as evidence for its clickable wrapper", () => {
|
|
||||||
const out = renderVom(
|
|
||||||
scene([
|
|
||||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
|
||||||
node({
|
|
||||||
id: 2,
|
|
||||||
parentId: 1,
|
|
||||||
role: "generic",
|
|
||||||
cursor: "pointer",
|
|
||||||
rect: { x: 20, y: 20, w: 40, h: 40 },
|
|
||||||
}),
|
|
||||||
node({
|
|
||||||
id: 3,
|
|
||||||
parentId: 2,
|
|
||||||
role: "img",
|
|
||||||
name: "Account",
|
|
||||||
cursor: "pointer",
|
|
||||||
tag: "div",
|
|
||||||
}),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Account"');
|
|
||||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||||
const out = renderVom(
|
const out = renderVom(
|
||||||
scene([
|
scene([
|
||||||
|
|||||||
@@ -1,9 +1,4 @@
|
|||||||
export {
|
export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render";
|
||||||
applyVomInteractionRecovery,
|
|
||||||
isVomReferenceNode,
|
|
||||||
isVomStructuralRole,
|
|
||||||
renderVom,
|
|
||||||
} from "./render";
|
|
||||||
export type {
|
export type {
|
||||||
ActiveScopeBlock,
|
ActiveScopeBlock,
|
||||||
BlockingLayer,
|
BlockingLayer,
|
||||||
@@ -15,8 +10,6 @@ export type {
|
|||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
VomRef,
|
VomRef,
|
||||||
VomRefCapability,
|
|
||||||
VomResult,
|
VomResult,
|
||||||
VomScene,
|
VomScene,
|
||||||
VomVisualSurface,
|
|
||||||
} from "./types";
|
} from "./types";
|
||||||
+17
-107
@@ -9,7 +9,6 @@ import type {
|
|||||||
VomOptions,
|
VomOptions,
|
||||||
VomResult,
|
VomResult,
|
||||||
VomScene,
|
VomScene,
|
||||||
VomVisualSurface,
|
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||||
@@ -76,10 +75,6 @@ const STRUCTURAL_ROLES = new Set([
|
|||||||
"webarea",
|
"webarea",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
export function isVomStructuralRole(role: string | undefined): boolean {
|
|
||||||
return STRUCTURAL_ROLES.has(role?.toLowerCase() ?? "");
|
|
||||||
}
|
|
||||||
|
|
||||||
const SENSITIVE_MASK = "•••";
|
const SENSITIVE_MASK = "•••";
|
||||||
const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY;
|
const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY;
|
||||||
const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY;
|
const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY;
|
||||||
@@ -198,16 +193,13 @@ function shouldRender(node: VomNode): boolean {
|
|||||||
const role = normalizedRole(node);
|
const role = normalizedRole(node);
|
||||||
if (SKIP_ROLES.has(role)) return false;
|
if (SKIP_ROLES.has(role)) return false;
|
||||||
if (INTERACTIVE_ROLES.has(role)) return true;
|
if (INTERACTIVE_ROLES.has(role)) return true;
|
||||||
if (isVomStructuralRole(role)) return true;
|
if (STRUCTURAL_ROLES.has(role)) return true;
|
||||||
|
|
||||||
return cleaned(node.name) !== undefined;
|
return cleaned(node.name) !== undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isVomReferenceNode(node: VomNode): boolean {
|
export function isVomReferenceNode(node: VomNode): boolean {
|
||||||
return (
|
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
|
||||||
node.referenceable !== false &&
|
|
||||||
(INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasRecoveryHandler(node: VomNode): boolean {
|
function hasRecoveryHandler(node: VomNode): boolean {
|
||||||
@@ -227,30 +219,14 @@ function isContentEditable(node: VomNode): boolean {
|
|||||||
return contentEditable === "true" || contentEditable === "plaintext-only";
|
return contentEditable === "true" || contentEditable === "plaintext-only";
|
||||||
}
|
}
|
||||||
|
|
||||||
const ICON_REFERENCE_MARKERS = new Set(["glyph", "icon", "icons", "symbol"]);
|
function iconKeyword(value: string | undefined): string | undefined {
|
||||||
const ICON_REFERENCE_DECORATORS = new Set(["fill", "filled", "line", "outline", "outlined"]);
|
const cleanedValue = cleaned(value);
|
||||||
|
if (!cleanedValue) return undefined;
|
||||||
function iconReferenceName(value: string | undefined): string | undefined {
|
const last = cleanedValue
|
||||||
const reference = cleaned(value);
|
.split(/[#/.\s:_-]+/)
|
||||||
if (!reference) return undefined;
|
|
||||||
const fragment = reference.includes("#") ? reference.slice(reference.lastIndexOf("#") + 1) : "";
|
|
||||||
const tokens = (fragment || reference)
|
|
||||||
?.split(/[#/.\s:_-]+/)
|
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.map((token) => token.toLowerCase());
|
.pop();
|
||||||
if (tokens.length === 0) return undefined;
|
return last && /^[a-z][a-z0-9_-]{1,32}$/i.test(last) ? last.toLowerCase() : undefined;
|
||||||
if (fragment) {
|
|
||||||
const semanticTokens = tokens.filter(
|
|
||||||
(token) => !ICON_REFERENCE_MARKERS.has(token) && !ICON_REFERENCE_DECORATORS.has(token),
|
|
||||||
);
|
|
||||||
return semanticTokens.length > 0 ? semanticTokens.join(" ") : undefined;
|
|
||||||
}
|
|
||||||
const marker = tokens.findLastIndex((token) => ICON_REFERENCE_MARKERS.has(token));
|
|
||||||
if (marker < 0) return undefined;
|
|
||||||
const semanticTokens = tokens
|
|
||||||
.slice(marker + 1)
|
|
||||||
.filter((token) => !ICON_REFERENCE_DECORATORS.has(token));
|
|
||||||
return semanticTokens.length > 0 ? semanticTokens.join(" ") : undefined;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function iconHintFromSubtree(
|
function iconHintFromSubtree(
|
||||||
@@ -262,12 +238,12 @@ function iconHintFromSubtree(
|
|||||||
for (const child of children.get(node.id) ?? []) {
|
for (const child of children.get(node.id) ?? []) {
|
||||||
const attrs = child.attrs ?? {};
|
const attrs = child.attrs ?? {};
|
||||||
const hint =
|
const hint =
|
||||||
cleaned(attrs["aria-label"]) ??
|
iconKeyword(attrs["aria-label"]) ??
|
||||||
cleaned(attrs.title) ??
|
iconKeyword(attrs.title) ??
|
||||||
(normalizedRole(child) === "img" ? cleaned(child.name) : undefined) ??
|
iconKeyword(attrs.href) ??
|
||||||
(["img", "svg", "use"].includes(normalizedTag(child))
|
iconKeyword(attrs["xlink:href"]) ??
|
||||||
? (iconReferenceName(attrs.href) ?? iconReferenceName(attrs["xlink:href"]))
|
iconKeyword(attrs.class) ??
|
||||||
: undefined);
|
iconKeyword(child.text);
|
||||||
if (hint) return hint;
|
if (hint) return hint;
|
||||||
const nested = iconHintFromSubtree(child, children, depth + 1);
|
const nested = iconHintFromSubtree(child, children, depth + 1);
|
||||||
if (nested) return nested;
|
if (nested) return nested;
|
||||||
@@ -293,7 +269,6 @@ function recoveredRole(
|
|||||||
node: VomNode,
|
node: VomNode,
|
||||||
children?: Map<number | null, VomNode[]>,
|
children?: Map<number | null, VomNode[]>,
|
||||||
): string | undefined {
|
): string | undefined {
|
||||||
const currentRole = normalizedRole(node);
|
|
||||||
const explicitRole = node.attrs?.role?.toLowerCase() ?? "";
|
const explicitRole = node.attrs?.role?.toLowerCase() ?? "";
|
||||||
const hasHandler = hasRecoveryHandler(node);
|
const hasHandler = hasRecoveryHandler(node);
|
||||||
const focusable = isFocusable(node);
|
const focusable = isFocusable(node);
|
||||||
@@ -315,9 +290,7 @@ function recoveredRole(
|
|||||||
: "button";
|
: "button";
|
||||||
}
|
}
|
||||||
if (hasHandler && hasName) return "button";
|
if (hasHandler && hasName) return "button";
|
||||||
if ((!currentRole || SKIP_ROLES.has(currentRole)) && node.cursor === "pointer" && hasName) {
|
if (node.cursor === "pointer" && hasName) return "button";
|
||||||
return "button";
|
|
||||||
}
|
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -738,53 +711,6 @@ interface RenderState {
|
|||||||
duplicateRefNames: Set<string>;
|
duplicateRefNames: Set<string>;
|
||||||
surfaceMap: Map<number, CondSurface>;
|
surfaceMap: Map<number, CondSurface>;
|
||||||
scopeMap: Map<number, ActiveScopeBlock>;
|
scopeMap: Map<number, ActiveScopeBlock>;
|
||||||
visualSurfaces: Map<number | null, VomVisualSurface[]>;
|
|
||||||
}
|
|
||||||
|
|
||||||
function groupedByParent<T extends { parentId: number | null }>(
|
|
||||||
items: T[],
|
|
||||||
): Map<number | null, T[]> {
|
|
||||||
const grouped = new Map<number | null, T[]>();
|
|
||||||
for (const item of items) {
|
|
||||||
const siblings = grouped.get(item.parentId) ?? [];
|
|
||||||
siblings.push(item);
|
|
||||||
grouped.set(item.parentId, siblings);
|
|
||||||
}
|
|
||||||
return grouped;
|
|
||||||
}
|
|
||||||
|
|
||||||
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
|
|
||||||
if (state.stopped) return;
|
|
||||||
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
|
|
||||||
if (depth > state.maxDepth) {
|
|
||||||
state.truncated = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const ref = `e${state.nextRef}`;
|
|
||||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
|
||||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
|
||||||
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
|
||||||
const nextTokens = state.tokens + estimateTokens(line);
|
|
||||||
if (nextTokens > state.maxTokens) {
|
|
||||||
state.truncated = true;
|
|
||||||
state.stopped = true;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
state.lines.push(line);
|
|
||||||
state.tokens = nextTokens;
|
|
||||||
state.refs.push({
|
|
||||||
ref,
|
|
||||||
backendNodeId: surface.backendNodeId,
|
|
||||||
frameId: surface.frameId,
|
|
||||||
visibleRect: surface.visibleRect,
|
|
||||||
role: "surface",
|
|
||||||
name: label,
|
|
||||||
kind: "surface",
|
|
||||||
capabilities: ["screenshot"],
|
|
||||||
line: state.lines.length - 1,
|
|
||||||
});
|
|
||||||
state.nextRef += 1;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
||||||
@@ -860,8 +786,6 @@ function renderTree(
|
|||||||
...(node.role ? { role: node.role } : {}),
|
...(node.role ? { role: node.role } : {}),
|
||||||
...(name ? { name } : {}),
|
...(name ? { name } : {}),
|
||||||
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||||
kind: "dom",
|
|
||||||
capabilities: ["interact", "screenshot"],
|
|
||||||
line: state.lines.length - 1,
|
line: state.lines.length - 1,
|
||||||
});
|
});
|
||||||
state.nextRef += 1;
|
state.nextRef += 1;
|
||||||
@@ -871,12 +795,10 @@ function renderTree(
|
|||||||
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
||||||
|
|
||||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||||
emitVisualEntries(node.id, depth + 1, state);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
renderTree(children, node.id, depth + 1, state);
|
renderTree(children, node.id, depth + 1, state);
|
||||||
}
|
}
|
||||||
emitVisualEntries(parentId, depth, state);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderNodes(
|
function renderNodes(
|
||||||
@@ -885,7 +807,6 @@ function renderNodes(
|
|||||||
initialLines: string[],
|
initialLines: string[],
|
||||||
surfaces: CondSurface[] = [],
|
surfaces: CondSurface[] = [],
|
||||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||||
visualSurfaces: VomVisualSurface[] = [],
|
|
||||||
): RenderState {
|
): RenderState {
|
||||||
const children = buildChildren(nodes);
|
const children = buildChildren(nodes);
|
||||||
const state: RenderState = {
|
const state: RenderState = {
|
||||||
@@ -904,7 +825,6 @@ function renderNodes(
|
|||||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||||
visualSurfaces: groupedByParent(visualSurfaces),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
renderTree(children, null, 1, state);
|
renderTree(children, null, 1, state);
|
||||||
@@ -1110,16 +1030,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
|||||||
"@layers 2 focus=L1",
|
"@layers 2 focus=L1",
|
||||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||||
];
|
];
|
||||||
const visibleSurface = (parentId: number | null): boolean =>
|
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
|
||||||
parentId === null || included.has(parentId);
|
|
||||||
const state = renderNodes(
|
|
||||||
visibleNodes,
|
|
||||||
options,
|
|
||||||
header,
|
|
||||||
scene.surfaces,
|
|
||||||
scene.activeScopeBlocks,
|
|
||||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
|
||||||
);
|
|
||||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -1147,7 +1058,6 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
|||||||
],
|
],
|
||||||
scene.surfaces,
|
scene.surfaces,
|
||||||
scene.activeScopeBlocks,
|
scene.activeScopeBlocks,
|
||||||
scene.visualSurfaces,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -24,8 +24,6 @@ export interface VomNode {
|
|||||||
backendNodeId?: number;
|
backendNodeId?: number;
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
contextScopeId?: string;
|
contextScopeId?: string;
|
||||||
/** False for semantic-only nodes that cannot be addressed through CDP. */
|
|
||||||
referenceable?: boolean;
|
|
||||||
|
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
@@ -61,8 +59,6 @@ export interface VomScene {
|
|||||||
rootFrameId?: string;
|
rootFrameId?: string;
|
||||||
surfaces?: CondSurface[];
|
surfaces?: CondSurface[];
|
||||||
activeScopeBlocks?: ActiveScopeBlock[];
|
activeScopeBlocks?: ActiveScopeBlock[];
|
||||||
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
|
|
||||||
visualSurfaces?: VomVisualSurface[];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface VomOptions {
|
export interface VomOptions {
|
||||||
@@ -96,22 +92,6 @@ export interface VomRef {
|
|||||||
ref: string;
|
ref: string;
|
||||||
backendNodeId: number;
|
backendNodeId: number;
|
||||||
frameId?: string;
|
frameId?: string;
|
||||||
/** Observation-time top-viewport crop for screenshot-only surfaces. */
|
|
||||||
visibleRect?: Rect;
|
|
||||||
kind?: "dom" | "surface";
|
|
||||||
capabilities?: VomRefCapability[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export type VomRefCapability = "interact" | "screenshot";
|
|
||||||
|
|
||||||
export interface VomVisualSurface {
|
|
||||||
parentId: number | null;
|
|
||||||
backendNodeId: number;
|
|
||||||
frameId: string;
|
|
||||||
renderingKind: "canvas";
|
|
||||||
visibleRect: Rect;
|
|
||||||
label?: string;
|
|
||||||
memberCount: number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RenderedRef extends VomRef {
|
export interface RenderedRef extends VomRef {
|
||||||
|
|||||||
Generated
-6
@@ -114,12 +114,6 @@ importers:
|
|||||||
'@deepseek-ai/dsh-llm':
|
'@deepseek-ai/dsh-llm':
|
||||||
specifier: ^0.1.0-rc.6
|
specifier: ^0.1.0-rc.6
|
||||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
|
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
|
||||||
'@deepseek-ai/dsh-scope':
|
|
||||||
specifier: ^0.1.0-rc.6
|
|
||||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))
|
|
||||||
'@deepseek-ai/dsh-skill':
|
|
||||||
specifier: ^0.1.0-rc.6
|
|
||||||
version: 0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-llm@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-attachment@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-brand@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))(@deepseek-ai/dsh-timeout@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1))))(@deepseek-ai/dsh-scope@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)(@deepseek-ai/dsh-invariants@0.1.0-rc.6(@deepseek-ai/cordis@4.0.1)))
|
|
||||||
'@deepseek-ai/dsh-tools':
|
'@deepseek-ai/dsh-tools':
|
||||||
specifier: ^0.1.0-rc.6
|
specifier: ^0.1.0-rc.6
|
||||||
version: 0.1.0-rc.6(f8724372086ccc1457fc84e7becee2e0)
|
version: 0.1.0-rc.6(f8724372086ccc1457fc84e7becee2e0)
|
||||||
|
|||||||
+4
-14
@@ -29,7 +29,7 @@ Drive the user's **real Chromium browser** (with their logins and cookies) throu
|
|||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||||
|
|
||||||
## Mandatory workflow
|
## Mandatory workflow
|
||||||
@@ -80,16 +80,6 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
|||||||
|
|
||||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||||
|
|
||||||
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
|
||||||
|
|
||||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
|
||||||
```
|
|
||||||
|
|
||||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||||
@@ -184,7 +174,7 @@ bsk emulate --session <id> --off
|
|||||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
|
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||||
|
|
||||||
### Console & network debugging (read-only; require `--session`)
|
### Console & network debugging (read-only; require `--session`)
|
||||||
|
|
||||||
@@ -210,7 +200,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
|||||||
|
|
||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||||
@@ -309,7 +299,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
|||||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||||
|
|
||||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user