mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
435a6bb9be |
No files matched your search
@@ -13,8 +13,6 @@ describe("RefStore", () => {
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
generation: 0,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
expect(s.size()).toBe(1);
|
||||
expect(s.isEmpty()).toBe(false);
|
||||
@@ -50,31 +48,4 @@ describe("RefStore", () => {
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit capabilities for visual surface refs", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
});
|
||||
|
||||
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, { tabId: 7, kind: "surface" });
|
||||
s.set("e2", 43, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
||||
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
|
||||
});
|
||||
});
|
||||
@@ -1,67 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
|
||||
|
||||
function input(): SurfaceCaptureInput {
|
||||
return {
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "main",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: 2,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 100,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
};
|
||||
}
|
||||
|
||||
describe("SurfaceCaptureStore", () => {
|
||||
it("creates metadata-only, short-lived captures and consumes them once", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 500,
|
||||
createId: () => "sc_test",
|
||||
});
|
||||
const capture = store.create(input());
|
||||
|
||||
expect(capture).toMatchObject({
|
||||
id: "sc_test",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 1_500,
|
||||
consumed: false,
|
||||
});
|
||||
expect(capture).not.toHaveProperty("imageBase64");
|
||||
expect(store.consume("sc_test")).toMatchObject({ ok: true });
|
||||
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
|
||||
|
||||
now = 1_600;
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects expired and unknown captures", () => {
|
||||
let now = 1_000;
|
||||
const store = new SurfaceCaptureStore({
|
||||
now: () => now,
|
||||
ttlMs: 10,
|
||||
createId: () => "sc_expired",
|
||||
});
|
||||
store.create(input());
|
||||
now = 1_010;
|
||||
|
||||
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
|
||||
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
|
||||
});
|
||||
|
||||
it("clears every capture on session cleanup", () => {
|
||||
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
|
||||
store.create(input());
|
||||
store.clear();
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,10 @@
|
||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||
import { RefStore } from "./ref-store";
|
||||
import { SurfaceCaptureStore } from "./surface-capture-store";
|
||||
|
||||
export interface SessionContext {
|
||||
sessionId: string;
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
surfaceCaptures: SurfaceCaptureStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
/**
|
||||
* Tabs explicitly claimed by the agent because it created them. This
|
||||
@@ -212,7 +210,6 @@ export class SessionManager {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
|
||||
borrowedTabs: new Map(),
|
||||
// The home tab is the session's first explicit claim. Every other
|
||||
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
@@ -14,17 +12,12 @@ import type { Rect } from "@browser-skill/vom";
|
||||
* inside the `SessionContext`.
|
||||
*/
|
||||
export type BackendNodeId = number;
|
||||
export type RefCapability = "interact" | "screenshot";
|
||||
export type RefTargetKind = "dom" | "surface";
|
||||
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
@@ -35,19 +28,8 @@ export type RefInput =
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
};
|
||||
|
||||
function capabilitiesFor(
|
||||
kind: RefTargetKind,
|
||||
capabilities: RefCapability[] | undefined,
|
||||
): RefCapability[] {
|
||||
if (kind === "surface") return ["screenshot"];
|
||||
return capabilities ?? ["interact", "screenshot"];
|
||||
}
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
private generation = 0;
|
||||
@@ -88,20 +70,13 @@ export class RefStore {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind?: RefTargetKind;
|
||||
capabilities?: RefCapability[];
|
||||
} = {},
|
||||
): void {
|
||||
const kind = opts.kind ?? "dom";
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, opts.capabilities),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -119,20 +94,14 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input,
|
||||
tabId: null,
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
const kind = input.kind ?? "dom";
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
|
||||
kind,
|
||||
capabilities: capabilitiesFor(kind, input.capabilities),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export const SURFACE_CAPTURE_TTL_MS = 30_000;
|
||||
|
||||
export interface SurfaceCapture {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
tabId: number;
|
||||
navigationIdentity: string;
|
||||
surface: {
|
||||
ref: string;
|
||||
frameId?: string;
|
||||
backendNodeId: number;
|
||||
observationGeneration: number;
|
||||
};
|
||||
topViewportRect: Rect;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
createdAt: number;
|
||||
expiresAt: number;
|
||||
consumed: boolean;
|
||||
}
|
||||
|
||||
export type SurfaceCaptureInput = Omit<
|
||||
SurfaceCapture,
|
||||
"id" | "createdAt" | "expiresAt" | "consumed"
|
||||
>;
|
||||
|
||||
export type SurfaceCaptureConsumeResult =
|
||||
| { ok: true; capture: SurfaceCapture }
|
||||
| { ok: false; reason: "not_found" | "expired" | "consumed" };
|
||||
|
||||
export interface SurfaceCaptureStoreOptions {
|
||||
now?: () => number;
|
||||
ttlMs?: number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
function randomCaptureId(): string {
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(12));
|
||||
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
||||
}
|
||||
|
||||
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
|
||||
export class SurfaceCaptureStore {
|
||||
private readonly captures = new Map<string, SurfaceCapture>();
|
||||
private readonly now: () => number;
|
||||
private readonly ttlMs: number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(options: SurfaceCaptureStoreOptions = {}) {
|
||||
this.now = options.now ?? Date.now;
|
||||
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
|
||||
this.createId = options.createId ?? randomCaptureId;
|
||||
}
|
||||
|
||||
create(input: SurfaceCaptureInput): SurfaceCapture {
|
||||
this.purgeExpired();
|
||||
const createdAt = this.now();
|
||||
const capture: SurfaceCapture = {
|
||||
...input,
|
||||
id: this.createId(),
|
||||
createdAt,
|
||||
expiresAt: createdAt + this.ttlMs,
|
||||
consumed: false,
|
||||
};
|
||||
this.captures.set(capture.id, capture);
|
||||
return capture;
|
||||
}
|
||||
|
||||
consume(id: string): SurfaceCaptureConsumeResult {
|
||||
const capture = this.captures.get(id);
|
||||
if (!capture) return { ok: false, reason: "not_found" };
|
||||
if (capture.expiresAt <= this.now()) {
|
||||
this.captures.delete(id);
|
||||
return { ok: false, reason: "expired" };
|
||||
}
|
||||
if (capture.consumed) return { ok: false, reason: "consumed" };
|
||||
capture.consumed = true;
|
||||
return { ok: true, capture };
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.captures.clear();
|
||||
}
|
||||
|
||||
size(): number {
|
||||
this.purgeExpired();
|
||||
return this.captures.size;
|
||||
}
|
||||
|
||||
private purgeExpired(): void {
|
||||
const now = this.now();
|
||||
for (const [id, capture] of this.captures) {
|
||||
if (capture.expiresAt <= now) this.captures.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
parseKeySpec,
|
||||
resolveKeyDescriptor,
|
||||
} from "../interaction";
|
||||
import { captureSurfaceEnvironment } from "../surface-target";
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
@@ -55,61 +54,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
return { cdp, tabsApi, sent };
|
||||
}
|
||||
|
||||
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Page.getFrameTree": () => ({
|
||||
frameTree: {
|
||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
||||
},
|
||||
}),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
scale: 1,
|
||||
},
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { overlayHostPresent: false, overlayHostConnected: false } },
|
||||
}),
|
||||
"Input.dispatchMouseEvent": (params) => {
|
||||
onMouse?.(params as Record<string, unknown>);
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const environment = await captureSurfaceEnvironment(fake.cdp, 4, undefined);
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing Surface ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
return { manager, context, fake, capture };
|
||||
}
|
||||
|
||||
describe("modifiersBitfield", () => {
|
||||
it("matches CDP's expected bit layout", () => {
|
||||
expect(modifiersBitfield([])).toBe(0);
|
||||
@@ -179,29 +123,6 @@ describe("handleClick", () => {
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||
});
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -246,74 +167,6 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
"left",
|
||||
"middle",
|
||||
"right",
|
||||
] as const)("uses the common click executor for a %s-button Surface click", async (button) => {
|
||||
const { manager, fake, capture } = await makeSurfaceClickFixture();
|
||||
const result = await handleClick(
|
||||
manager,
|
||||
{
|
||||
session_id: "aa11",
|
||||
ref: "@e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
button,
|
||||
click_count: 2,
|
||||
modifiers: ["ctrl", "shift"],
|
||||
},
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
used_ref: "e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
x: 35,
|
||||
y: 30,
|
||||
});
|
||||
const mouse = fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent");
|
||||
expect(mouse).toHaveLength(3);
|
||||
expect(mouse[1].params).toMatchObject({
|
||||
type: "mousePressed",
|
||||
button,
|
||||
clickCount: 2,
|
||||
modifiers: 2 | 8,
|
||||
});
|
||||
expect(mouse[2].params).toMatchObject({
|
||||
type: "mouseReleased",
|
||||
button,
|
||||
clickCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("consumes a Surface capture when the common click executor fails", async () => {
|
||||
let mouseCalls = 0;
|
||||
const { manager, fake, capture } = await makeSurfaceClickFixture((params) => {
|
||||
mouseCalls += 1;
|
||||
if (params.type === "mousePressed") throw new Error("input failed");
|
||||
});
|
||||
const params = {
|
||||
session_id: "aa11",
|
||||
ref: "@e3",
|
||||
capture_id: capture.id,
|
||||
image_x: 50,
|
||||
image_y: 20,
|
||||
};
|
||||
|
||||
expect(
|
||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
||||
).toMatchObject({ code: "cdp_failed" });
|
||||
expect(mouseCalls).toBe(2);
|
||||
expect(
|
||||
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
|
||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { dispatchMouseClick } from "../mouse-input";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
function fakeCdp(onSend?: (method: string, params: Record<string, unknown>) => void): {
|
||||
cdp: CdpRunner;
|
||||
events: Record<string, unknown>[];
|
||||
} {
|
||||
const events: Record<string, unknown>[] = [];
|
||||
return {
|
||||
events,
|
||||
cdp: {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
const event = params as Record<string, unknown>;
|
||||
events.push(event);
|
||||
onSend?.(method, event);
|
||||
return {} as never;
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("dispatchMouseClick", () => {
|
||||
it("emits one move, press, and release sequence", async () => {
|
||||
const { cdp, events } = fakeCdp();
|
||||
|
||||
expect(
|
||||
await dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 12, y: 34 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
),
|
||||
).toBe("completed");
|
||||
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "mouseMoved",
|
||||
x: 12,
|
||||
y: 34,
|
||||
modifiers: 0,
|
||||
},
|
||||
{
|
||||
type: "mousePressed",
|
||||
x: 12,
|
||||
y: 34,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
},
|
||||
{
|
||||
type: "mouseReleased",
|
||||
x: 12,
|
||||
y: 34,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("releases a pressed button when cancellation arrives", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, events } = fakeCdp((_method, event) => {
|
||||
if (event.type === "mousePressed") controller.abort();
|
||||
});
|
||||
|
||||
expect(
|
||||
await dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 1, y: 2 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
signal: controller.signal,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
),
|
||||
).toBe("cancelled");
|
||||
expect(events.at(-1)).toMatchObject({ type: "mouseReleased" });
|
||||
});
|
||||
|
||||
it("settles pointer-move hit testing before pressing", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const { cdp, events } = fakeCdp();
|
||||
const click = dispatchMouseClick(
|
||||
cdp,
|
||||
4,
|
||||
{ x: 1, y: 2 },
|
||||
{
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
modifiers: 0,
|
||||
moveSettleMs: 32,
|
||||
},
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(31);
|
||||
expect(events.map((event) => event.type)).toEqual(["mouseMoved"]);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await click;
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"mouseMoved",
|
||||
"mousePressed",
|
||||
"mouseReleased",
|
||||
]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -36,14 +36,6 @@ function fakeAgentWindow(ids: number[]) {
|
||||
const TINY_PNG =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABh6FO1AAAAABJRU5ErkJggg==";
|
||||
|
||||
function pngWithIhdrDimensions(width: number, height: number): string {
|
||||
const bytes = Uint8Array.from(atob(TINY_PNG), (character) => character.charCodeAt(0));
|
||||
const view = new DataView(bytes.buffer);
|
||||
view.setUint32(16, width);
|
||||
view.setUint32(20, height);
|
||||
return btoa(String.fromCharCode(...bytes));
|
||||
}
|
||||
|
||||
function makeScreenshotDeps(
|
||||
opts: {
|
||||
cdp?: CdpRunner;
|
||||
@@ -75,9 +67,6 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler && method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -97,7 +86,6 @@ describe("stripDataUrlPrefix", () => {
|
||||
describe("parsePngDimensions", () => {
|
||||
it("parses width/height from the IHDR chunk", () => {
|
||||
expect(parsePngDimensions(TINY_PNG)).toEqual({ width: 1, height: 1 });
|
||||
expect(parsePngDimensions(pngWithIhdrDimensions(100, 80))).toEqual({ width: 100, height: 80 });
|
||||
});
|
||||
it("returns null on non-PNG input", () => {
|
||||
expect(parsePngDimensions("not-a-png-payload-just-random-base64-text-zzzzzzzzz")).toBeNull();
|
||||
@@ -132,20 +120,6 @@ describe("handleScreenshot", () => {
|
||||
expect(get).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails closed when a full-tab capture is not a valid PNG", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
makeScreenshotDeps({
|
||||
captureVisibleTab: vi.fn(async () => "data:image/png;base64,bm90LWEtcG5n"),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cdp_failed", message: /invalid PNG data/ });
|
||||
});
|
||||
|
||||
it("returns not_found when Agent Window has no active tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -326,171 +300,6 @@ describe("handleScreenshot", () => {
|
||||
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
|
||||
});
|
||||
|
||||
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { width: number; height: number; scale: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
|
||||
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
|
||||
expect(res.capture).toMatchObject({
|
||||
surface_ref: "@e5",
|
||||
coordinate_space: "capture-image-pixel",
|
||||
});
|
||||
expect(ctx.surfaceCaptures.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("crops a visual surface screenshot to its observation-time visible region", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
||||
"Page.captureScreenshot": () => ({ data: pngWithIhdrDimensions(100, 80) }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
||||
expect(res).toMatchObject({ width: 100, height: 80 });
|
||||
expect(res.capture?.id).toMatch(/^sc_/);
|
||||
});
|
||||
|
||||
it("routes OOPIF surface geometry through its child session and captures top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "surface",
|
||||
visibleRect: { x: 224, y: 346, w: 200, h: 80 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
|
||||
}),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 7 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("fails closed when a ref capture is not a valid PNG", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" });
|
||||
const { cdp } = makeFakeCdp({
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }),
|
||||
"Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cdp_failed", message: /invalid PNG data/ });
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -3024,85 +2833,6 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Canvas app" },
|
||||
backendDOMNodeId: 100,
|
||||
};
|
||||
const strings = ["body", "canvas", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("canvas")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 30, 800, 500],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
conditionalSurfaceProbe: false,
|
||||
};
|
||||
|
||||
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
|
||||
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
|
||||
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 200,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
|
||||
expect(snapshot.text).not.toContain("visual surface");
|
||||
expect(snapshot.ref_count).toBe(0);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
|
||||
@@ -278,23 +278,12 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clears refs and Surface captures before window teardown", async () => {
|
||||
it("clears the RefStore before window teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Insert a fake ref so we can verify clear() ran.
|
||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||
ctx.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 7,
|
||||
navigationIdentity: "navigation",
|
||||
surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 },
|
||||
topViewportRect: { x: 0, y: 0, w: 100, h: 50 },
|
||||
imageWidth: 100,
|
||||
imageHeight: 50,
|
||||
viewportSignature: "viewport",
|
||||
frameProjectionSignature: "frame",
|
||||
});
|
||||
const state: FakeState = {
|
||||
tabs: new Map(),
|
||||
windowsClosed: new Set(),
|
||||
@@ -303,7 +292,6 @@ describe("handleSessionStop with auto-return", () => {
|
||||
const { tabs, windows } = makeApis(state);
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
expect(ctx.surfaceCaptures.size()).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -24,16 +24,10 @@ describe("lookupSnapshotRef", () => {
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -59,8 +53,6 @@ describe("resolveSnapshotRef", () => {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
});
|
||||
|
||||
const expected = {
|
||||
@@ -68,9 +60,6 @@ describe("resolveSnapshotRef", () => {
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "dom" as const,
|
||||
capabilities: ["interact", "screenshot"] as const,
|
||||
generation: 0,
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
@@ -116,29 +105,6 @@ describe("resolveSnapshotRef", () => {
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
generation: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an operation outside the ref's declared capabilities", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
|
||||
code: "permission_denied",
|
||||
message: "ref @e3 does not support interact",
|
||||
data: {
|
||||
reason: "ref_capability_denied",
|
||||
required_capability: "interact",
|
||||
kind: "surface",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,52 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
mapImagePointToViewport,
|
||||
pointInRegion,
|
||||
sameRect,
|
||||
surfaceVisibleRect,
|
||||
} from "../surface-coordinate";
|
||||
|
||||
describe("surface coordinate mapping", () => {
|
||||
it("maps capture image pixels through the actual screenshot dimensions", () => {
|
||||
expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({
|
||||
x: 200,
|
||||
y: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects invalid image coordinates and dimensions", () => {
|
||||
const rect = { x: 0, y: 0, w: 100, h: 50 };
|
||||
expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull();
|
||||
expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull();
|
||||
});
|
||||
|
||||
it("intersects live and observed visible regions and compares them strictly", () => {
|
||||
const clipped = surfaceVisibleRect(
|
||||
{ x: 0, y: 0, width: 200, height: 100 },
|
||||
{ x: 25, y: 30, w: 50, h: 40 },
|
||||
);
|
||||
expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 });
|
||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 25, y: 30, w: 50, h: 40 })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 26, y: 30, w: 50, h: 40 })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("checks the projected visible region rather than only its bounding box", () => {
|
||||
const triangle = [
|
||||
[
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 100, y: 0 },
|
||||
{ x: 0, y: 100 },
|
||||
],
|
||||
];
|
||||
expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true);
|
||||
expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,306 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import {
|
||||
captureSurfaceEnvironment,
|
||||
resolveSurfacePointerTarget,
|
||||
SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
} from "../surface-target";
|
||||
|
||||
function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeBrowser() {
|
||||
let loaderId = "loader-1";
|
||||
let pageY = 0;
|
||||
let quad = [10, 20, 110, 20, 110, 60, 10, 60];
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Page.getFrameTree") {
|
||||
return {
|
||||
frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } },
|
||||
} as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY,
|
||||
scale: 1,
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "DOM.getContentQuads") return { quads: [quad] } as never;
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
return {
|
||||
cdp,
|
||||
sent,
|
||||
setLoaderId: (value: string) => {
|
||||
loaderId = value;
|
||||
},
|
||||
setPageY: (value: number) => {
|
||||
pageY = value;
|
||||
},
|
||||
setQuad: (value: number[]) => {
|
||||
quad = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function setupCapture() {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
const browser = fakeBrowser();
|
||||
const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined);
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
return { manager, context, browser, capture };
|
||||
}
|
||||
|
||||
function pointParams(captureId: string) {
|
||||
return {
|
||||
ref: "@e3",
|
||||
captureId,
|
||||
imageX: 50,
|
||||
imageY: 20,
|
||||
};
|
||||
}
|
||||
|
||||
describe("Surface screenshot-bound pointer target", () => {
|
||||
it("maps one fresh capture coordinate without executing an action", async () => {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
usedRef: "e3",
|
||||
point: { x: 35, y: 30 },
|
||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
capture: { id: capture.id, imageX: 50, imageY: 20 },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
|
||||
const repeated = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(repeated).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_consumed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a new observation generation before sending input", async () => {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
context.refStore.replace([
|
||||
[
|
||||
"e3",
|
||||
{
|
||||
backendNodeId: 99,
|
||||
tabId: 4,
|
||||
kind: "surface" as const,
|
||||
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_stale" },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects navigation, scroll, and visible geometry changes", async () => {
|
||||
for (const mutate of [
|
||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setLoaderId("loader-2"),
|
||||
(browser: ReturnType<typeof fakeBrowser>) => browser.setPageY(20),
|
||||
(browser: ReturnType<typeof fakeBrowser>) =>
|
||||
browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]),
|
||||
]) {
|
||||
const { context, browser, capture } = await setupCapture();
|
||||
mutate(browser);
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
pointParams(capture.id),
|
||||
{ cdp: browser.cdp },
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "surface_capture_stale" },
|
||||
});
|
||||
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("consumes the capture when coordinate validation fails", async () => {
|
||||
const invalid = await setupCapture();
|
||||
const outside = await resolveSurfacePointerTarget(
|
||||
invalid.context,
|
||||
{ tabId: 4 },
|
||||
{ ...pointParams(invalid.capture.id), imageX: 200 },
|
||||
{ cdp: invalid.browser.cdp },
|
||||
);
|
||||
expect(outside).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "surface_coordinate_invalid" },
|
||||
});
|
||||
expect(
|
||||
await resolveSurfacePointerTarget(
|
||||
invalid.context,
|
||||
{ tabId: 4 },
|
||||
pointParams(invalid.capture.id),
|
||||
{ cdp: invalid.browser.cdp },
|
||||
),
|
||||
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
|
||||
});
|
||||
|
||||
it("maps image coordinates through an OOPIF frame projection", async () => {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const context = await manager.start("aa11");
|
||||
context.refStore.set("e3", 99, {
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "surface",
|
||||
visibleRect: { x: 120, y: 110, w: 100, h: 40 },
|
||||
});
|
||||
const graph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 77,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Page.getFrameTree") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
scale: 1,
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never;
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {} as never;
|
||||
throw new Error(`unexpected root CDP call ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never;
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never;
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => graph),
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
const environment = await captureSurfaceEnvironment(cdp, 4, "child");
|
||||
if ("code" in environment) throw new Error(environment.message);
|
||||
const entry = context.refStore.resolveEntry("e3");
|
||||
if (!entry) throw new Error("missing ref");
|
||||
const capture = context.surfaceCaptures.create({
|
||||
sessionId: "aa11",
|
||||
tabId: 4,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: "e3",
|
||||
frameId: "child",
|
||||
backendNodeId: 99,
|
||||
observationGeneration: entry.generation,
|
||||
},
|
||||
topViewportRect: { x: 120, y: 110, w: 100, h: 40 },
|
||||
imageWidth: 200,
|
||||
imageHeight: 80,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
|
||||
const result = await resolveSurfacePointerTarget(
|
||||
context,
|
||||
{ tabId: 4 },
|
||||
{
|
||||
ref: "@e3",
|
||||
captureId: capture.id,
|
||||
imageX: 100,
|
||||
imageY: 40,
|
||||
},
|
||||
{ cdp },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({ point: { x: 170, y: 130 } });
|
||||
expect(sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -32,7 +32,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import { dispatchMouseClick } from "./mouse-input";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -45,7 +44,6 @@ import {
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||
|
||||
export interface InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
@@ -171,7 +169,7 @@ export async function resolveBackendNode(
|
||||
};
|
||||
}
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
@@ -244,25 +242,11 @@ export async function resolveActionTarget(
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface ResolvedClickPointerTarget {
|
||||
point: { x: number; y: number };
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
moveSettleMs: number;
|
||||
capture?: {
|
||||
id: string;
|
||||
imageX: number;
|
||||
imageY: number;
|
||||
};
|
||||
}
|
||||
|
||||
export async function handleClick(
|
||||
manager: SessionManager,
|
||||
params: ClickParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const hasSurfacePointParams =
|
||||
params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined;
|
||||
const ctxOrErr = lookupSession(manager, params, "click");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -272,23 +256,6 @@ export async function handleClick(
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "click");
|
||||
if (denied) return denied;
|
||||
if (hasSurfacePointParams) {
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
const pointerTarget = await resolveSurfacePointerTarget(
|
||||
ctx,
|
||||
target,
|
||||
{
|
||||
ref: params.ref,
|
||||
selector: params.selector,
|
||||
captureId: params.capture_id,
|
||||
imageX: params.image_x,
|
||||
imageY: params.image_y,
|
||||
},
|
||||
{ cdp: deps.cdp, signal: deps.signal },
|
||||
);
|
||||
if (isRpcError(pointerTarget)) return pointerTarget;
|
||||
return executeResolvedClick(ctx, target, pointerTarget, params, deps, dialogCursor);
|
||||
}
|
||||
const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return clickResolvedTarget(ctx, resolved, params, deps);
|
||||
@@ -302,6 +269,11 @@ export async function clickResolvedTarget(
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const { tab: target } = resolved;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
@@ -314,47 +286,20 @@ export async function clickResolvedTarget(
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
return executeResolvedClick(
|
||||
ctx,
|
||||
target,
|
||||
{
|
||||
point: geometry.actionPoint,
|
||||
usedRef: resolved.usedRef,
|
||||
usedSelector: resolved.usedSelector,
|
||||
moveSettleMs: 0,
|
||||
},
|
||||
params,
|
||||
deps,
|
||||
dialogCursor,
|
||||
);
|
||||
}
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
async function executeResolvedClick(
|
||||
ctx: SessionContext,
|
||||
target: ResolvedTargetTab,
|
||||
pointerTarget: ResolvedClickPointerTarget,
|
||||
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
|
||||
deps: InteractionDeps,
|
||||
dialogCursor: number,
|
||||
): Promise<ClickResult | RpcError> {
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
const button: MouseButton = params.button ?? "left";
|
||||
const clickCount = params.click_count ?? 1;
|
||||
if (!Number.isSafeInteger(clickCount) || clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be a positive integer" };
|
||||
if (clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be greater than zero" };
|
||||
}
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const overlayBlocking = await checkOverlayAtPoint(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
pointerTarget.point.x,
|
||||
pointerTarget.point.y,
|
||||
);
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
@@ -366,16 +311,47 @@ async function executeResolvedClick(
|
||||
}
|
||||
|
||||
try {
|
||||
const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, {
|
||||
// Move first so hover state activates, then press → release.
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
modifiers,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
signal: deps.signal,
|
||||
moveSettleMs: pointerTarget.moveSettleMs,
|
||||
});
|
||||
if (dispatch === "cancelled") {
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err);
|
||||
}
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -393,17 +369,10 @@ async function executeResolvedClick(
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: pointerTarget.usedRef,
|
||||
used_selector: pointerTarget.usedSelector,
|
||||
x: pointerTarget.point.x,
|
||||
y: pointerTarget.point.y,
|
||||
...(pointerTarget.capture
|
||||
? {
|
||||
capture_id: pointerTarget.capture.id,
|
||||
image_x: pointerTarget.capture.imageX,
|
||||
image_y: pointerTarget.capture.imageY,
|
||||
}
|
||||
: {}),
|
||||
used_ref: resolved.usedRef,
|
||||
used_selector: resolved.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
import type { MouseButton } from "@/transport/types";
|
||||
import type { Point } from "./geometry";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
interface MouseClickDispatchOptions {
|
||||
button: MouseButton;
|
||||
clickCount: number;
|
||||
modifiers: number;
|
||||
signal?: AbortSignal;
|
||||
moveSettleMs?: number;
|
||||
}
|
||||
|
||||
type MouseClickDispatchResult = "completed" | "cancelled";
|
||||
|
||||
function waitForSettle(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
|
||||
if (ms <= 0) return Promise.resolve(signal?.aborted !== true);
|
||||
if (signal?.aborted) return Promise.resolve(false);
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (completed: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve(completed);
|
||||
};
|
||||
const timer = setTimeout(() => finish(true), ms);
|
||||
const onAbort = () => finish(false);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch one coherent CDP mouse click in top-viewport CSS coordinates.
|
||||
*
|
||||
* Keeping the sequence here prevents DOM and Surface interaction paths from
|
||||
* duplicating cancellation and best-effort release behavior. Callers that
|
||||
* target asynchronously hit-tested content may request a settle interval.
|
||||
*/
|
||||
export async function dispatchMouseClick(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
point: Point,
|
||||
options: MouseClickDispatchOptions,
|
||||
): Promise<MouseClickDispatchResult> {
|
||||
const { button, clickCount, modifiers, signal, moveSettleMs = 0 } = options;
|
||||
let pressed = false;
|
||||
const release = () =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
modifiers,
|
||||
});
|
||||
if (!(await waitForSettle(moveSettleMs, signal))) return "cancelled";
|
||||
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
pressed = true;
|
||||
|
||||
if (signal?.aborted) {
|
||||
await release();
|
||||
pressed = false;
|
||||
return "cancelled";
|
||||
}
|
||||
|
||||
await release();
|
||||
pressed = false;
|
||||
return "completed";
|
||||
} catch (error) {
|
||||
if (pressed) {
|
||||
try {
|
||||
await release();
|
||||
} catch (releaseError) {
|
||||
console.debug("[bsk mouse-input] best-effort mouse release failed", releaseError);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
type Rect,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
@@ -48,8 +47,6 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { surfaceVisibleRect } from "./surface-coordinate";
|
||||
import { captureSurfaceEnvironment } from "./surface-target";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
@@ -65,11 +62,6 @@ import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "./vom/rendered-surfaces";
|
||||
import {
|
||||
buildSemanticGraph,
|
||||
buildSemanticVomScene,
|
||||
@@ -101,9 +93,6 @@ export const normaliseRef = sharedNormaliseRef;
|
||||
// screenshot — `tool.screenshot`
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
|
||||
|
||||
/**
|
||||
* Strip the `data:image/...;base64,` prefix from a Chrome
|
||||
* `captureVisibleTab` dataURL and return the raw base64 payload.
|
||||
@@ -118,8 +107,8 @@ export function stripDataUrlPrefix(dataUrl: string): string {
|
||||
|
||||
/**
|
||||
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
||||
* `null` on malformed input so callers can reject dimensions that are
|
||||
* not authoritative instead of guessing from a CSS-space rectangle.
|
||||
* `null` on any malformed input so callers fall back to `0/0` instead
|
||||
* of throwing.
|
||||
*
|
||||
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
||||
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
||||
@@ -131,9 +120,9 @@ export function parsePngDimensions(base64: string): { width: number; height: num
|
||||
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
||||
const bin = atob(head);
|
||||
if (bin.length < 24) return null;
|
||||
const expectedSignature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
||||
if (expectedSignature.some((byte, index) => bin.charCodeAt(index) !== byte)) return null;
|
||||
if (bin.slice(12, 16) !== "IHDR") return null;
|
||||
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
|
||||
return null;
|
||||
}
|
||||
const u32 = (off: number) =>
|
||||
(bin.charCodeAt(off) << 24) |
|
||||
(bin.charCodeAt(off + 1) << 16) |
|
||||
@@ -196,42 +185,18 @@ async function captureElementScreenshot(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
visualSurface = false,
|
||||
observedVisibleRect?: Rect,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
{ image_base64: string; width: number; height: number; topViewportRect: Rect } | RpcError
|
||||
> {
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: !visualSurface },
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const topViewportRect = surfaceVisibleRect(
|
||||
geometry.topBounds,
|
||||
visualSurface ? observedVisibleRect : undefined,
|
||||
);
|
||||
if (!topViewportRect) {
|
||||
return { code: "permission_denied", message: "surface is no longer visible" };
|
||||
}
|
||||
const rect = {
|
||||
x: topViewportRect.x,
|
||||
y: topViewportRect.y,
|
||||
width: topViewportRect.w,
|
||||
height: topViewportRect.h,
|
||||
};
|
||||
const scale = visualSurface
|
||||
? Math.min(
|
||||
1,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
|
||||
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
|
||||
)
|
||||
: 1;
|
||||
const rect = geometry.topBounds;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
@@ -241,7 +206,7 @@ async function captureElementScreenshot(
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
@@ -249,11 +214,11 @@ async function captureElementScreenshot(
|
||||
if (!image_base64) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64);
|
||||
if (!dims) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
|
||||
}
|
||||
return { image_base64, width: dims.width, height: dims.height, topViewportRect };
|
||||
const dims = parsePngDimensions(image_base64) ?? {
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -348,7 +313,7 @@ export async function handleScreenshot(
|
||||
if (!deps.cdp) {
|
||||
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -368,54 +333,18 @@ export async function handleScreenshot(
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
node.kind === "surface",
|
||||
node.visibleRect,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const capture =
|
||||
node.kind === "surface"
|
||||
? await (async () => {
|
||||
const environment = await captureSurfaceEnvironment(cdp, target.tabId, node.frameId);
|
||||
if (isRpcError(environment)) return environment;
|
||||
return ctx.surfaceCaptures.create({
|
||||
sessionId: ctx.sessionId,
|
||||
tabId: target.tabId,
|
||||
navigationIdentity: environment.navigationIdentity,
|
||||
surface: {
|
||||
ref: node.refKey,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
backendNodeId: node.backendNodeId,
|
||||
observationGeneration: node.generation,
|
||||
},
|
||||
topViewportRect: captured.topViewportRect,
|
||||
imageWidth: captured.width,
|
||||
imageHeight: captured.height,
|
||||
viewportSignature: environment.viewportSignature,
|
||||
frameProjectionSignature: environment.frameProjectionSignature,
|
||||
});
|
||||
})()
|
||||
: null;
|
||||
if (capture && isRpcError(capture)) return capture;
|
||||
return withShotDialogs({
|
||||
image_base64: captured.image_base64,
|
||||
width: captured.width,
|
||||
height: captured.height,
|
||||
format: "png",
|
||||
tab_id: target.tabId,
|
||||
...(capture
|
||||
? {
|
||||
capture: {
|
||||
id: capture.id,
|
||||
surface_ref: `@${capture.surface.ref}`,
|
||||
coordinate_space: "capture-image-pixel" as const,
|
||||
expires_at: capture.expiresAt,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -435,10 +364,7 @@ export async function handleScreenshot(
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = captured;
|
||||
const dims = parsePngDimensions(image_base64);
|
||||
if (!dims) {
|
||||
return { code: "cdp_failed", message: "screenshot capture returned invalid PNG data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||
return withShotDialogs({
|
||||
image_base64,
|
||||
width: dims.width,
|
||||
@@ -1157,7 +1083,6 @@ async function runHoverProbes(
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
renderedSurfaceDiscovery?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
@@ -1206,20 +1131,7 @@ export async function captureVomObservation(
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
const observedScene = options.renderedSurfaceDiscovery
|
||||
? projectRenderedSurfaces(
|
||||
decoratedScene,
|
||||
normalizedDocuments,
|
||||
clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
normalizedDocuments,
|
||||
captured.viewport,
|
||||
captured.excludedBackendNodeIds,
|
||||
),
|
||||
),
|
||||
)
|
||||
: decoratedScene;
|
||||
const rendered = renderVom(observedScene, {
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
@@ -1274,7 +1186,6 @@ async function handleVomObservation(
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
renderedSurfaceDiscovery: toolName === "observe",
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
@@ -1292,9 +1203,6 @@ async function handleVomObservation(
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
|
||||
...(ref.kind ? { kind: ref.kind } : {}),
|
||||
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
|
||||
@@ -271,7 +271,6 @@ export async function handleSessionStop(
|
||||
|
||||
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
||||
ctx.refStore.clear();
|
||||
ctx.surfaceCaptures.clear();
|
||||
|
||||
clearRecordingForSession(params.session_id);
|
||||
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
|
||||
@@ -13,10 +12,6 @@ export interface SnapshotRefLookup {
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
visibleRect?: Rect;
|
||||
kind: RefTargetKind;
|
||||
capabilities: RefCapability[];
|
||||
generation: number;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
@@ -42,10 +37,6 @@ export function lookupSnapshotRef(
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
|
||||
kind: entry.kind,
|
||||
capabilities: entry.capabilities,
|
||||
generation: entry.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -58,7 +49,6 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
requiredCapability?: RefCapability,
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
@@ -68,13 +58,5 @@ export function resolveSnapshotRef(
|
||||
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
|
||||
);
|
||||
}
|
||||
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"ref_capability_denied",
|
||||
`ref ${ref} does not support ${requiredCapability}`,
|
||||
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
|
||||
);
|
||||
}
|
||||
return looked;
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { Point, Region, ViewportRect } from "./geometry";
|
||||
|
||||
const RECT_EPSILON = 0.01;
|
||||
|
||||
export function surfaceVisibleRect(
|
||||
liveRect: ViewportRect,
|
||||
observedRect: Rect | undefined,
|
||||
): Rect | null {
|
||||
const observed = observedRect ?? {
|
||||
x: liveRect.x,
|
||||
y: liveRect.y,
|
||||
w: liveRect.width,
|
||||
h: liveRect.height,
|
||||
};
|
||||
const x = Math.max(liveRect.x, observed.x);
|
||||
const y = Math.max(liveRect.y, observed.y);
|
||||
const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w);
|
||||
const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h);
|
||||
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
|
||||
}
|
||||
|
||||
export function sameRect(a: Rect, b: Rect): boolean {
|
||||
return (
|
||||
Math.abs(a.x - b.x) <= RECT_EPSILON &&
|
||||
Math.abs(a.y - b.y) <= RECT_EPSILON &&
|
||||
Math.abs(a.w - b.w) <= RECT_EPSILON &&
|
||||
Math.abs(a.h - b.h) <= RECT_EPSILON
|
||||
);
|
||||
}
|
||||
|
||||
export function mapImagePointToViewport(
|
||||
rect: Rect,
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
imageX: number,
|
||||
imageY: number,
|
||||
): Point | null {
|
||||
if (
|
||||
!Number.isFinite(imageX) ||
|
||||
!Number.isFinite(imageY) ||
|
||||
!Number.isSafeInteger(imageWidth) ||
|
||||
!Number.isSafeInteger(imageHeight) ||
|
||||
imageWidth <= 0 ||
|
||||
imageHeight <= 0 ||
|
||||
imageX < 0 ||
|
||||
imageY < 0 ||
|
||||
imageX >= imageWidth ||
|
||||
imageY >= imageHeight ||
|
||||
!Number.isFinite(rect.x) ||
|
||||
!Number.isFinite(rect.y) ||
|
||||
!Number.isFinite(rect.w) ||
|
||||
!Number.isFinite(rect.h) ||
|
||||
rect.w <= 0 ||
|
||||
rect.h <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
x: rect.x + (imageX / imageWidth) * rect.w,
|
||||
y: rect.y + (imageY / imageHeight) * rect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function pointInPolygon(point: Point, polygon: Point[]): boolean {
|
||||
for (let index = 0; index < polygon.length; index += 1) {
|
||||
const a = polygon[index];
|
||||
const b = polygon[(index + 1) % polygon.length];
|
||||
const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x);
|
||||
const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9;
|
||||
const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9;
|
||||
if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true;
|
||||
}
|
||||
let inside = false;
|
||||
for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) {
|
||||
const a = polygon[index];
|
||||
const b = polygon[previous];
|
||||
const intersects =
|
||||
a.y > point.y !== b.y > point.y &&
|
||||
point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x;
|
||||
if (intersects) inside = !inside;
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
export function pointInRegion(point: Point, region: Region): boolean {
|
||||
return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon));
|
||||
}
|
||||
@@ -1,260 +0,0 @@
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { type CdpRunner, isRpcError, normaliseRef } from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
mapImagePointToViewport,
|
||||
pointInRegion,
|
||||
sameRect,
|
||||
surfaceVisibleRect,
|
||||
} from "./surface-coordinate";
|
||||
|
||||
export interface SurfaceCaptureEnvironment {
|
||||
navigationIdentity: string;
|
||||
viewportSignature: string;
|
||||
frameProjectionSignature: string;
|
||||
}
|
||||
|
||||
export interface SurfaceTargetResolverDeps {
|
||||
cdp: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface SurfacePointerTargetInput {
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
captureId?: string;
|
||||
imageX?: number;
|
||||
imageY?: number;
|
||||
}
|
||||
|
||||
export interface ResolvedSurfacePointerTarget {
|
||||
point: { x: number; y: number };
|
||||
usedRef: string;
|
||||
moveSettleMs: number;
|
||||
capture: {
|
||||
id: string;
|
||||
imageX: number;
|
||||
imageY: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Canvas-style controls commonly update their pointer hit-test on the next
|
||||
// animation frame. Leave a scheduling boundary after moving and before pressing.
|
||||
export const SURFACE_POINTER_MOVE_SETTLE_MS = 32;
|
||||
|
||||
function stale(message: string): RpcError {
|
||||
return rpcError("permission_denied", "surface_capture_stale", message);
|
||||
}
|
||||
|
||||
function finiteObject(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(finiteObject);
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value)
|
||||
.filter(
|
||||
([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)),
|
||||
)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, entry]) => [key, finiteObject(entry)]),
|
||||
);
|
||||
}
|
||||
|
||||
function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null {
|
||||
if (!frameId) return "top";
|
||||
const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
const path: Array<{
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
targetSessionId?: string;
|
||||
}> = [];
|
||||
const seen = new Set<string>();
|
||||
let current = frames.get(frameId);
|
||||
while (current) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push({
|
||||
frameId: current.frameId,
|
||||
...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}),
|
||||
...(current.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: current.ownerBackendNodeId }
|
||||
: {}),
|
||||
...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}),
|
||||
});
|
||||
if (!current.parentFrameId) return JSON.stringify(path);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function captureSurfaceEnvironment(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameId: string | undefined,
|
||||
): Promise<SurfaceCaptureEnvironment | RpcError> {
|
||||
try {
|
||||
const [tree, metrics, graph] = await Promise.all([
|
||||
cdp.send<{
|
||||
frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } };
|
||||
}>(tabId, "Page.getFrameTree", {}),
|
||||
cdp.send<Record<string, unknown>>(tabId, "Page.getLayoutMetrics", {}),
|
||||
frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null),
|
||||
]);
|
||||
const root = tree.frameTree?.frame;
|
||||
if (!root?.id || !root.loaderId) {
|
||||
return { code: "cdp_failed", message: "could not identify the current navigation" };
|
||||
}
|
||||
const frameProjectionSignature = frameId
|
||||
? graph
|
||||
? framePathSignature(graph, frameId)
|
||||
: null
|
||||
: "top";
|
||||
if (!frameProjectionSignature) {
|
||||
return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` };
|
||||
}
|
||||
return {
|
||||
navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }),
|
||||
viewportSignature: JSON.stringify(
|
||||
finiteObject({
|
||||
cssLayoutViewport: metrics.cssLayoutViewport,
|
||||
cssVisualViewport: metrics.cssVisualViewport,
|
||||
}),
|
||||
),
|
||||
frameProjectionSignature,
|
||||
};
|
||||
} catch (error) {
|
||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function validatePointParams(input: SurfacePointerTargetInput): RpcError | null {
|
||||
if (!input.captureId || typeof input.captureId !== "string") {
|
||||
return { code: "invalid_params", message: "surface point click requires capture_id" };
|
||||
}
|
||||
if (typeof input.imageX !== "number" || !Number.isFinite(input.imageX)) {
|
||||
return { code: "invalid_params", message: "image_x must be a finite number" };
|
||||
}
|
||||
if (typeof input.imageY !== "number" || !Number.isFinite(input.imageY)) {
|
||||
return { code: "invalid_params", message: "image_y must be a finite number" };
|
||||
}
|
||||
if (!input.ref || input.selector) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "surface point click requires one Surface ref and does not accept a selector",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError {
|
||||
const code = reason === "not_found" ? "not_found" : "permission_denied";
|
||||
return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`);
|
||||
}
|
||||
|
||||
function aborted(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted
|
||||
? { code: "cancelled", message: "surface target resolution aborted" }
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Resolve one fresh Surface capture coordinate into a validated top-viewport pointer target. */
|
||||
export async function resolveSurfacePointerTarget(
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number; url?: string },
|
||||
input: SurfacePointerTargetInput,
|
||||
deps: SurfaceTargetResolverDeps,
|
||||
): Promise<ResolvedSurfacePointerTarget | RpcError> {
|
||||
const invalid = validatePointParams(input);
|
||||
if (invalid) return invalid;
|
||||
const earlyAbort = aborted(deps.signal);
|
||||
if (earlyAbort) return earlyAbort;
|
||||
|
||||
const consumed = ctx.surfaceCaptures.consume(input.captureId as string);
|
||||
if (!consumed.ok) return captureConsumeError(consumed.reason);
|
||||
const capture = consumed.capture;
|
||||
const postConsumeAbort = aborted(deps.signal);
|
||||
if (postConsumeAbort) return postConsumeAbort;
|
||||
|
||||
if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) {
|
||||
return stale("surface capture belongs to a different session or tab");
|
||||
}
|
||||
if (normaliseRef(input.ref as string) !== capture.surface.ref) {
|
||||
return stale("surface ref does not match the capture");
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, input.ref as string, target.tabId, "screenshot");
|
||||
if (isRpcError(node)) return stale(node.message);
|
||||
if (
|
||||
node.kind !== "surface" ||
|
||||
node.backendNodeId !== capture.surface.backendNodeId ||
|
||||
node.frameId !== capture.surface.frameId ||
|
||||
node.generation !== capture.surface.observationGeneration
|
||||
) {
|
||||
return stale("surface observation generation or node identity changed");
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
try {
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
} catch (error) {
|
||||
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId);
|
||||
if (isRpcError(environment)) return environment;
|
||||
if (
|
||||
environment.navigationIdentity !== capture.navigationIdentity ||
|
||||
environment.viewportSignature !== capture.viewportSignature ||
|
||||
environment.frameProjectionSignature !== capture.frameProjectionSignature
|
||||
) {
|
||||
return stale("navigation, viewport, zoom, scroll, or frame projection changed");
|
||||
}
|
||||
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
},
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: false },
|
||||
);
|
||||
if (isRpcError(geometry)) return stale(geometry.message);
|
||||
const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect);
|
||||
if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) {
|
||||
return stale("surface visible region changed");
|
||||
}
|
||||
const point = mapImagePointToViewport(
|
||||
capture.topViewportRect,
|
||||
capture.imageWidth,
|
||||
capture.imageHeight,
|
||||
input.imageX as number,
|
||||
input.imageY as number,
|
||||
);
|
||||
if (!point || !pointInRegion(point, geometry.topVisibleRegions)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"surface_coordinate_invalid",
|
||||
"image coordinate is outside the captured Surface",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
point,
|
||||
usedRef: capture.surface.ref,
|
||||
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
|
||||
capture: {
|
||||
id: capture.id,
|
||||
imageX: input.imageX as number,
|
||||
imageY: input.imageY as number,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
import { discoverRenderedSurfaces } from "../rendered-surfaces";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -736,44 +735,6 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||
});
|
||||
|
||||
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
|
||||
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
|
||||
const i = (value: string) => S.indexOf(value);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 100],
|
||||
[10, 10, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1, 2],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const canvas = nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(canvas?.rendered).toBe(true);
|
||||
expect(canvas?.visuallySuppressed).toBe(true);
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
@@ -1189,97 +1150,6 @@ describe("captureViewModel", () => {
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => {
|
||||
const captureAtDpr = async (dpr: number) => {
|
||||
const S = ["html", "body", "iframe", "canvas", "static", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const scale = (values: number[]) => values.map((value) => value * dpr);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [scale([0, 0, 1000, 800]), scale([100, 150, 300, 200])],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("canvas")],
|
||||
backendNodeId: [20, 21],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [scale([0, 0, 300, 200]), scale([20, 30, 120, 60])],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
layoutViewport: { clientWidth: 1000 * dpr, clientHeight: 800 * dpr },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const childNodes = captured.frameNodes?.get("child") ?? [];
|
||||
return discoverRenderedSurfaces(
|
||||
[
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 13,
|
||||
target: { tabId: 4 },
|
||||
domNodes: childNodes,
|
||||
axNodes: [],
|
||||
},
|
||||
],
|
||||
captured.viewport,
|
||||
);
|
||||
};
|
||||
|
||||
const atDpr1 = await captureAtDpr(1);
|
||||
const atDpr2 = await captureAtDpr(2);
|
||||
|
||||
expect(atDpr1).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 21,
|
||||
frameId: "child",
|
||||
visibleRect: { x: 120, y: 180, w: 120, h: 60 },
|
||||
}),
|
||||
]);
|
||||
expect(atDpr2).toEqual(atDpr1);
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
import type { VomScene } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import {
|
||||
clusterRenderedSurfaces,
|
||||
discoverRenderedSurfaces,
|
||||
projectRenderedSurfaces,
|
||||
} from "../rendered-surfaces";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
|
||||
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "canvas",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
localRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
rendered: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function document(
|
||||
domNodes: CapturedNode[],
|
||||
axNodes: SemanticAxNode[] = [],
|
||||
): CapturedFrameDocument<SemanticAxNode> {
|
||||
return {
|
||||
frameId: domNodes[0]?.frameId ?? "main",
|
||||
contextScopeId: domNodes[0]?.frameId ?? "main",
|
||||
target: { tabId: 4 },
|
||||
domNodes,
|
||||
axNodes,
|
||||
};
|
||||
}
|
||||
|
||||
describe("rendered surface discovery", () => {
|
||||
it("finds prominent rendered canvases and excludes hidden descendants", () => {
|
||||
const visible = domNode(2, { parentBackendNodeId: 1 });
|
||||
const hidden = domNode(3, { parentBackendNodeId: 4 });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
visible,
|
||||
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
|
||||
hidden,
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
|
||||
expect(surfaces).toHaveLength(1);
|
||||
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
|
||||
});
|
||||
|
||||
it("keeps tiny canvases as individually addressable exact discoveries", () => {
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("keeps a surface when the canvas only has partial native accessibility content", () => {
|
||||
const axNodes = [
|
||||
{
|
||||
nodeId: "canvas",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "presentation" },
|
||||
childIds: ["button"],
|
||||
frameId: "main",
|
||||
},
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
frameId: "main",
|
||||
},
|
||||
] as SemanticAxNode[];
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
|
||||
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
|
||||
});
|
||||
|
||||
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
|
||||
const canvas = domNode(2, {
|
||||
attrs: {
|
||||
id: "must-not-be-a-label",
|
||||
class: "must-not-be-a-label",
|
||||
"data-testid": "must-not-be-a-label",
|
||||
},
|
||||
});
|
||||
|
||||
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
|
||||
expect.not.objectContaining({
|
||||
label: expect.anything(),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats blank explicit Canvas names as unnamed", () => {
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
|
||||
});
|
||||
|
||||
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
|
||||
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces(
|
||||
[document([owner, canvas])],
|
||||
{ width: 1280, height: 720 },
|
||||
new Set([1]),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("excludes canvases suppressed by a fully transparent ancestor", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 900, h: 600 },
|
||||
visuallySuppressed: true,
|
||||
});
|
||||
const canvas = domNode(2, { parentBackendNodeId: 1 });
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
|
||||
const owner = domNode(1, {
|
||||
tag: "div",
|
||||
rect: { x: 10, y: 20, w: 100, h: 70 },
|
||||
overflowX: "hidden",
|
||||
overflowY: "hidden",
|
||||
});
|
||||
const partial = domNode(2, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 55, y: 30, w: 140, h: 60 },
|
||||
});
|
||||
const fullyClipped = domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 140, y: 30, w: 80, h: 50 },
|
||||
});
|
||||
|
||||
expect(
|
||||
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
|
||||
width: 1280,
|
||||
height: 720,
|
||||
}),
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 2,
|
||||
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rendered surface clustering and projection", () => {
|
||||
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
|
||||
const body = domNode(1, { tag: "body", rect: null, localRect: null });
|
||||
const surfaces = discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
body,
|
||||
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
|
||||
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
const groups = clusterRenderedSurfaces(surfaces);
|
||||
const scene: VomScene = {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [
|
||||
{
|
||||
id: 10,
|
||||
parentId: null,
|
||||
backendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
role: "RootWebArea",
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]?.members).toHaveLength(2);
|
||||
expect(groups[0]?.label).toBeUndefined();
|
||||
expect(
|
||||
projectRenderedSurfaces(
|
||||
scene,
|
||||
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
|
||||
groups,
|
||||
),
|
||||
).toMatchObject({
|
||||
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge a contained canvas with a larger independent canvas", () => {
|
||||
const groups = clusterRenderedSurfaces([
|
||||
...discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "main", rect: null, localRect: null }),
|
||||
domNode(2, { parentBackendNodeId: 1 }),
|
||||
domNode(3, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: 30, y: 40, w: 200, h: 100 },
|
||||
}),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
]);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not merge coincident canvases owned by different containers", () => {
|
||||
const groups = clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
domNode(1, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(2, { tag: "section", rect: null, localRect: null }),
|
||||
domNode(3, { parentBackendNodeId: 1 }),
|
||||
domNode(4, { parentBackendNodeId: 2 }),
|
||||
]),
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -11,15 +11,7 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
] as const;
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
@@ -47,10 +39,6 @@ export interface CapturedNode {
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
overflowX?: string;
|
||||
overflowY?: string;
|
||||
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
|
||||
visuallySuppressed?: boolean;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
@@ -861,8 +849,6 @@ function parseDocumentNodes(
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const overflowXCol = STYLE_COL["overflow-x"];
|
||||
const overflowYCol = STYLE_COL["overflow-y"];
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
@@ -888,7 +874,6 @@ function parseDocumentNodes(
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
@@ -908,11 +893,7 @@ function parseDocumentNodes(
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let overflowX = "visible";
|
||||
let overflowY = "visible";
|
||||
let rendered = false;
|
||||
let visuallySuppressed =
|
||||
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
@@ -935,19 +916,12 @@ function parseDocumentNodes(
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
|
||||
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
|
||||
visuallySuppressed ||=
|
||||
visibility === "hidden" ||
|
||||
visibility === "collapse" ||
|
||||
(Number.parseFloat(opacity) || 0) <= 0;
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
@@ -982,9 +956,6 @@ function parseDocumentNodes(
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
overflowX,
|
||||
overflowY,
|
||||
visuallySuppressed,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
|
||||
const STACK_IOU_RATIO = 0.9;
|
||||
const STACK_SIZE_RATIO = 0.9;
|
||||
|
||||
function area(rect: Rect): number {
|
||||
return Math.max(0, rect.w) * Math.max(0, rect.h);
|
||||
}
|
||||
|
||||
function intersectionArea(a: Rect, b: Rect): number {
|
||||
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
|
||||
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
|
||||
return width * height;
|
||||
}
|
||||
|
||||
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
||||
if (a.frameId !== b.frameId) return false;
|
||||
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
|
||||
const aArea = area(a.visibleRect);
|
||||
const bArea = area(b.visibleRect);
|
||||
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
|
||||
const union = aArea + bArea - intersection;
|
||||
const widthRatio =
|
||||
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
|
||||
const heightRatio =
|
||||
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
|
||||
return (
|
||||
union > 0 &&
|
||||
intersection / union >= STACK_IOU_RATIO &&
|
||||
widthRatio >= STACK_SIZE_RATIO &&
|
||||
heightRatio >= STACK_SIZE_RATIO
|
||||
);
|
||||
}
|
||||
|
||||
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
|
||||
const representative = [...members].sort(
|
||||
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
|
||||
)[0];
|
||||
return {
|
||||
frameId: representative.frameId,
|
||||
members,
|
||||
representative,
|
||||
label: members.map((surface) => surface.label).find((label) => label !== undefined),
|
||||
};
|
||||
}
|
||||
|
||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
|
||||
const groups: RenderedSurface[][] = [];
|
||||
for (const surface of surfaces) {
|
||||
const matching = groups.find((group) =>
|
||||
group.some((member) => sameVisualStack(member, surface)),
|
||||
);
|
||||
if (matching) matching.push(surface);
|
||||
else groups.push([surface]);
|
||||
}
|
||||
return groups
|
||||
.map(toGroup)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
||||
);
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurface } from "./types";
|
||||
|
||||
function clean(value: string | undefined): string | undefined {
|
||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
|
||||
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
function intersectRect(a: Rect, b: Rect): Rect | null {
|
||||
const x = Math.max(a.x, b.x);
|
||||
const y = Math.max(a.y, b.y);
|
||||
const right = Math.min(a.x + a.w, b.x + b.w);
|
||||
const bottom = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, w: right - x, h: bottom - y };
|
||||
}
|
||||
|
||||
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
|
||||
|
||||
function clipToOverflowAncestors(
|
||||
node: CapturedNode,
|
||||
rect: Rect,
|
||||
nodesByBackend: ReadonlyMap<number, CapturedNode>,
|
||||
): Rect | null {
|
||||
let clipped: Rect | null = rect;
|
||||
let parentBackendNodeId = node.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
|
||||
const parent = nodesByBackend.get(parentBackendNodeId);
|
||||
if (!parent) break;
|
||||
if (parent.rect) {
|
||||
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
|
||||
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
|
||||
if (clipX || clipY) {
|
||||
const clipRect = {
|
||||
x: clipX ? parent.rect.x : clipped.x,
|
||||
y: clipY ? parent.rect.y : clipped.y,
|
||||
w: clipX ? parent.rect.w : clipped.w,
|
||||
h: clipY ? parent.rect.h : clipped.h,
|
||||
};
|
||||
clipped = intersectRect(clipped, clipRect);
|
||||
}
|
||||
}
|
||||
parentBackendNodeId = parent.parentBackendNodeId;
|
||||
guard += 1;
|
||||
}
|
||||
return clipped;
|
||||
}
|
||||
|
||||
function excludedByDomPolicy(
|
||||
node: CapturedNode,
|
||||
nodesByBackend: Map<number, CapturedNode>,
|
||||
excludedBackendNodeIds: ReadonlySet<number>,
|
||||
): boolean {
|
||||
let current: CapturedNode | undefined = node;
|
||||
let guard = 0;
|
||||
while (current && guard <= nodesByBackend.size) {
|
||||
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
|
||||
if (current.visuallySuppressed === true) return true;
|
||||
const attrs = current.attrs;
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
|
||||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
|
||||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
current =
|
||||
current.parentBackendNodeId === null
|
||||
? undefined
|
||||
: nodesByBackend.get(current.parentBackendNodeId);
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function discoverRenderedSurfaces(
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
viewport: Viewport,
|
||||
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
|
||||
): RenderedSurface[] {
|
||||
const surfaces: RenderedSurface[] = [];
|
||||
for (const document of documents) {
|
||||
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
|
||||
for (const node of document.domNodes) {
|
||||
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
|
||||
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
|
||||
const viewportRect = intersectViewport(node.rect, viewport);
|
||||
const visibleRect = viewportRect
|
||||
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
|
||||
: null;
|
||||
if (!visibleRect) continue;
|
||||
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
|
||||
surfaces.push({
|
||||
renderingKind: "canvas",
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
parentBackendNodeId: node.parentBackendNodeId,
|
||||
visibleRect,
|
||||
paintOrder: node.paintOrder,
|
||||
...(label ? { label } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return surfaces;
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
export { clusterRenderedSurfaces } from "./cluster";
|
||||
export { discoverRenderedSurfaces } from "./discovery";
|
||||
export { projectRenderedSurfaces } from "./project";
|
||||
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
@@ -1,100 +0,0 @@
|
||||
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurfaceGroup } from "./types";
|
||||
|
||||
function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
interface ProjectionIndex {
|
||||
sceneByBackend: ReadonlyMap<string, number>;
|
||||
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
|
||||
domByFrame: ReadonlyMap<
|
||||
string,
|
||||
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
|
||||
>;
|
||||
rootSceneIdByFrame: ReadonlyMap<string, number>;
|
||||
}
|
||||
|
||||
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
|
||||
const document = index.documentByFrame.get(group.frameId);
|
||||
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
|
||||
let parentBackendNodeId = group.representative.parentBackendNodeId;
|
||||
let guard = 0;
|
||||
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
|
||||
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
|
||||
if (sceneId !== undefined) return sceneId;
|
||||
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
|
||||
guard += 1;
|
||||
}
|
||||
|
||||
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
|
||||
const owner = index.sceneByBackend.get(
|
||||
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
|
||||
);
|
||||
if (owner !== undefined) return owner;
|
||||
}
|
||||
|
||||
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
|
||||
}
|
||||
|
||||
function buildProjectionIndex(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
): ProjectionIndex {
|
||||
const rootSceneIdByFrame = new Map<string, number>();
|
||||
for (const node of scene.nodes) {
|
||||
if (
|
||||
node.frameId &&
|
||||
!rootSceneIdByFrame.has(node.frameId) &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
|
||||
) {
|
||||
rootSceneIdByFrame.set(node.frameId, node.id);
|
||||
}
|
||||
}
|
||||
return {
|
||||
sceneByBackend: new Map(
|
||||
scene.nodes.flatMap((node) =>
|
||||
node.backendNodeId === undefined
|
||||
? []
|
||||
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
|
||||
),
|
||||
),
|
||||
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
|
||||
domByFrame: new Map(
|
||||
documents.map((document) => [
|
||||
document.frameId,
|
||||
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
|
||||
]),
|
||||
),
|
||||
rootSceneIdByFrame,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectRenderedSurfaces(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
groups: RenderedSurfaceGroup[],
|
||||
): VomScene {
|
||||
const index = buildProjectionIndex(scene, documents);
|
||||
const visualSurfaces: VomVisualSurface[] = [];
|
||||
|
||||
for (const group of groups) {
|
||||
const parentId = nearestSceneParent(index, group);
|
||||
visualSurfaces.push({
|
||||
parentId,
|
||||
backendNodeId: group.representative.backendNodeId,
|
||||
frameId: group.frameId,
|
||||
renderingKind: "canvas",
|
||||
visibleRect: group.representative.visibleRect,
|
||||
...(group.label ? { label: group.label } : {}),
|
||||
memberCount: group.members.length,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...scene,
|
||||
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
|
||||
};
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface RenderedSurface {
|
||||
renderingKind: "canvas";
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
visibleRect: Rect;
|
||||
paintOrder: number;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface RenderedSurfaceGroup {
|
||||
frameId: string;
|
||||
members: RenderedSurface[];
|
||||
representative: RenderedSurface;
|
||||
label?: string;
|
||||
}
|
||||
@@ -23,7 +23,6 @@ export type ErrorCode =
|
||||
export type RpcErrorReason =
|
||||
| "agent_window_scope"
|
||||
| "element_not_visible"
|
||||
| "ref_capability_denied"
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
@@ -49,11 +48,6 @@ export type RpcErrorReason =
|
||||
| "download_capture_failed"
|
||||
| "transfer_outcome_unknown"
|
||||
| "transfer_timeout"
|
||||
| "surface_capture_not_found"
|
||||
| "surface_capture_expired"
|
||||
| "surface_capture_consumed"
|
||||
| "surface_capture_stale"
|
||||
| "surface_coordinate_invalid"
|
||||
| "cleanup_failed";
|
||||
|
||||
export type TransferEffectState = "none" | "committed" | "unknown";
|
||||
@@ -325,17 +319,9 @@ export interface ScreenshotResult {
|
||||
height: number;
|
||||
format: string;
|
||||
tab_id: number;
|
||||
capture?: SurfaceCaptureInfo;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface SurfaceCaptureInfo {
|
||||
id: string;
|
||||
surface_ref: string;
|
||||
coordinate_space: "capture-image-pixel";
|
||||
expires_at: number;
|
||||
}
|
||||
|
||||
export interface SnapshotParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
@@ -454,9 +440,6 @@ export interface ClickParams {
|
||||
click_count?: number;
|
||||
modifiers?: KeyModifier[];
|
||||
timeout_ms?: number;
|
||||
capture_id?: string;
|
||||
image_x?: number;
|
||||
image_y?: number;
|
||||
}
|
||||
|
||||
export interface ClickResult {
|
||||
@@ -465,9 +448,6 @@ export interface ClickResult {
|
||||
used_selector?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
capture_id?: string;
|
||||
image_x?: number;
|
||||
image_y?: number;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
# Rendered Surface browser fixtures
|
||||
|
||||
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
|
||||
|
||||
## Serve
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
python3 -m http.server 4173 --bind 127.0.0.1 \
|
||||
--directory apps/extension/test-fixtures/rendered-surfaces
|
||||
```
|
||||
|
||||
Use an Agent Window large enough to keep the primary matrix visible:
|
||||
|
||||
```bash
|
||||
bsk session start --width 1440 --height 1100
|
||||
bsk navigate http://127.0.0.1:4173/index.html --session <id>
|
||||
bsk observe --session <id>
|
||||
```
|
||||
|
||||
Always stop the session after the run.
|
||||
|
||||
## Pages
|
||||
|
||||
### `index.html`
|
||||
|
||||
Expected individual Surface labels:
|
||||
|
||||
- `fixture-basic-large`
|
||||
- `fixture-tiny-8px`
|
||||
- `fixture-pointer-events-none`
|
||||
- `fixture-partial-ax`
|
||||
- `fixture-contained-outer`
|
||||
- `fixture-contained-inner`
|
||||
- `fixture-different-parent-a`
|
||||
- `fixture-different-parent-b`
|
||||
|
||||
Expected one grouped Surface:
|
||||
|
||||
- `fixture-layered-group`, with `layers=2`
|
||||
|
||||
Expected absent:
|
||||
|
||||
- `fixture-display-none`
|
||||
- `fixture-visibility-hidden`
|
||||
- `fixture-opacity-zero`
|
||||
- `fixture-parent-opacity-zero`
|
||||
- `fixture-hidden-attribute`
|
||||
- `fixture-inert-parent`
|
||||
- `fixture-aria-hidden-parent`
|
||||
- `fixture-zero-size`
|
||||
- `fixture-offscreen-right`
|
||||
- `fixture-fully-overflow-clipped`
|
||||
|
||||
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
|
||||
|
||||
### `unlabeled.html`
|
||||
|
||||
This page contains eight Canvas elements but must render seven generic Surface markers because
|
||||
the two coincident layers form one group.
|
||||
|
||||
- Every marker uses the stable fallback label `canvas visual surface`.
|
||||
- Exactly one marker has `layers=2`.
|
||||
- The two contained canvases remain separate markers.
|
||||
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
|
||||
- Blank `aria-label` and `title` values behave as no label.
|
||||
- Fallback AX content does not suppress the corresponding Surface.
|
||||
|
||||
### `frames.html`
|
||||
|
||||
Expected Surface labels:
|
||||
|
||||
- `fixture-same-origin-frame`
|
||||
- `fixture-cross-origin-frame`
|
||||
- `fixture-nested-frame`
|
||||
|
||||
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
|
||||
|
||||
Each frame Canvas also exposes a `pointer status` live region. A successful
|
||||
screenshot-bound point click must report
|
||||
`down button=0 buttons=1 hoverReady=true; click received` and turn the Canvas
|
||||
yellow. The hover flag is set on the animation frame after `pointermove`, so the
|
||||
fixture verifies both child-document delivery and the scheduling boundary
|
||||
required before `pointerdown`, rather than merely validating projected
|
||||
coordinates or a successful CDP response.
|
||||
|
||||
### `viewport.html`
|
||||
|
||||
Run with a `900x700` Agent Window.
|
||||
|
||||
- `fixture-partial-viewport` is partly visible and must be represented.
|
||||
- `fixture-below-fold` is absent initially.
|
||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
||||
|
||||
### `dynamic.html`
|
||||
|
||||
Use the normal DOM buttons and re-observe after each action:
|
||||
|
||||
- Initial state: `fixture-dynamic` absent.
|
||||
- After **Insert canvas**: present.
|
||||
- After **Resize to zero**: absent.
|
||||
- After **Restore size**: present with a fresh ref.
|
||||
- After **Remove canvas**: absent.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `snapshot` must not emit any of these Surface markers.
|
||||
- `observe` must emit only currently visible Canvas surfaces.
|
||||
- Existing DOM/AX controls remain readable alongside Surface markers.
|
||||
- Missing labels never prevent discovery; naming and discovery are independent.
|
||||
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
|
||||
- No fixture URL, id, label, or layout constant may appear in production discovery code.
|
||||
@@ -1,46 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface dynamic matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
|
||||
button { padding: 8px 12px; }
|
||||
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Dynamic Canvas lifecycle</h1>
|
||||
<div class="controls">
|
||||
<button id="insert" type="button">Insert canvas</button>
|
||||
<button id="zero" type="button">Resize to zero</button>
|
||||
<button id="restore" type="button">Restore size</button>
|
||||
<button id="remove" type="button">Remove canvas</button>
|
||||
</div>
|
||||
<div id="mount"></div>
|
||||
<script>
|
||||
const mount = document.querySelector("#mount");
|
||||
const getCanvas = () => mount.querySelector("canvas");
|
||||
document.querySelector("#insert").addEventListener("click", () => {
|
||||
if (getCanvas()) return;
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 260;
|
||||
canvas.height = 120;
|
||||
canvas.setAttribute("aria-label", "fixture-dynamic");
|
||||
mount.append(canvas);
|
||||
});
|
||||
document.querySelector("#zero").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
|
||||
});
|
||||
document.querySelector("#restore").addEventListener("click", () => {
|
||||
const canvas = getCanvas();
|
||||
if (canvas) canvas.removeAttribute("style");
|
||||
});
|
||||
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,52 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 12px; background: #fdfefe; }
|
||||
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
|
||||
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
|
||||
#pointer-status { margin: 8px 0 0; min-height: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="frame-canvas" width="220" height="90"></canvas>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const canvas = document.querySelector("#frame-canvas");
|
||||
if (params.get("canvas") === "0") canvas.remove();
|
||||
else {
|
||||
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
||||
if (params.get("interactive") === "1") {
|
||||
const status = document.createElement("p");
|
||||
status.id = "pointer-status";
|
||||
status.setAttribute("aria-live", "polite");
|
||||
status.textContent = "pointer status: idle";
|
||||
canvas.insertAdjacentElement("afterend", status);
|
||||
let hoverReady = false;
|
||||
canvas.addEventListener("pointermove", () => {
|
||||
hoverReady = false;
|
||||
requestAnimationFrame(() => {
|
||||
hoverReady = true;
|
||||
});
|
||||
});
|
||||
canvas.addEventListener("pointerdown", (event) => {
|
||||
status.textContent = `pointer status: down button=${event.button} buttons=${event.buttons} hoverReady=${hoverReady}`;
|
||||
if (event.buttons === 1 && hoverReady) canvas.style.background = "#f9e79f";
|
||||
});
|
||||
canvas.addEventListener("click", () => {
|
||||
status.textContent += "; click received";
|
||||
});
|
||||
}
|
||||
}
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested fixture frame";
|
||||
iframe.src = "frame.html?label=fixture-nested-frame&interactive=1";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface frame matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame matrix</h1>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame&interactive=1"></iframe>
|
||||
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame&interactive=1"></iframe>
|
||||
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,118 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface discovery matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
.same-place { position: relative; width: 180px; height: 90px; }
|
||||
.same-place > div { position: absolute; inset: 0; }
|
||||
.same-place canvas { width: 180px; height: 90px; }
|
||||
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
|
||||
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
|
||||
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
|
||||
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
|
||||
.tiny { width: 8px; height: 8px; border-width: 1px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Rendered Surface discovery matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Visible named canvas</h2>
|
||||
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Tiny canvas remains addressable</h2>
|
||||
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Visible but pointer-events none</h2>
|
||||
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partial AX/fallback content</h2>
|
||||
<canvas aria-label="fixture-partial-ax" width="180" height="80">
|
||||
<button type="button">Fallback button must not suppress the surface</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Two real rendering layers</h2>
|
||||
<div class="stack">
|
||||
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Contained canvases are independent</h2>
|
||||
<div class="contained">
|
||||
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
|
||||
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Same geometry, different parents</h2>
|
||||
<div class="same-place">
|
||||
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
|
||||
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Partially overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fully overflow-clipped</h2>
|
||||
<div class="clip-box">
|
||||
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>CSS-hidden variants</h2>
|
||||
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
|
||||
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
|
||||
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Hidden ancestor variants</h2>
|
||||
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
|
||||
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
|
||||
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
|
||||
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Zero-size and offscreen</h2>
|
||||
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
|
||||
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Unlabeled Rendered Surface matrix</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
|
||||
body { padding: 16px; background: #f4f6f8; color: #17202a; }
|
||||
h1 { margin: 0 0 12px; font-size: 20px; }
|
||||
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
|
||||
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
|
||||
.case h2 { margin: 0 0 8px; font-size: 13px; }
|
||||
.case p { margin: 0 0 8px; }
|
||||
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
|
||||
.stack { position: relative; width: 180px; height: 90px; }
|
||||
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
|
||||
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
|
||||
.contained { position: relative; width: 190px; height: 100px; }
|
||||
.contained > canvas:first-child { width: 190px; height: 100px; }
|
||||
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Unlabeled Rendered Surface matrix</h1>
|
||||
<main class="matrix">
|
||||
<section class="case">
|
||||
<h2>Only engineering attributes</h2>
|
||||
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Blank explicit names</h2>
|
||||
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Nearby prose is not a Canvas name</h2>
|
||||
<p>must-not-be-inferred-as-a-surface-label</p>
|
||||
<canvas width="180" height="80"></canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Fallback AX content does not name or suppress the Canvas</h2>
|
||||
<canvas width="180" height="80">
|
||||
<button type="button">Fallback control remains separate</button>
|
||||
</canvas>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed rendering layers still cluster</h2>
|
||||
<div class="stack">
|
||||
<canvas width="180" height="90"></canvas>
|
||||
<canvas width="180" height="90"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="case">
|
||||
<h2>Unnamed contained canvases stay independent</h2>
|
||||
<div class="contained">
|
||||
<canvas width="190" height="100"></canvas>
|
||||
<canvas width="70" height="40"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface viewport matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { min-height: 1800px; padding: 16px; }
|
||||
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
|
||||
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
|
||||
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Viewport matrix</h1>
|
||||
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
|
||||
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
|
||||
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# Visual observation design fixtures
|
||||
|
||||
These are synthetic, site-independent inputs for the visual observation architecture in
|
||||
[the design](../../../../docs/design/visual-observation.md). They contain no original page
|
||||
data, account identifiers, external fonts or production selectors. They are not a production
|
||||
discovery implementation.
|
||||
|
||||
Use [the acceptance checklist](../../../../docs/design/visual-observation-acceptance.md)
|
||||
for each PR's steps and success criteria. No separate report is required.
|
||||
Historical probe success does not certify future production modules.
|
||||
|
||||
## Browser contract probe
|
||||
|
||||
Requires Node.js with built-in WebSocket support (Node 22+) and an installed Chrome. No extra
|
||||
packages, user profile, extension, or login are required.
|
||||
|
||||
From the repository root:
|
||||
|
||||
```sh
|
||||
node apps/extension/test-fixtures/visual-observation/probe.mjs /tmp/bsk-visual-evidence
|
||||
```
|
||||
|
||||
On platforms other than macOS, set `CHROME_PATH` to the installed Chrome executable. The probe
|
||||
starts its own headless Chrome with an ephemeral profile and a loopback fixture server. It
|
||||
removes the profile and stops Chrome/server on completion or failure. `report.json`, raw CDP
|
||||
replies and PNGs are written to the explicitly supplied directory; do not commit them.
|
||||
|
||||
The probe verifies CSS exposure versus visibility, snapshot units, DPR raster dimensions,
|
||||
CSS zoom, frame content origins, child scrolling, nested same-target geometry, a confirmed
|
||||
OOPIF target, and navigation/document.open identity behavior. It records page-scale emulation
|
||||
separately; that is **not** browser UI zoom. It does not certify production VOM behavior, arbitrary
|
||||
occlusion, exact clip-path geometry, or all browser versions.
|
||||
|
||||
## Minimal end-to-end reproduction
|
||||
|
||||
```sh
|
||||
python3 -m http.server 4177 --bind 127.0.0.1 \
|
||||
--directory apps/extension/test-fixtures/visual-observation
|
||||
bsk session start --width 1440 --height 1100
|
||||
bsk navigate http://127.0.0.1:4177/table.html --session <session>
|
||||
bsk observe --session <session>
|
||||
bsk screenshot --session <session> --out /tmp/bsk-table.png
|
||||
bsk session stop <session>
|
||||
```
|
||||
|
||||
Always stop the session and the local server. Use a foreground Agent Window and confirm the
|
||||
painted grid is visible before interpreting the observation. `load`, AX toolbar appearance,
|
||||
and background screenshots alone do not prove a canvas has drawn.
|
||||
|
||||
`table.html` embeds `grid.html` through `localhost` from a `127.0.0.1` parent. The grid has DOM
|
||||
toolbar controls and one unlabeled Canvas with three synthetic columns and six painted rows.
|
||||
|
||||
Main baseline: `Page action`, `Insert record`, and `Filter` are referenceable; the painted
|
||||
`Canvas column A/B/C` and `Canvas row 1..6` have no textual observation representation or visual
|
||||
ref, despite an untruncated result. This is a **known gap**, not a golden expectation that future
|
||||
implementations must preserve.
|
||||
|
||||
Future acceptance: retain the DOM controls and add a screenshot-only visual ref for the visible
|
||||
grid, with correct frame projection. Do not invent AX cell text. Validate the corresponding
|
||||
screenshot, then move/remove/resize and re-observe according to the freshness contract.
|
||||
|
||||
## Discovery policy cases
|
||||
|
||||
At a 1440×1100 window and without scrolling:
|
||||
|
||||
| Canvas ID | Expected candidate in the new design | Reason |
|
||||
|---|---|---|
|
||||
| plain | yes | No name required |
|
||||
| named | yes | Name does not establish complete pixel semantics |
|
||||
| aria-hidden | yes | Accessibility exclusion is not visual exclusion |
|
||||
| inert | yes | Inertness is not visual hiding |
|
||||
| pointer-none | yes | Screenshot observation does not require pointer interaction |
|
||||
| visibility-override | yes | Child restores computed visibility |
|
||||
| opacity-parent | no | Ancestor makes subtree transparent |
|
||||
| display-none | no | No layout box |
|
||||
| visibility-hidden | no | Computed visibility suppresses this element |
|
||||
| zero | no | Empty geometry |
|
||||
| hidden-overridden | yes | CSS display overrides normal hidden presentation |
|
||||
| partial-clip | yes, 60×30 crop | Supported overflow intersection |
|
||||
| full-clip | no | Fully outside ancestor clip |
|
||||
| axis-clip | yes, x clipped only | overflow-x:clip and overflow-y:visible remain independent |
|
||||
| border-clip | yes, clipped to client box | Ancestor border box is not its scrollport |
|
||||
| border-canvas | yes, border-box region | Observation and live screenshot resolver must use the same box kind |
|
||||
| fallback | yes | AX fallback is not full pixel coverage |
|
||||
| stack-a / stack-b | one output entry after selection | Same document/parent/exact crop; not proof of a logical object |
|
||||
|
||||
`checkVisibility()` is used in the probe only for CSS suppression evidence; it does not establish
|
||||
viewport intersection, ancestor overflow clipping, or absence of occlusion. `frame.html` supplies
|
||||
known local coordinates, bordered owners, nested frames and scrollable content for independent
|
||||
geometry verification. The fixture IDs must never enter production discovery code.
|
||||
|
||||
## Renderer operation counts
|
||||
|
||||
After installing the repository's normal dependencies:
|
||||
|
||||
```sh
|
||||
node apps/extension/test-fixtures/visual-observation/measure-render.mjs /tmp/bsk-render-evidence
|
||||
```
|
||||
|
||||
This instruments a temporary copy of the current renderer, runs the installed Vitest, and writes
|
||||
the source hash and loop counters to `render-counts.json`. It checks instrumentation sites and
|
||||
fails explicitly when source structure changes; update the diagnostic deliberately then. It never
|
||||
modifies production sources and never asserts that today's quadratic costs must persist.
|
||||
|
||||
The wide fixture contains N headings followed by N equally named buttons in a common context.
|
||||
The deep fixture contains 2,000/4,000 transparent wrappers. Counters cover render-stack visits,
|
||||
ordered DOM context candidates, and same-container sibling visits. They are not a complete CPU
|
||||
profile or an assertion about the complexity of all VOM stages.
|
||||
|
||||
Use these existing-renderer counts only as a before/after regression baseline. Improving the
|
||||
historical algorithm is not a prerequisite for Canvas work. New Canvas stages need their own
|
||||
workload models and measurements; do not copy this renderer's algorithms or use these counts
|
||||
as evidence that visual discovery, clipping, selection, or screenshot execution is efficient.
|
||||
|
||||
## Review boundary
|
||||
|
||||
Commit only these synthetic fixtures, diagnostic tools, and reviewed design/evidence summaries.
|
||||
Keep local browser profiles, original internal-site screenshots/HTML/observations, generated
|
||||
captures, temporary source instrumentation and build outputs outside the changeset.
|
||||
@@ -0,0 +1,25 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<meta charset="utf-8">
|
||||
<title>Frame geometry contract</title>
|
||||
<style>
|
||||
body { margin: 0; width: 600px; height: 900px; }
|
||||
canvas { position: absolute; left: 17px; top: 23px; width: 120px; height: 40px; }
|
||||
iframe { position: absolute; left: 30px; top: 90px; width: 180px; height: 100px; border: 7px solid #555; }
|
||||
</style>
|
||||
<canvas id="frame-canvas" width="120" height="40"></canvas>
|
||||
<script>
|
||||
const context = document.querySelector("canvas").getContext("2d");
|
||||
context.fillStyle = "#15803d";
|
||||
context.fillRect(0, 0, 120, 40);
|
||||
context.fillStyle = "white";
|
||||
context.fillText("FRAME PIXELS", 4, 24);
|
||||
if (new URLSearchParams(location.search).has("nested")) {
|
||||
const child = document.createElement("iframe");
|
||||
child.title = "nested same-target frame";
|
||||
child.src = "/frame.html";
|
||||
document.body.append(child);
|
||||
}
|
||||
window.fixtureReady = true;
|
||||
</script>
|
||||
</html>
|
||||
@@ -0,0 +1,26 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<meta charset="utf-8">
|
||||
<title>Canvas grid</title>
|
||||
<style>
|
||||
body { margin: 0; font: 16px sans-serif; }
|
||||
nav { height: 40px; }
|
||||
canvas { display: block; width: 780px; height: 360px; }
|
||||
</style>
|
||||
<nav><button type="button">Insert record</button> <button type="button">Filter</button></nav>
|
||||
<canvas width="1560" height="720"></canvas>
|
||||
<script>
|
||||
const context = document.querySelector("canvas").getContext("2d");
|
||||
context.scale(2, 2);
|
||||
context.fillStyle = "white";
|
||||
context.fillRect(0, 0, 780, 360);
|
||||
context.strokeStyle = "#777";
|
||||
for (let x = 0; x <= 780; x += 260) { context.beginPath(); context.moveTo(x, 0); context.lineTo(x, 350); context.stroke(); }
|
||||
for (let y = 0; y <= 350; y += 50) { context.beginPath(); context.moveTo(0, y); context.lineTo(780, y); context.stroke(); }
|
||||
context.fillStyle = "#111";
|
||||
context.font = "16px sans-serif";
|
||||
["Canvas column A", "Canvas column B", "Canvas column C"].forEach((label, i) => context.fillText(label, i * 260 + 12, 30));
|
||||
for (let row = 1; row <= 6; row++) context.fillText(`Canvas row ${row}`, 12, row * 50 + 30);
|
||||
window.fixtureReady = true;
|
||||
</script>
|
||||
</html>
|
||||
@@ -0,0 +1,56 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<meta charset="utf-8">
|
||||
<title>Visual observation contract fixture</title>
|
||||
<style>
|
||||
body { margin: 0; min-height: 1800px; font: 14px sans-serif; }
|
||||
#cases { display: grid; grid-template-columns: repeat(4, 180px); gap: 12px; padding: 20px; }
|
||||
.case { width: 160px; height: 70px; }
|
||||
canvas { width: 120px; height: 40px; }
|
||||
.clip { width: 60px; height: 30px; overflow: hidden; }
|
||||
.axis-clip { width: 60px; height: 20px; overflow-x: clip; overflow-y: visible; }
|
||||
#same, #cross { position: absolute; top: 600px; width: 300px; height: 220px; border: 10px solid #555; }
|
||||
#same { left: 40px; }
|
||||
#cross { left: 400px; }
|
||||
</style>
|
||||
<h1>Canvas contract</h1>
|
||||
<button type="button" id="control">Ordinary DOM control</button>
|
||||
<div id="cases">
|
||||
<div class="case"><canvas id="plain"></canvas></div>
|
||||
<div class="case"><canvas id="named" aria-label="Named chart"></canvas></div>
|
||||
<div class="case" aria-hidden="true"><canvas id="aria-hidden"></canvas></div>
|
||||
<div class="case" inert><canvas id="inert"></canvas></div>
|
||||
<div class="case"><canvas id="pointer-none" style="pointer-events:none"></canvas></div>
|
||||
<div class="case" style="visibility:hidden"><canvas id="visibility-override" style="visibility:visible"></canvas></div>
|
||||
<div class="case" style="opacity:0"><canvas id="opacity-parent"></canvas></div>
|
||||
<div class="case"><canvas id="display-none" style="display:none"></canvas></div>
|
||||
<div class="case"><canvas id="visibility-hidden" style="visibility:hidden"></canvas></div>
|
||||
<div class="case"><canvas id="zero" style="width:0"></canvas></div>
|
||||
<div class="case"><canvas id="hidden-overridden" hidden style="display:block"></canvas></div>
|
||||
<div class="case"><div class="clip"><canvas id="partial-clip"></canvas></div></div>
|
||||
<div class="case"><div class="clip"><canvas id="full-clip" style="margin-left:80px"></canvas></div></div>
|
||||
<div class="case"><canvas id="fallback">Accessible fallback is not pixel content</canvas></div>
|
||||
<div class="case"><div class="axis-clip"><canvas id="axis-clip"></canvas></div></div>
|
||||
<div class="case"><div id="border-clip-owner" style="width:60px;height:30px;border:5px solid black;padding:4px;overflow:hidden"><canvas id="border-clip"></canvas></div></div>
|
||||
<div class="case"><canvas id="border-canvas" style="border:5px solid black"></canvas></div>
|
||||
<div class="case" style="position:relative"><canvas id="stack-a"></canvas><canvas id="stack-b" style="position:absolute;left:0;top:0"></canvas></div>
|
||||
</div>
|
||||
<iframe id="same" title="same target"></iframe>
|
||||
<iframe id="cross" title="cross site"></iframe>
|
||||
<script>
|
||||
for (const canvas of document.querySelectorAll("canvas")) {
|
||||
canvas.width = 120;
|
||||
canvas.height = 40;
|
||||
const context = canvas.getContext("2d");
|
||||
context.fillStyle = "#1d4ed8";
|
||||
context.fillRect(0, 0, 120, 40);
|
||||
context.fillStyle = "white";
|
||||
context.fillText("PIXEL TABLE 42", 5, 24);
|
||||
}
|
||||
if (!new URLSearchParams(location.search).has("noFrames")) {
|
||||
document.querySelector("#same").src = "/frame.html?nested=1";
|
||||
document.querySelector("#cross").src = `${location.protocol}//localhost:${location.port}/frame.html`;
|
||||
}
|
||||
window.fixtureReady = true;
|
||||
</script>
|
||||
</html>
|
||||
@@ -0,0 +1,86 @@
|
||||
// Diagnostic, not a golden test requiring the current quadratic cost to persist.
|
||||
// Instrument a temporary copy; never modify the production renderer.
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../..");
|
||||
const output = process.argv[2];
|
||||
if (!output || !path.isAbsolute(output))
|
||||
throw new Error("Pass an absolute evidence output directory");
|
||||
const require = createRequire(path.join(root, "apps/extension/package.json"));
|
||||
const vitestDirectory = path.dirname(require.resolve("vitest/package.json"));
|
||||
const temporary = await mkdtemp(path.join(os.tmpdir(), "bsk-render-measure-"));
|
||||
await mkdir(output, { recursive: true });
|
||||
try {
|
||||
const source = await readFile(path.join(root, "packages/vom/src/render.ts"), "utf8");
|
||||
let instrumented = source;
|
||||
for (const [needle, field] of [
|
||||
["const { node, depth } = stack.pop() as { node: VomNode; depth: number };", "renderVisits"],
|
||||
["for (const candidate of orderedCandidates) {", "contextCandidates"],
|
||||
["for (const sibling of siblings) {", "siblingVisits"],
|
||||
]) {
|
||||
assert.equal(instrumented.split(needle).length, 2, `Update measurement site: ${field}`);
|
||||
instrumented = instrumented.replace(needle, `${needle} designCounts.${field}++;`);
|
||||
}
|
||||
instrumented = `export const designCounts = {renderVisits:0,contextCandidates:0,siblingVisits:0};\n${instrumented}`;
|
||||
await writeFile(path.join(temporary, "render.ts"), instrumented);
|
||||
for (const file of ["layers.ts", "types.ts"])
|
||||
await copyFile(path.join(root, "packages/vom/src", file), path.join(temporary, file));
|
||||
const reportPath = path.join(output, "render-counts.json");
|
||||
await writeFile(
|
||||
path.join(temporary, "measure.test.ts"),
|
||||
`
|
||||
import {writeFileSync} from 'node:fs';
|
||||
import {test,expect} from ${JSON.stringify(path.join(vitestDirectory, "dist/index.js"))};
|
||||
import {renderVom,designCounts} from './render';
|
||||
test('measure operation counts',()=>{
|
||||
const rows=[];
|
||||
const make=(id,extra={})=>({id,parentId:1,backendNodeId:id,tag:'div',role:'generic',rect:{x:0,y:0,w:100,h:20},paintOrder:0,position:'static',pointerEvents:'auto',contextScopeId:'root',...extra});
|
||||
const run=(shape,n,nodes)=>{
|
||||
Object.assign(designCounts,{renderVisits:0,contextCandidates:0,siblingVisits:0});
|
||||
const result=renderVom({viewport:{width:1440,height:1000},nodes},{maxTokens:Infinity});
|
||||
rows.push({shape,n,counts:{...designCounts},refs:result.refs.length});
|
||||
};
|
||||
for(const n of [100,200,400]){
|
||||
const nodes=[make(1,{parentId:null,role:'RootWebArea',name:'Doc'})];
|
||||
for(let i=0;i<n;i++)nodes.push(make(i+2,{role:'heading',name:'Region '+i,domParentId:1,domAncestorIds:[1]}));
|
||||
for(let i=0;i<n;i++)nodes.push(make(n+i+2,{tag:'button',role:'button',name:'View',domParentId:1,domAncestorIds:[1]}));
|
||||
run('wide',n,nodes);
|
||||
}
|
||||
for(const n of [2000,4000]){
|
||||
const nodes=[make(1,{parentId:null,role:'RootWebArea',name:'Doc'})];
|
||||
for(let i=2;i<=n;i++)nodes.push(make(i,{parentId:i-1}));
|
||||
nodes.push(make(n+1,{parentId:n,role:'button',tag:'button',name:'Unique control'}));
|
||||
run('deep',n,nodes);
|
||||
}
|
||||
expect(rows.at(-1).refs).toBe(1);
|
||||
writeFileSync(${JSON.stringify(reportPath)},JSON.stringify({sourceSha256:${JSON.stringify(createHash("sha256").update(source).digest("hex"))},rows},null,2));
|
||||
});
|
||||
`,
|
||||
);
|
||||
const config = path.join(temporary, "vitest.config.mjs");
|
||||
await writeFile(
|
||||
config,
|
||||
`export default {test:{environment:'node',include:[${JSON.stringify(path.join(temporary, "measure.test.ts"))}]}};`,
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[path.join(vitestDirectory, "vitest.mjs"), "run", "--config", config],
|
||||
{ cwd: root, encoding: "utf8", timeout: 60000 },
|
||||
);
|
||||
await writeFile(
|
||||
path.join(output, "render-measure.log"),
|
||||
`${result.stdout ?? ""}\n${result.stderr ?? ""}`,
|
||||
);
|
||||
if (result.error) throw result.error;
|
||||
assert.equal(result.status, 0, `Measurement failed; see ${output}/render-measure.log`);
|
||||
process.stdout.write(`${reportPath}\n`);
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
// Standalone browser contract probe. No extension, credentials or user profile.
|
||||
// Raw replies and screenshots are written only to the caller-supplied directory.
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const fixtureDirectory = path.dirname(fileURLToPath(import.meta.url));
|
||||
const output = process.argv[2];
|
||||
if (!output || !path.isAbsolute(output))
|
||||
throw new Error("Pass an absolute evidence output directory");
|
||||
const chromePath =
|
||||
process.env.CHROME_PATH ?? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const styles = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
"display",
|
||||
];
|
||||
const checks = [];
|
||||
function check(name, condition) {
|
||||
checks.push({ name, passed: Boolean(condition) });
|
||||
assert.ok(condition, name);
|
||||
}
|
||||
|
||||
class Cdp {
|
||||
nextId = 0;
|
||||
pending = new Map();
|
||||
events = [];
|
||||
counts = {};
|
||||
async connect(url) {
|
||||
this.socket = new WebSocket(url);
|
||||
this.socket.addEventListener("message", ({ data }) => {
|
||||
const reply = JSON.parse(data);
|
||||
if (!reply.id) {
|
||||
this.events.push(reply);
|
||||
return;
|
||||
}
|
||||
const task = this.pending.get(reply.id);
|
||||
if (!task) return;
|
||||
this.pending.delete(reply.id);
|
||||
clearTimeout(task.timer);
|
||||
if (reply.error) task.reject(new Error(JSON.stringify(reply.error)));
|
||||
else task.resolve(reply.result);
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
this.socket.addEventListener("open", resolve, { once: true });
|
||||
this.socket.addEventListener("error", reject, { once: true });
|
||||
});
|
||||
}
|
||||
send(method, params = {}, sessionId) {
|
||||
const id = ++this.nextId;
|
||||
this.counts[method] = (this.counts[method] ?? 0) + 1;
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id);
|
||||
reject(new Error(`CDP timeout: ${method}`));
|
||||
}, 10000);
|
||||
this.pending.set(id, { resolve, reject, timer });
|
||||
this.socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
|
||||
});
|
||||
}
|
||||
close() {
|
||||
for (const task of this.pending.values()) {
|
||||
clearTimeout(task.timer);
|
||||
task.reject(new Error("CDP closed"));
|
||||
}
|
||||
this.pending.clear();
|
||||
this.socket?.close();
|
||||
}
|
||||
}
|
||||
|
||||
function decodedCanvases(snapshot) {
|
||||
return snapshot.documents.flatMap((document) => {
|
||||
const layoutByNode = new Map(document.layout.nodeIndex.map((node, index) => [node, index]));
|
||||
return document.nodes.nodeName.flatMap((name, index) => {
|
||||
if (snapshot.strings[name] !== "CANVAS") return [];
|
||||
const attrs = document.nodes.attributes[index];
|
||||
const attributes = {};
|
||||
for (let i = 0; i < attrs.length; i += 2)
|
||||
attributes[snapshot.strings[attrs[i]]] = snapshot.strings[attrs[i + 1]];
|
||||
const layoutIndex = layoutByNode.get(index) ?? -1;
|
||||
return [
|
||||
{
|
||||
id: attributes.id,
|
||||
backendNodeId: document.nodes.backendNodeId[index],
|
||||
frameId: snapshot.strings[document.frameId],
|
||||
attributes,
|
||||
bounds: layoutIndex < 0 ? null : document.layout.bounds[layoutIndex],
|
||||
styles:
|
||||
layoutIndex < 0
|
||||
? null
|
||||
: Object.fromEntries(
|
||||
styles.map((style, i) => [
|
||||
style,
|
||||
snapshot.strings[document.layout.styles[layoutIndex][i]],
|
||||
]),
|
||||
),
|
||||
},
|
||||
];
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
await mkdir(output, { recursive: true });
|
||||
const profile = await mkdtemp(path.join(os.tmpdir(), "bsk-visual-probe-"));
|
||||
const server = http.createServer(async (request, response) => {
|
||||
const name = new URL(request.url, "http://fixture").pathname;
|
||||
if (!["/index.html", "/frame.html"].includes(name)) {
|
||||
response.writeHead(404).end();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
response.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
response.end(await readFile(path.join(fixtureDirectory, name.slice(1))));
|
||||
} catch {
|
||||
response.writeHead(500).end();
|
||||
}
|
||||
});
|
||||
const cdp = new Cdp();
|
||||
let chrome;
|
||||
let report = {};
|
||||
try {
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
const port = server.address().port;
|
||||
chrome = spawn(
|
||||
chromePath,
|
||||
[
|
||||
"--headless=new",
|
||||
`--user-data-dir=${profile}`,
|
||||
"--remote-debugging-port=0",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--site-per-process",
|
||||
"--window-size=1440,1100",
|
||||
"about:blank",
|
||||
],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
chrome.on("error", (error) => {
|
||||
report.launchError = error.message;
|
||||
});
|
||||
let activePort;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
try {
|
||||
activePort = (await readFile(path.join(profile, "DevToolsActivePort"), "utf8"))
|
||||
.trim()
|
||||
.split("\n");
|
||||
break;
|
||||
} catch {
|
||||
await pause(100);
|
||||
}
|
||||
}
|
||||
if (!activePort)
|
||||
throw new Error(report.launchError ?? "Chrome did not expose DevToolsActivePort");
|
||||
await cdp.connect(`ws://127.0.0.1:${activePort[0]}${activePort[1]}`);
|
||||
report.browser = await cdp.send("Browser.getVersion");
|
||||
const { targetId } = await cdp.send("Target.createTarget", { url: "about:blank" });
|
||||
const { sessionId: root } = await cdp.send("Target.attachToTarget", { targetId, flatten: true });
|
||||
const send = (method, params = {}) => cdp.send(method, params, root);
|
||||
const evaluate = async (expression, session = root, contextId) => {
|
||||
const result = await cdp.send(
|
||||
"Runtime.evaluate",
|
||||
{ expression, returnByValue: true, awaitPromise: true, ...(contextId ? { contextId } : {}) },
|
||||
session,
|
||||
);
|
||||
if (result.exceptionDetails) throw new Error(JSON.stringify(result.exceptionDetails));
|
||||
return result.result.value;
|
||||
};
|
||||
await send("Page.enable");
|
||||
await send("Runtime.enable");
|
||||
async function navigate(url) {
|
||||
await send("Page.navigate", { url });
|
||||
for (let i = 0; i < 100; i++) {
|
||||
if (await evaluate("document.readyState === 'complete' && window.fixtureReady === true"))
|
||||
return;
|
||||
await pause(50);
|
||||
}
|
||||
throw new Error("Fixture did not become ready");
|
||||
}
|
||||
const url = `http://127.0.0.1:${port}/index.html`;
|
||||
await navigate(`${url}?noFrames=1`);
|
||||
async function capture(label) {
|
||||
const metrics = await send("Page.getLayoutMetrics");
|
||||
const snapshot = await send("DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: styles,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
const ax = await send("Accessibility.getFullAXTree");
|
||||
const runtime = await evaluate(`Array.from(document.querySelectorAll('canvas'), el => {
|
||||
const r=el.getBoundingClientRect(), s=getComputedStyle(el);
|
||||
return {id:el.id,rect:{x:r.x,y:r.y,w:r.width,h:r.height},visibility:s.visibility,display:s.display,
|
||||
checkVisibility:el.checkVisibility({opacityProperty:true,visibilityProperty:true}),
|
||||
dpr:devicePixelRatio,scrollX,scrollY}; })`);
|
||||
const result = { metrics, snapshot, ax, runtime, canvases: decodedCanvases(snapshot) };
|
||||
await writeFile(path.join(output, `${label}.json`), JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
}
|
||||
const baseline = await capture("root-baseline");
|
||||
const rootImage = await send("Page.captureScreenshot", { format: "png" });
|
||||
await writeFile(path.join(output, "root.png"), Buffer.from(rootImage.data, "base64"));
|
||||
const runtime = Object.fromEntries(baseline.runtime.map((node) => [node.id, node]));
|
||||
for (const id of [
|
||||
"plain",
|
||||
"named",
|
||||
"aria-hidden",
|
||||
"inert",
|
||||
"pointer-none",
|
||||
"visibility-override",
|
||||
"hidden-overridden",
|
||||
"fallback",
|
||||
])
|
||||
check(`CSS visible: ${id}`, runtime[id].checkVisibility);
|
||||
for (const id of ["opacity-parent", "display-none", "visibility-hidden"])
|
||||
check(`CSS suppressed: ${id}`, !runtime[id].checkVisibility);
|
||||
check("painted text is absent from AX", !JSON.stringify(baseline.ax).includes("PIXEL TABLE 42"));
|
||||
report.css = baseline.runtime;
|
||||
const bordered = baseline.canvases.find((node) => node.id === "border-canvas");
|
||||
report.canvasBox = await send("DOM.getBoxModel", { backendNodeId: bordered.backendNodeId });
|
||||
check(
|
||||
"Canvas snapshot uses border box",
|
||||
bordered.bounds[2] === 130 &&
|
||||
report.canvasBox.model.border[2] - report.canvasBox.model.border[0] === 130 &&
|
||||
report.canvasBox.model.content[2] - report.canvasBox.model.content[0] === 120,
|
||||
);
|
||||
const rootDocument = baseline.snapshot.documents[0];
|
||||
const ownerNode = rootDocument.nodes.attributes.findIndex((attrs) =>
|
||||
attrs.some(
|
||||
(value, index) => index % 2 === 1 && baseline.snapshot.strings[value] === "border-clip-owner",
|
||||
),
|
||||
);
|
||||
const ownerLayout = rootDocument.layout.nodeIndex.indexOf(ownerNode);
|
||||
report.clipClientBox = {
|
||||
snapshot: rootDocument.layout.clientRects[ownerLayout],
|
||||
runtime: await evaluate(
|
||||
"(() => {const e=document.getElementById('border-clip-owner');return [e.clientLeft,e.clientTop,e.clientWidth,e.clientHeight]})()",
|
||||
),
|
||||
};
|
||||
check(
|
||||
"snapshot client rect contains local client offsets and size",
|
||||
JSON.stringify(report.clipClientBox.snapshot) ===
|
||||
JSON.stringify(report.clipClientBox.runtime) &&
|
||||
JSON.stringify(report.clipClientBox.runtime) === "[5,5,68,38]",
|
||||
);
|
||||
report.clips = await evaluate(`['partial-clip','full-clip','axis-clip'].map(id=>{
|
||||
const el=document.getElementById(id),p=el.parentElement,s=getComputedStyle(p);
|
||||
const a=el.getBoundingClientRect(),b=p.getBoundingClientRect();
|
||||
return {id,overflowX:s.overflowX,overflowY:s.overflowY,
|
||||
child:{x:a.x,y:a.y,w:a.width,h:a.height},parent:{x:b.x,y:b.y,w:b.width,h:b.height}};
|
||||
})`);
|
||||
const [partial, full, axis] = report.clips;
|
||||
check(
|
||||
"partial overflow crop dimensions",
|
||||
partial.parent.w === 60 &&
|
||||
partial.parent.h === 30 &&
|
||||
partial.child.x === partial.parent.x &&
|
||||
partial.child.y === partial.parent.y,
|
||||
);
|
||||
check(
|
||||
"fully clipped candidate lies beyond ancestor",
|
||||
full.child.x >= full.parent.x + full.parent.w,
|
||||
);
|
||||
check(
|
||||
"single-axis clip remains independent in computed style",
|
||||
axis.overflowX === "clip" && axis.overflowY === "visible" && axis.child.h > axis.parent.h,
|
||||
);
|
||||
const samples = [];
|
||||
for (let i = 0; i < 12; i++)
|
||||
samples.push(
|
||||
await evaluate(
|
||||
"JSON.stringify(document.querySelector('#plain').getBoundingClientRect().toJSON())",
|
||||
),
|
||||
);
|
||||
check("static rect stable over 12 reads", new Set(samples).size === 1);
|
||||
await evaluate("document.querySelector('#plain').style.transform='translateX(0.5px)'");
|
||||
report.halfPixelMovement = await evaluate(
|
||||
"document.querySelector('#plain').getBoundingClientRect().x",
|
||||
);
|
||||
await evaluate("document.querySelector('#plain').style.transform=''");
|
||||
report.raster = [];
|
||||
for (const dpr of [1, 1.5, 2]) {
|
||||
await send("Emulation.setDeviceMetricsOverride", {
|
||||
width: 1000,
|
||||
height: 900,
|
||||
deviceScaleFactor: dpr,
|
||||
mobile: false,
|
||||
});
|
||||
const state = await capture(`dpr-${dpr}`);
|
||||
const image = await send("Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: { x: 20, y: 100, width: 120, height: 40, scale: 1 },
|
||||
});
|
||||
const bytes = Buffer.from(image.data, "base64");
|
||||
const dimensions = { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
|
||||
report.raster.push({
|
||||
dpr,
|
||||
dimensions,
|
||||
metrics: state.metrics,
|
||||
plain: state.canvases.find((node) => node.id === "plain"),
|
||||
});
|
||||
check(
|
||||
`screenshot raster DPR ${dpr}`,
|
||||
dimensions.width === 120 * dpr && dimensions.height === 40 * dpr,
|
||||
);
|
||||
}
|
||||
await send("Emulation.clearDeviceMetricsOverride");
|
||||
await evaluate("document.body.style.zoom='1.25'");
|
||||
const cssZoom = await capture("css-zoom-1.25");
|
||||
check(
|
||||
"CSS zoom is reflected in snapshot geometry",
|
||||
cssZoom.canvases.find((n) => n.id === "plain").bounds[2] === 150,
|
||||
);
|
||||
report.cssZoom = {
|
||||
runtime: cssZoom.runtime[0],
|
||||
snapshot: cssZoom.canvases.find((n) => n.id === "plain"),
|
||||
};
|
||||
await evaluate("document.body.style.zoom=''");
|
||||
await send("Emulation.setPageScaleFactor", { pageScaleFactor: 1.5 });
|
||||
report.pageScale = (await capture("page-scale-1.5")).metrics;
|
||||
await send("Emulation.setPageScaleFactor", { pageScaleFactor: 1 });
|
||||
await evaluate("scrollTo(0,100)");
|
||||
const scrolled = await capture("scroll-100");
|
||||
check(
|
||||
"snapshot document bounds stable under scroll",
|
||||
JSON.stringify(scrolled.canvases.find((n) => n.id === "plain").bounds) ===
|
||||
JSON.stringify(baseline.canvases.find((n) => n.id === "plain").bounds),
|
||||
);
|
||||
check(
|
||||
"runtime viewport y follows scroll",
|
||||
scrolled.runtime[0].rect.y === baseline.runtime[0].rect.y - 100,
|
||||
);
|
||||
await navigate(url);
|
||||
await pause(200);
|
||||
const targets = await cdp.send("Target.getTargets");
|
||||
const iframeTarget = targets.targetInfos.find(
|
||||
(target) => target.type === "iframe" && target.url.startsWith(`http://localhost:${port}/`),
|
||||
);
|
||||
check("cross-site frame is an actual OOPIF target", iframeTarget);
|
||||
const { sessionId: child } = await cdp.send("Target.attachToTarget", {
|
||||
targetId: iframeTarget.targetId,
|
||||
flatten: true,
|
||||
});
|
||||
report.frames = {};
|
||||
for (const [name, session] of [
|
||||
["root", root],
|
||||
["oopif", child],
|
||||
]) {
|
||||
const tree = await cdp.send("Page.getFrameTree", {}, session);
|
||||
const snapshot = await cdp.send(
|
||||
"DOMSnapshot.captureSnapshot",
|
||||
{ computedStyles: styles, includeDOMRects: true },
|
||||
session,
|
||||
);
|
||||
const frameRows = [];
|
||||
const stack = [tree.frameTree];
|
||||
while (stack.length) {
|
||||
const node = stack.pop();
|
||||
const world = await cdp.send(
|
||||
"Page.createIsolatedWorld",
|
||||
{ frameId: node.frame.id, worldName: "bsk-design-probe" },
|
||||
session,
|
||||
);
|
||||
const local = await evaluate(
|
||||
"({url:location.pathname,innerWidth,innerHeight,scrollX,scrollY,canvas:document.querySelector('canvas').getBoundingClientRect().toJSON()})",
|
||||
session,
|
||||
world.executionContextId,
|
||||
);
|
||||
let owner;
|
||||
if (node.frame.parentId) {
|
||||
// All owners in this fixture live in the root target; child content can be OOPIF.
|
||||
const address = await send("DOM.getFrameOwner", { frameId: node.frame.id });
|
||||
owner = await send("DOM.getBoxModel", { backendNodeId: address.backendNodeId });
|
||||
}
|
||||
frameRows.push({ frame: node.frame, local, owner });
|
||||
stack.push(...(node.childFrames ?? []));
|
||||
}
|
||||
report.frames[name] = { tree, snapshot, canvases: decodedCanvases(snapshot), frameRows };
|
||||
}
|
||||
check(
|
||||
"same-target snapshot includes nested documents",
|
||||
report.frames.root.snapshot.documents.length === 3,
|
||||
);
|
||||
check("oopif snapshot isolated from root", report.frames.oopif.snapshot.documents.length === 1);
|
||||
const sameRow = report.frames.root.frameRows.find(
|
||||
(row) => row.owner && row.local.innerWidth === 300,
|
||||
);
|
||||
const nestedRow = report.frames.root.frameRows.find(
|
||||
(row) => row.owner && row.local.innerWidth === 180,
|
||||
);
|
||||
check(
|
||||
"same frame content origin includes border",
|
||||
sameRow.owner.model.content[0] === 50 && sameRow.owner.model.content[1] === 610,
|
||||
);
|
||||
check(
|
||||
"nested owner quad is already target-root relative",
|
||||
nestedRow.owner.model.content[0] === 87 && nestedRow.owner.model.content[1] === 707,
|
||||
);
|
||||
const crossRow = report.frames.oopif.frameRows[0];
|
||||
check(
|
||||
"OOPIF owner content origin",
|
||||
crossRow.owner.model.content[0] === 410 && crossRow.owner.model.content[1] === 610,
|
||||
);
|
||||
const sameFrame = sameRow.frame.id;
|
||||
const world = await send("Page.createIsolatedWorld", {
|
||||
frameId: sameFrame,
|
||||
worldName: "bsk-design-probe",
|
||||
});
|
||||
await evaluate("scrollTo(0,10)", root, world.executionContextId);
|
||||
const frameScroll = await send("DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: styles,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
report.frameScroll = {
|
||||
documents: frameScroll.documents.map((d) => ({
|
||||
frameId: frameScroll.strings[d.frameId],
|
||||
scrollOffsetY: d.scrollOffsetY,
|
||||
})),
|
||||
canvases: decodedCanvases(frameScroll),
|
||||
local: await evaluate(
|
||||
"document.querySelector('canvas').getBoundingClientRect().toJSON()",
|
||||
root,
|
||||
world.executionContextId,
|
||||
),
|
||||
};
|
||||
check("frame scroll is local", report.frameScroll.local.y === 13);
|
||||
check(
|
||||
"frame snapshot stays document-relative after child scroll",
|
||||
report.frameScroll.canvases.find((n) => n.frameId === sameFrame).bounds[1] === 23,
|
||||
);
|
||||
await evaluate("scrollTo(0,0)", root, world.executionContextId);
|
||||
await evaluate(
|
||||
"document.querySelector('#same').style.transformOrigin='0 0';document.querySelector('#same').style.transform='scale(1.25)'",
|
||||
);
|
||||
const sameOwner = await send("DOM.getFrameOwner", { frameId: sameFrame });
|
||||
report.transformedOwner = await send("DOM.getBoxModel", {
|
||||
backendNodeId: sameOwner.backendNodeId,
|
||||
});
|
||||
check(
|
||||
"scaled frame owner content width",
|
||||
report.transformedOwner.model.content[2] - report.transformedOwner.model.content[0] === 375,
|
||||
);
|
||||
await evaluate("document.querySelector('#same').style.transform=''");
|
||||
const screenshot = await send("Page.captureScreenshot", { format: "png" });
|
||||
await writeFile(path.join(output, "frames.png"), Buffer.from(screenshot.data, "base64"));
|
||||
const before = (await send("Page.getFrameTree")).frameTree.frame;
|
||||
const oldDocument = (await send("DOM.getDocument")).root.backendNodeId;
|
||||
await evaluate("history.replaceState(null,'','#same-document')");
|
||||
const sameDocument = (await send("Page.getFrameTree")).frameTree.frame;
|
||||
const sameDocumentBackend = (await send("DOM.getDocument")).root.backendNodeId;
|
||||
await navigate(`${url}?noFrames=1&navigation=2`);
|
||||
const after = (await send("Page.getFrameTree")).frameTree.frame;
|
||||
const newDocument = (await send("DOM.getDocument")).root.backendNodeId;
|
||||
report.identity = { before, sameDocument, after, oldDocument, sameDocumentBackend, newDocument };
|
||||
check("same-document navigation preserves loader", before.loaderId === sameDocument.loaderId);
|
||||
check("cross-document navigation changes loader", before.loaderId !== after.loaderId);
|
||||
check("frame id survives document replacement", before.id === after.id);
|
||||
check(
|
||||
"document backend identity changes on navigation",
|
||||
oldDocument !== newDocument && oldDocument === sameDocumentBackend,
|
||||
);
|
||||
const documentTree = await send("DOM.getDocument");
|
||||
report.identity.htmlBeforeOpen = documentTree.root.children.find(
|
||||
(node) => node.nodeType === 1,
|
||||
).backendNodeId;
|
||||
const anchorNode = await send("DOM.querySelector", {
|
||||
nodeId: documentTree.root.nodeId,
|
||||
selector: "canvas",
|
||||
});
|
||||
const anchor = await send("DOM.describeNode", { nodeId: anchorNode.nodeId });
|
||||
await evaluate(
|
||||
"document.open(); document.write('<!doctype html><title>Replacement</title><canvas></canvas>'); document.close()",
|
||||
);
|
||||
report.identity.documentOpen = (await send("Page.getFrameTree")).frameTree.frame;
|
||||
const afterOpenTree = await send("DOM.getDocument");
|
||||
report.identity.documentOpenBackend = afterOpenTree.root.backendNodeId;
|
||||
report.identity.htmlAfterOpen = afterOpenTree.root.children.find(
|
||||
(node) => node.nodeType === 1,
|
||||
).backendNodeId;
|
||||
check(
|
||||
"document.open replaces documentElement identity",
|
||||
report.identity.htmlBeforeOpen !== report.identity.htmlAfterOpen,
|
||||
);
|
||||
try {
|
||||
const object = await send("DOM.resolveNode", { backendNodeId: anchor.node.backendNodeId });
|
||||
const connected = await send("Runtime.callFunctionOn", {
|
||||
objectId: object.object.objectId,
|
||||
functionDeclaration: "function(){return this.isConnected}",
|
||||
returnByValue: true,
|
||||
});
|
||||
report.identity.oldAnchorConnected = connected.result.value;
|
||||
} catch {
|
||||
report.identity.oldAnchorConnected = false;
|
||||
}
|
||||
check(
|
||||
"document.open invalidates old anchor even if document identifiers survive",
|
||||
report.identity.oldAnchorConnected === false,
|
||||
);
|
||||
report.counts = cdp.counts;
|
||||
report.checks = checks;
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ browser: report.browser.product, checks: checks.length, output })}\n`,
|
||||
);
|
||||
} catch (error) {
|
||||
report.error = error.stack;
|
||||
report.checks = checks;
|
||||
process.exitCode = 1;
|
||||
process.stderr.write(`${error.stack}\n`);
|
||||
} finally {
|
||||
await writeFile(path.join(output, "report.json"), JSON.stringify(report, null, 2));
|
||||
cdp.close();
|
||||
if (chrome && chrome.exitCode === null) {
|
||||
const exited = once(chrome, "exit");
|
||||
chrome.kill("SIGTERM");
|
||||
await Promise.race([exited, pause(2000)]);
|
||||
if (chrome.exitCode === null) {
|
||||
chrome.kill("SIGKILL");
|
||||
await exited;
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
await rm(profile, { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<meta charset="utf-8">
|
||||
<title>Embedded Canvas table reproduction</title>
|
||||
<style>
|
||||
body { font: 16px sans-serif; }
|
||||
iframe { display: block; width: 800px; height: 440px; border: 8px solid #ccc; }
|
||||
</style>
|
||||
<h1>Embedded table</h1>
|
||||
<button type="button">Page action</button>
|
||||
<iframe title="Table application"></iframe>
|
||||
<script>
|
||||
document.querySelector("iframe").src = `http://localhost:${location.port}/grid.html`;
|
||||
</script>
|
||||
</html>
|
||||
@@ -73,28 +73,6 @@ CSS-only hover menu looks like from here. It hovers a bounded set of likely trig
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
When `bsk observe` renders
|
||||
`@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered
|
||||
canvas content that is not represented by the text observation, not an interactable DOM control.
|
||||
Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or
|
||||
image-reading capability, do not take a screenshot and pretend to know its contents, and never guess
|
||||
coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
||||
|
||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain
|
||||
the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a
|
||||
point action and the image provides a clear target, bind that exact screenshot to the click command:
|
||||
|
||||
```bash
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing,
|
||||
navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture
|
||||
makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates.
|
||||
A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press
|
||||
workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and
|
||||
click/fill/hover/select reject them.
|
||||
|
||||
Escalate page reading only as needed:
|
||||
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
@@ -147,7 +125,6 @@ Required flags that are easy to get wrong:
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
bsk upload <ref> --file <path> bsk download <ref> --out <path>
|
||||
bsk click <surface-ref> --capture <id> --image-x <px> --image-y <px>
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
|
||||
@@ -136,18 +136,6 @@ pub struct ClickArgs {
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
|
||||
/// Capture id returned by `bsk screenshot --ref <surface>`.
|
||||
#[arg(long)]
|
||||
pub capture: Option<String>,
|
||||
|
||||
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
|
||||
#[arg(long = "image-x", allow_hyphen_values = true)]
|
||||
pub image_x: Option<f64>,
|
||||
|
||||
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
|
||||
#[arg(long = "image-y", allow_hyphen_values = true)]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -159,14 +147,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let point_arg_count = usize::from(args.capture.is_some())
|
||||
+ usize::from(args.image_x.is_some())
|
||||
+ usize::from(args.image_y.is_some());
|
||||
if point_arg_count != 0 && point_arg_count != 3 {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"--capture, --image-x, and --image-y must be given together"
|
||||
)));
|
||||
}
|
||||
let params = ClickParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
@@ -180,9 +160,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Some(modifiers)
|
||||
},
|
||||
timeout_ms: Some(args.timeout),
|
||||
capture_id: args.capture.clone(),
|
||||
image_x: args.image_x,
|
||||
image_y: args.image_y,
|
||||
};
|
||||
let reply: ClickResult = call(
|
||||
info.sock_path,
|
||||
@@ -206,14 +183,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
"click ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
if let Some(capture_id) = &reply.capture_id {
|
||||
println!(
|
||||
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
|
||||
capture_id,
|
||||
reply.image_x.unwrap_or_default(),
|
||||
reply.image_y.unwrap_or_default()
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,7 +40,6 @@ use bsk_protocol::ErrorCode;
|
||||
pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
@@ -65,11 +64,6 @@ pub mod reason {
|
||||
pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed";
|
||||
pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown";
|
||||
pub const TRANSFER_TIMEOUT: &str = "transfer_timeout";
|
||||
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
|
||||
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
|
||||
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
|
||||
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
|
||||
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
|
||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||
}
|
||||
@@ -305,13 +299,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||
summary: "snapshot ref does not support this operation",
|
||||
hint: Some(
|
||||
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
|
||||
summary: "target element is not a <select>",
|
||||
hint: Some(
|
||||
@@ -354,41 +341,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
|
||||
summary: "surface capture transaction was not found",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and use its capture id with coordinates from that image",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
|
||||
summary: "surface capture transaction has expired",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
|
||||
summary: "surface capture transaction was already used",
|
||||
hint: Some(
|
||||
"each capture id is single-use; take a fresh surface screenshot before another point action",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
|
||||
summary: "page state changed since the surface screenshot",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
|
||||
summary: "surface image coordinate is invalid",
|
||||
hint: Some(
|
||||
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(
|
||||
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
|
||||
reason::RECORD_START_PAGE_UNREACHABLE,
|
||||
@@ -612,18 +564,6 @@ mod tests {
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_capability_denied_explains_visual_surface_refs() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(info.summary, "snapshot ref does not support this operation");
|
||||
assert!(
|
||||
info.hint.unwrap().contains("capture id"),
|
||||
"expected visual-surface-specific hint"
|
||||
);
|
||||
assert_eq!(info.exit_code, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_busy_overrides_timeout_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||
@@ -724,25 +664,6 @@ mod tests {
|
||||
assert!(info.summary.contains("timed out after browser dispatch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_capture_stale_requests_a_fresh_screenshot() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"page state changed since the surface screenshot"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
|
||||
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
|
||||
assert_eq!(info.summary, "surface image coordinate is invalid");
|
||||
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_not_found_overrides_not_found_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||
|
||||
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
.map_err(CliError::Local)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let mut json = serde_json::json!({
|
||||
let json = serde_json::json!({
|
||||
"tab_id": reply.tab_id,
|
||||
"width": reply.width,
|
||||
"height": reply.height,
|
||||
@@ -67,10 +67,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
"path": out_path.to_string_lossy(),
|
||||
"byte_size": bytes.len(),
|
||||
});
|
||||
if let Some(capture) = &reply.capture {
|
||||
json["capture"] = serde_json::to_value(capture)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json)
|
||||
@@ -79,12 +75,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
}
|
||||
Format::Human => {
|
||||
println!("{}", out_path.display());
|
||||
if let Some(capture) = &reply.capture {
|
||||
println!(
|
||||
"capture={} surface={} coordinate_space={} expires_at={}",
|
||||
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,29 +237,6 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_surface_capture_image_coordinates() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"click",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"160.5",
|
||||
"--image-y",
|
||||
"-1",
|
||||
]);
|
||||
let Command::Click(args) = cli.command else {
|
||||
panic!("expected click command");
|
||||
};
|
||||
assert_eq!(args.capture.as_deref(), Some("sc_test"));
|
||||
assert_eq!(args.image_x, Some(160.5));
|
||||
assert_eq!(args.image_y, Some(-1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
|
||||
@@ -238,7 +238,6 @@ async fn screenshot_returns_image_base64_with_dimensions() {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -287,7 +286,6 @@ async fn screenshot_forwards_ref_to_extension() {
|
||||
height: 60,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
|
||||
@@ -354,9 +354,6 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
used_selector: None,
|
||||
x: 12.5,
|
||||
y: 34.0,
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -376,9 +373,6 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -616,9 +610,6 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
click_count: None,
|
||||
modifiers: None,
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -16,13 +16,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"capture_id": {
|
||||
"description": "Short-lived capture id returned by a visual Surface screenshot.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"click_count": {
|
||||
"description": "Number of consecutive mouse presses (double-click = 2).",
|
||||
"type": [
|
||||
@@ -32,22 +25,6 @@
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
},
|
||||
"image_x": {
|
||||
"description": "X coordinate in the capture image's pixel coordinate space.",
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"image_y": {
|
||||
"description": "Y coordinate in the capture image's pixel coordinate space.",
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
|
||||
@@ -8,12 +8,6 @@
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"capture_id": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"description": "Native JS dialogs observed and auto-handled during this call.",
|
||||
"type": "array",
|
||||
@@ -21,20 +15,6 @@
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"image_x": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"image_y": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
|
||||
@@ -8,16 +8,6 @@
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"capture": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/SurfaceCaptureInfo"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -119,31 +109,6 @@
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"SurfaceCaptureInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"coordinate_space",
|
||||
"expires_at",
|
||||
"id",
|
||||
"surface_ref"
|
||||
],
|
||||
"properties": {
|
||||
"coordinate_space": {
|
||||
"type": "string"
|
||||
},
|
||||
"expires_at": {
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"surface_ref": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,15 +67,6 @@ pub struct ClickParams {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
/// Short-lived capture id returned by a visual Surface screenshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture_id: Option<String>,
|
||||
/// X coordinate in the capture image's pixel coordinate space.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_x: Option<f64>,
|
||||
/// Y coordinate in the capture image's pixel coordinate space.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -89,12 +80,6 @@ pub struct ClickResult {
|
||||
/// agents can correlate with a follow-up `tool.screenshot`.
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_x: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_y: Option<f64>,
|
||||
/// Native JS dialogs observed and auto-handled during this call.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
@@ -303,9 +288,6 @@ mod tests {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: Some("sc_test".into()),
|
||||
image_x: Some(160.0),
|
||||
image_y: Some(90.0),
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||
|
||||
@@ -233,20 +233,10 @@ pub struct ScreenshotResult {
|
||||
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
|
||||
pub format: String,
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub capture: Option<SurfaceCaptureInfo>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SurfaceCaptureInfo {
|
||||
pub id: String,
|
||||
pub surface_ref: String,
|
||||
pub coordinate_space: String,
|
||||
pub expires_at: u64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -328,12 +318,6 @@ mod tests {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: Some(SurfaceCaptureInfo {
|
||||
id: "sc_test".into(),
|
||||
surface_ref: "@e3".into(),
|
||||
coordinate_space: "capture-image-pixel".into(),
|
||||
expires_at: 1_700_000_030_000,
|
||||
}),
|
||||
dialogs: vec![],
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
# Visual observation:验收清单
|
||||
|
||||
新 agent 先读 [架构与 PR 划分](visual-observation.md),再按本清单验收当前 PR;无需聊天记录,也无需另写报告、逐项编号或填写模板。历史结果见 [PR 0 证据](visual-observation-evidence.md),不能代替本次执行。
|
||||
|
||||
## 怎么开始
|
||||
|
||||
- 查看分支、HEAD 和工作区改动,确认当前做哪个 PR。PR 0 基于 main `3c5f838`;后续模块对照其前置基线,最终回归对照 main。注意未跟踪文件也属于改动。
|
||||
- 当前仅有设计、合成 fixture 和诊断脚本;未来模块尚未实现。只验收当前阶段及受影响的已有功能,不要求提前完成后续 PR。
|
||||
- 测试放在模块附近,调用实际生产代码。未接线模块可用人工输入和 CDP stub;实现 PR 写明运行命令即可,不强制新建指定目录、命名或诊断 API。
|
||||
- 浏览器验收前核实 CLI 版本,并重新 build、确认扩展 reload。PATH 中的 bsk 曾经比仓库版本旧,不能只凭版本号认定运行了目标代码。
|
||||
- 不 commit/push。截图、原始页面内容和一次性实验放临时目录;只把长期需要的设计、fixture 和测试留在仓库。
|
||||
|
||||
## 现有验证入口
|
||||
|
||||
从仓库根目录运行,命令成功退出且相关断言通过才算成功:
|
||||
|
||||
```sh
|
||||
pnpm vom:test
|
||||
pnpm --filter @browser-skill/extension test src/tools/__tests__/frame-geometry.test.ts src/tools/__tests__/element-geometry.test.ts src/tools/__tests__/observation.test.ts src/tools/__tests__/snapshot-ref.test.ts src/tools/__tests__/interaction.test.ts src/tools/vom src/session-manager/__tests__/ref-store.test.ts
|
||||
pnpm exec biome check apps/extension/test-fixtures/visual-observation
|
||||
VISUAL_EVIDENCE="$(mktemp -d /tmp/bsk-visual-acceptance.XXXXXX)"
|
||||
node apps/extension/test-fixtures/visual-observation/probe.mjs "$VISUAL_EVIDENCE/browser-contract"
|
||||
node apps/extension/test-fixtures/visual-observation/measure-render.mjs "$VISUAL_EVIDENCE/performance"
|
||||
```
|
||||
|
||||
probe 需要 Node 22+ 和 Chrome,配置见 [fixture 说明](../../apps/extension/test-fixtures/visual-observation/README.md)。它验证浏览器事实,不代表后续生产模块已经通过。measure-render 用于旧 renderer 对照,不要求修复历史复杂度。
|
||||
|
||||
实现 PR 还需运行所属模块的新测试、`pnpm --filter @browser-skill/extension compile`、`pnpm ext:test` 和 `pnpm ext:build`。仅修改文档时检查链接和内容即可,不必重跑浏览器。已有失败应说明,不在范围外顺手修复。
|
||||
|
||||
## 每个 PR 怎么算完成
|
||||
|
||||
### PR 0:设计与复现
|
||||
|
||||
只有文档、合成 fixture 和诊断变动;架构、边界及后续 PR 的完成条件明确。现有测试通过;probe 的 checks 非空且全部通过(本版 39 项);measure-render 产生五组 wide/deep 结果。
|
||||
|
||||
按下方浏览器步骤,在 main 重现“表格已绘制,但 observe 没有视觉入口”。合成页面和原始 iWiki 分别确认。原始页面未加载、无权限或缺入口时说明阻塞,不能算已复现。入库内容不含内部页面地址、标识或截图。
|
||||
|
||||
### PR 1:几何与身份
|
||||
|
||||
数值误差:纯函数输入允许 1e-6 CSS px;真实浏览器每条边允许 0.25 CSS px。坐标类型不能混传;动态 frame 归属需运行时检查。
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| top document bounds=(30,50,120,40),document scroll=(10,20),viewport=800×600 | top border region=(20,30,120,40);不受 backing store 尺寸影响 |
|
||||
| 使用 frame.html:local Canvas=(17,23,120,40),same owner content origin=(50,610);nested origin=(87,707);OOPIF origin=(410,610) | top origins 分别=(67,633)、(104,730)、(427,633);不能重复给 nested 加父偏移;owner border 不能代替 content |
|
||||
| child scrollY=10;same frame owner origin 不变 | Canvas top y=623,local y=13;snapshot document y=23;scroll 只扣一次 |
|
||||
| same iframe 以自身左上角为原点 scale(1.25),原 border origin=(40,600),border=10 | owner content origin=(52.5,612.5),Canvas top=(73.75,641.25),尺寸150×50 |
|
||||
| Canvas content 120×40,border 每边5;overflow owner CSS width60/height30/border5/padding4 | Canvas region 宽130高50,live 验证同 box;owner clientRect=(5,5,68,38),不能使用78×48 border范围作为clip |
|
||||
| DPR=1/1.5/2,CSS Canvas120×40;UI zoom与CSS zoom分别处理 | DPR不直接除掉 CSS rect;raster 预测与实际图片分别记录;不以layout/css宽度比作为万能DPR |
|
||||
| 分别构造缺失 owner、失效 frame、旋转、透视、mask、clip-path、活动pinch、变换的overflow祖先 | 前两项 geometry-unavailable,后六类 geometry-unsupported;无伪造精确 region ref、无按border猜偏移 |
|
||||
| 同一操作并发请求同一frame 20次,再开启第二次操作 | 第一次每个metrics来源、owner quad、frame投影最多构建/请求一次;第二次重新获取;异常promise不泄漏到其他操作 |
|
||||
|
||||
### PR 2:采集与共享事实
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| target A/B 各含 backendNodeId=7,frame/document不同 | 两个lookup结果独立;AX join、overlay排除和父子查找不串target |
|
||||
| 4个frame分属2个target,其中一个target有3份document;均成功采集 | 每target恰好一次DOMSnapshot;每frame恰好一次AX批次;失败重试不混入成功请求数;无按frame重复snapshot |
|
||||
| 采集中模拟frame detach或loader/documentElement更换 | 丢弃受影响document的混合事实并标记原因;其他完整document可保留;不得把新AX接到旧DOM |
|
||||
| 输入10,000深链、10,000兄弟、孤儿及A→B→A环 | 无栈溢出/无限循环;孤儿/环有明确不完整状态;不生成完整祖先证据;context推导每有效节点至多一次 |
|
||||
| opacity0祖先;visibility:hidden祖先+visible子;aria-hidden/inert祖先 | 分别得到视觉透明、子可见、语义/交互状态独立;不折叠为同一个excluded |
|
||||
| 一个frame采集失败,其他frame有语义节点;全部成功但无Canvas | 前者partial且保留其他语义;后者empty而非unavailable;错误原因可追踪到frame及阶段 |
|
||||
| 注入CDP调用记录和hover探针;纯计算中途取消 | 所有静态DOM/AX捕获先于hover输入;取消后下一工作块不继续处理,未执行的输入/截图为0;取消不变成普通空结果 |
|
||||
|
||||
### PR 3:视觉发现
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| index.html?noFrames=1 的plain、named、aria-hidden、inert、pointer-none、visibility-override、hidden-overridden、fallback | 八个均为候选;无label不影响发现,有AX/fallback不抑制候选 |
|
||||
| opacity-parent、display-none、visibility-hidden、zero、full-clip | 五个均无候选,原因分别能追踪到视觉/几何规则;不能把aria-hidden列为原因 |
|
||||
| partial-clip、axis-clip、border-clip、border-canvas | crop尺寸分别60×30、60×40、64×34、130×50;边界来自client box/同种Canvas box |
|
||||
| 同frame中删除语义parent,再保持DOM/几何不变 | 候选仍存在、地址不变;语义context可变为null;frame owner本身无法解析时不是该用例 |
|
||||
| 连续两次输入同一只读Facts;对Facts启用深冻结 | 候选内容一致,无输入mutation,无CDP调用;去重前stack-a/b仍是两个候选 |
|
||||
| 普通候选部分越出viewport;完全越界;overlay-owned Canvas;unsupported clip | 分别交集区域、无候选、排除、明确unsupported状态;不把未知状态伪装成empty |
|
||||
|
||||
### PR 4:选择与预算
|
||||
|
||||
预算按每个输出行的 `ceil(line.length/4)` 求和(UTF-16 length),包括头部和截断提示。H 为完整协议头成本,V 为固定行 `[visual] Canvas regions present; use screenshot` 的成本。
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| 同context:无名1000×800候选A,有名10×10候选B,K=1 | 选A;名称不能压倒区域面积 |
|
||||
| 同document/parent且crop完全相同的A/B;另一个parent的C;crop右边相差0.01的D;相邻tile E | A/B仅输出一个代表;C/D/E独立保留;不输出layers=N作为业务解释 |
|
||||
| 面积相同的候选随机打乱20次,地址/位置保持不变 | 排序相同;tie-break顺序固定为frameId、top y、left x、backendNodeId(同一观察内);去重代表取该顺序最小者;字符串按代码单元字典序、数值按数值升序 |
|
||||
| 50,001个不同key,最后一个为最大Canvas,K=1 | 最后最大者仍被选中;dedup key峰值≤50,000;标记dedup-degraded;未因容量耗尽停扫描 |
|
||||
| 100,000候选、K=1/32/512;另测K=0 | 选中数量≤K、heap峰值≤K、exact-key之外的模糊pair比较为0;K=0不产生ref候选;结果与独立全排序小规模oracle一致 |
|
||||
| 带blocking layer的纯输入,候选context在L1、背景、未知各一个 | L1优先,背景不标为物理不可见,未知不伪装L1;同层再按面积及固定 tie-break 排序;不修改Facts |
|
||||
| 预算H-1 | 新observe规划返回insufficient-budget,工具映射invalid_params并给minimum=H;无新文本/ref提交;snapshot旧路径不改变 |
|
||||
| 有候选,预算H、H+V-1、H+V | 前两个输出合法协议头、无悬空ref、truncated=true;H+V能保留固定视觉提示;全部账本≤预算 |
|
||||
| 没有显式maxTokens,候选足够多 | 视觉附加部分cost≤2048;最终视觉条目≤512;不是对整个旧语义输出强加2048 |
|
||||
| 标签含200个Unicode code point、引号、换行、代理对emoji | 标签先截至160 code point再JSON escape,按最终行核算;无断开的代理对,无换行注入额外条目;写不下的行不产生ref |
|
||||
| finite预算取H+V、64、128、256、2048(小于H时拒绝),混合长短条目 | 每次总账本≤budget;输出决定可重复;已知omitted才给精确数,候选/去重组/上游未知数量不混加 |
|
||||
|
||||
### PR 5:类型化 ref
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| 注册DOM和visual-region两个判别目标 | kind决定操作资格;类型测试不能构造visual+interact矛盾状态;不额外接受调用者capability数组 |
|
||||
| 同名ref置于两个session/tab;输入@e1与e1 | canonical形式可解析;不同session/tab不串目标;未知目标为既有not_found类错误 |
|
||||
| 对visual ref调用click/fill/hover/select/get_html/人工帮助DOM定位入口 | 全部按主设计拒绝;CDP输入、DOM HTML读取、DOM高亮等该工具副作用次数为0;不能偷偷降为anchor操作 |
|
||||
| loader变化、target重attach、documentElement替换;另测history.replaceState不改身份 | 前三者拒绝旧目标;最后不单凭URL变化拒绝;实际anchor仍需连接 |
|
||||
| document.open保留loader/document backend但替换html和Canvas | 旧ref失效;只比较loader的实现必须被此测试击败 |
|
||||
| 第二次observe重新绑定e1;第一次ref字符串再次输入 | 明确验证“最新观察映射”而非声称能识别旧字符串;新映射仅来自成功提交;旧DOM工具回归通过 |
|
||||
|
||||
### PR 6:截图执行
|
||||
|
||||
首次 clipScale 为 `min(1, 2048/(max(w,h)*r), sqrt(4000000/(w*h*r*r)))`;r 来自实际 metrics 适配,最终仍检查真实 PNG 尺寸。
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| visual ref解析成功并截图;DOM ref另测 | visual路径scrollIntoView次数0;DOM路径保持BASE策略;visual实时box种类与观察一致 |
|
||||
| 同anchor四边分别移动0.249、0.25、0.251 CSS px;分别改尺寸/裁剪策略 | 前两种仅数值误差可接受,0.251拒绝;document/裁剪策略变化不受容差豁免;不靠old∩live返回小片段 |
|
||||
| 保持身份/几何,重绘Canvas颜色;另移除Canvas | 重绘允许取得当前图像;移除拒绝;不承诺像素快照或把颜色改变误报为stale |
|
||||
| css4096×2048,rasterEstimate2;css4000×4000,rasterEstimate1 | 首次clipScale分别0.25、0.5(不放大,取edge/area/1最小值);实际成功尺寸每边≤2048、面积≤4,000,000 |
|
||||
| mock第一次图片3000×3000,第二次2000×2000;再测两次均3000×3000 | 第一组重试一次并成功,第二组恰好两次后pixel-budget-exceeded;不能无限重试或返回超限图 |
|
||||
| 空data、非法PNG头、0尺寸、负/非有限几何 | 明确失败,不能猜测width/height;此项不要求将PNG头检查宣称为完整解码校验 |
|
||||
| capture前取消;overlay抑制后CDP失败;采集时出现dialog | 取消前不再capture;成功/失败/取消均恢复overlay;dialog机制保持既有契约;无额外输入 |
|
||||
|
||||
### PR 7:observe 接线
|
||||
|
||||
| 怎么测 | 什么算成功 |
|
||||
|---|---|
|
||||
| 固定SemanticScene,无视觉候选;普通预算,与BASE replay同一输入 | 语义文本、ref相对顺序、role/name一致;不因为新通道重跑semantic pipeline |
|
||||
| 同SemanticScene加两个视觉候选,大预算 | 原DOM ref顺序不变,视觉ref在其后;每个输出ref均可在registry找到同目标,registry无未输出新ref |
|
||||
| DOM文本多到耗尽除预留外的预算 | 固定视觉提示可见(预算允许时);truncated=true;不是DOM stopped后无条件跳过视觉提示 |
|
||||
| 视觉capture unavailable但语义成功 | 语义结果保留,观察状态能区分empty/unavailable/truncated;debug reason准确且无原始DOM泄漏 |
|
||||
| snapshot、recording调用、hover probe on/off与BASE对比 | snapshot不新增visual;recording不隐式启用发现;probe默认off,on时仍在静态采集后;相应既有测试通过 |
|
||||
| blocking layer混合DOM/visual、重复候选、不同预算 | 输出层策略与选择模块的活跃层策略一致;头、摘要、条目、occlusion/omitted行均纳入账本;无悬空ref |
|
||||
|
||||
### PR 8:整体验收
|
||||
|
||||
复跑上述模块测试、下方性能对照和浏览器场景,确认原始 iWiki 的缺口已解决。检查没有站点特判、重复的几何/预算权威实现或遗留的替代路径;没有顺带扩展到 OCR、逻辑画布聚合或旧算法重写。不能只凭合成页面通过宣布原始问题解决。
|
||||
|
||||
## 性能怎么验
|
||||
|
||||
在相关模块实现时检查,不拖到最后统一补。旧路径和新增 Canvas 成本分开看:
|
||||
|
||||
- **旧路径**:同输入对照 main/前置基线,已有 render/context/sibling 工作量、语义构建次数和 DOM 操作请求不增加;保留 #165 的回归。不要求消除旧算法的历史二次项。
|
||||
- **无 Canvas**:允许必要检测与共享几何成本,但不构建视觉专用索引、不发视觉专用 CDP 请求、不重复跑语义链路。
|
||||
- **新增树处理**:deep/wide 节点数取 1k、10k、100k;每节点 context 至多推导一次,每条边每个索引 pass 至多访问一次,pass 数不随规模增加。无递归溢出、每候选完整祖先遍历或全祖先数组复制。
|
||||
- **选择**:候选数取 1k、10k、100k,K 取 1、32、512;堆峰值不超过 K,去重 key 不超过 50,000,无模糊两两比较。比较计数符合具体算法推导的上界;小规模结果与独立全排序对照一致,容量耗尽后仍能选中最后的大图。
|
||||
- **请求与取消**:每 target 一次 snapshot、每 frame 一次 AX 批次;单次操作内几何请求共享,不跨操作复用 live cache。几何最大并发 4,新增计算每块最多 256 项并实际让出事件循环,取消后不再启动下一块。
|
||||
- **接线后耗时**:同机器、浏览器和固定内容,分别测无 Canvas 页与合成表格页。各预热 3 次,再基线/新版本交替各 20 次,比较中位数和 p95;旧阶段与新增成本分开。旧阶段中位数增加超过 max(基线 10%, 2ms) 时复测,两轮均超过且可归因于改动则未通过;噪声无法排除则说明尚未确认。操作数仍须满足前述条件,不能用耗时阈值代替它们。
|
||||
|
||||
计数可以由测试插桩取得,不必为验收增加生产接口。性能结果只需说明旧路径是否回退、新增路径的规模与成本,不需要另填表格。
|
||||
|
||||
## 浏览器怎么验
|
||||
|
||||
### 合成表格(PR 0 复现;PR 7/8 验证修复)
|
||||
|
||||
1. 核实 `bsk --version`、`bsk browsers`、`bsk status`;运行 `pnpm ext:build`,确认用户已 reload `apps/extension/dist/chrome-mv3`。
|
||||
2. 单独启动服务器:`python3 -m http.server 4177 --bind 127.0.0.1 --directory apps/extension/test-fixtures/visual-observation`。
|
||||
3. `bsk session start --width 1440 --height 1100`,使用返回的 session id 打开下列页面。尖括号替换为实际值,每条命令传同一 session。
|
||||
|
||||
```sh
|
||||
bsk navigate http://127.0.0.1:4177/table.html --session <session> --json
|
||||
bsk observe --session <session> --json
|
||||
bsk screenshot --session <session> --out <临时目录>/table.png --json
|
||||
```
|
||||
|
||||
4. 把窗口放前台,实际看到三列六行后再采集 observe。load 完成、AX 工具栏出现或背景占位截图都不能证明表格已经绘制。
|
||||
5. main 应有 Page action、Insert record、Filter 三个 DOM 控件,truncated=false,但无表格视觉入口。这是 PR 0 的缺口复现。
|
||||
6. 新实现应保留三个控件,并增加 visual ref。用本次返回的 ref 执行 `bsk screenshot --ref <ref> --session <session> --out <临时目录>/crop.png`,实际查看图片:包含 Canvas column A/B/C 和六行,裁剪对应 780×360 CSS px 的表格区域,不能错到工具栏或 iframe 边框。PNG 满足每边 2048、总像素 4,000,000 的上限。无需生成单元格语义。
|
||||
|
||||
### 几何与动态场景
|
||||
|
||||
PR 1 验证实际生产几何模块;PR 6 验证截图执行;PR 8 复验完整链路。未接 observe 时可注入人工 facts/ref,但必须调用实际模块,不能以独立 probe 代替实现测试。
|
||||
|
||||
- 必测缩放集合:DPR 1/1.5/2 × top/frame;真实 UI zoom 100%/90% × top/frame;CSS zoom 1.25 的 top Canvas;frame scale 1.25 的 same/nested/OOPIF。按 PR 1 的坐标预期核对。OOPIF 需确认实际 iframe target,跨 origin 本身不够。
|
||||
- UI zoom 要通过浏览器菜单或原生快捷键确认,不能以 CSS zoom 或 pinch emulation 替代。旋转、mask、clip-path、活动 pinch、变换 overflow 按不支持策略降级,不输出猜测的精确 ref。
|
||||
- 使用 `index.html?noFrames=1` 的 `#plain`,每个实验从 reload 后的新 ref 开始:平移 1px、宽度改为 121px → 旧 ref 截图要求重新观察;移除节点或 document.open → 旧目标失效;仅重绘颜色 → 截图成功且是新颜色。不要在重新 observe 后拿重用的 eN 字符串声称验证了旧 ref。
|
||||
- visual ref 的 click/fill/hover/select/get_html/DOM 高亮拒绝测试只在合成页或自动测试运行,确认拒绝前没有派发输入、读 HTML 或高亮副作用。
|
||||
|
||||
### 原始 iWiki(PR 0 与 PR 8)
|
||||
|
||||
入口与就绪说明在本机临时文件 `/private/tmp/bsk-visual-design/private-reproduction.md`。文件丢失或登录失效时向用户取得入口/权限,说明这一项未完成;继续独立的合成验证。
|
||||
|
||||
保持只读。前台实际看到表格后采集 observe 和截图:PR 0 确认缺口;PR 8 确认外围 DOM 控件仍在、新 visual ref 能截到实际表格。不要编辑单元格,不要求原页面永久维持历史的 63 个 ref。私有内容不入库。
|
||||
|
||||
结束后停止自己的 session、服务器及临时 Chrome,恢复修改过的 zoom/emulation,不关闭其他任务资源。
|
||||
|
||||
## 做完怎么回复
|
||||
|
||||
直接告诉用户:验了哪个 PR、跑了什么、是否符合上述成功条件;若有失败或未做的检查,说明具体原因。必要时附关键测试输出或截图路径。无需单独报告文件、hash 清单、状态统计或全量日志归档;没运行、被跳过或受阻的项目不要写成通过。
|
||||
@@ -0,0 +1,235 @@
|
||||
# Visual observation:PR 0 验证结果与架构决策
|
||||
|
||||
日期:2026-09-08。生产基线:main `3c5f838`。分支:`codex/visual-observation-design`。
|
||||
|
||||
本次只新增设计、合成 fixture 和诊断工具。没有生产算法变更,没有 commit/push。
|
||||
总体设计及后续 PR 依赖见 [架构与实施计划](visual-observation.md)。
|
||||
本文是历史证据;新会话请按 [验收清单](visual-observation-acceptance.md) 执行相关检查并说明结果,不继承本文的通过结论。
|
||||
|
||||
## 1. 环境与证据等级
|
||||
|
||||
| 环境 | 实际版本/说明 |
|
||||
|---|---|
|
||||
| PATH 中的 bsk | 0.1.7,未用于有效页面基线 |
|
||||
| 仓库已有 target/debug/bsk | 0.2.0;`update --check` 报告发布最新版也是 0.2.0,没有安装更新 |
|
||||
| daemon / connected extension | 0.2.0,protocol 1.1 |
|
||||
| 页面基线 Chrome | 151.0.0.0,由 bsk browsers 报告 |
|
||||
| 生产扩展构建 | `pnpm ext:build`,用户 reload 当前 main 基线构建后采集 |
|
||||
| 独立协议实验 Chrome | 152.0.7977.76,临时无登录 profile;与用户浏览器不是同一个运行实例 |
|
||||
| 基线测试 | VOM 47 tests / 2 files;相关扩展 217 tests / 14 files;全部通过 |
|
||||
|
||||
页面结果是实际 bsk + reload 扩展的结果。协议实验是独立 Chrome 的真实 CDP 响应。
|
||||
临时 replay 是将合成响应送入 main adapter 的诊断,不代表另一条真实浏览器完整运行。
|
||||
不同证据不混用,版本号本身不证明构建 commit。
|
||||
|
||||
所有内部页面原文、地址、iframe 标识、截图和用户信息仅保存在本机临时目录,不进入仓库。
|
||||
本文仅保存匿名化事实,仓库 fixture 完全独立构造。
|
||||
|
||||
## 2. 原始问题已确认:表格加载完成后仍有观察缺口
|
||||
|
||||
有效就绪条件:前台可见页面已经绘制出网格、列标题和六行表格。不能使用以下替代条件:
|
||||
|
||||
- `document.readyState === complete`。
|
||||
- AX 已有嵌入应用的工具栏。
|
||||
- 页面出现 iframe。
|
||||
- 背景截图仍为站点加载 logo。
|
||||
|
||||
本次初始后台窗口恰好出现“AX 已有工具栏,图像仍为加载占位”的情况。这些样本已排除。
|
||||
在前台绘制完成后重新取得匹配的截图和 observe,结果为:
|
||||
|
||||
| 事实 | 结果 |
|
||||
|---|---|
|
||||
| 实际截图 | 可见列标题、网格与六行表格 |
|
||||
| main observe | 63 个 ref,含外围和嵌入应用工具栏 |
|
||||
| truncated | false |
|
||||
| 表格列内容 | 没有相应文本表示 |
|
||||
| visual ref | 没有 |
|
||||
| 直接打开已观察到的嵌入应用 | 发现一个无 aria-label 的 Canvas,CSS 尺寸 1450×835、backing store 2900×1670,DPR 2 |
|
||||
|
||||
直接打开嵌入应用用于验证其渲染介质;其宽度变化是独立 tab 布局,不能拿这组坐标当父页面中的 crop。
|
||||
父页面的截图/VOM 对照与直接嵌入页的 DOM 证据分别记录。
|
||||
|
||||
结论:主要能力缺口是视觉内容没有观察入口;不属于这次有效样本的文本预算耗尽。
|
||||
新架构不能承诺从 Canvas 自动恢复单元格语义;正确结果是保留工具栏,并提供可截图的区域入口。
|
||||
|
||||
## 3. 合成最小复现
|
||||
|
||||
`table.html` → 跨站点 `grid.html`:一个 iframe、DOM 工具栏、一个无名 Canvas。
|
||||
Canvas 绘制三列、六行,所有文字与原页面无关。
|
||||
|
||||
main 实际输出的关键部分:
|
||||
|
||||
```text
|
||||
@view 1440x923
|
||||
@e1 button "Page action"
|
||||
Iframe "Table application"
|
||||
navigation
|
||||
@e2 button "Insert record"
|
||||
@e3 button "Filter"
|
||||
```
|
||||
|
||||
`ref_count=3`,`truncated=false`。截图已人工确认包含 `Canvas column A/B/C` 和六行。
|
||||
这些像素文字既没有文本表示,也没有 visual ref。该最小复现已覆盖原场景的关键结构,
|
||||
无需站点权限或生产选择器。
|
||||
|
||||
## 4. CSS 与语义/交互分离
|
||||
|
||||
真实浏览器实验确认:
|
||||
|
||||
- aria-hidden 祖先、inert 祖先、pointer-events:none 不会使 Canvas 自动视觉消失。
|
||||
- `visibility:hidden` 祖先下的 `visibility:visible` 子 Canvas 可以恢复可见。
|
||||
- `hidden` 元素被 `display:block` 覆盖时仍可绘制。
|
||||
- opacity:0 祖先会使子树透明;不能仅检查 Canvas 自己的 opacity。
|
||||
- display:none 无 layout bounds;visibility:hidden 可以仍有 bounds。
|
||||
- AX fallback 文本不包含绘制的 `PIXEL TABLE 42`,不能用 fallback 存在作为发现去重条件。
|
||||
|
||||
决定:事实层分别保存 computed visibility、透明继承、语义暴露和交互属性。
|
||||
视觉发现不读取 semantic keep/drop 结果;祖先索引只能传播符合属性语义的状态。
|
||||
|
||||
`checkVisibility()` 的实验作用仅限 CSS suppression,不用它证明 overflow、viewport 或遮挡结果。
|
||||
常规 overflow 由独立几何规则负责;任意遮挡不进入首版精确能力范围。
|
||||
|
||||
## 5. 单位、frame 投影与截图
|
||||
|
||||
### 5.1 DPR 与截图像素是不同契约
|
||||
|
||||
独立 Chrome 上对相同 120×40 CSS clip 的结果:
|
||||
|
||||
| Emulated DPR | DOMSnapshot Canvas w×h | Screenshot w×h | layoutViewport/cssLayoutViewport 宽度比 |
|
||||
|---|---|---|---|
|
||||
| 1 | 120×40 | 120×40 | 1 |
|
||||
| 1.5 | 120×40 | 180×60 | 1 |
|
||||
| 2 | 120×40 | 240×80 | 1 |
|
||||
|
||||
因此该比值不能用作通用截图 DPR。原始 DOMSnapshot 单位与截图 raster scale 必须分别适配,
|
||||
输出 PNG 尺寸是最终预算事实。不能把本组 DPR emulation 结果泛化为所有浏览器缩放组合。
|
||||
|
||||
### 5.2 三种缩放分别记账
|
||||
|
||||
- CSS `zoom:1.25`:120px 宽 Canvas 的布局 bounds 为 150px,运行时几何同步变化。
|
||||
- Emulation.setPageScaleFactor(1.5):记录为 page-scale/pinch 类证据,不充当 UI zoom。
|
||||
- 用户 Chrome 原生 UI 缩放到 90%:界面明确显示 90%,DPR 从 2 变为约 1.8,innerWidth 从 1440 变为 1600,visualViewport.scale 仍为 1;Canvas CSS 宽约 120.0000076,截图仍是 2880px 宽。main VOM 视口为 1600×1025。已恢复 100%。
|
||||
|
||||
现有 bsk 的 Meta+- CDP 输入未改变浏览器缩放,所以该无效果尝试不算缩放验证;
|
||||
有效样本来自原生浏览器 UI 操作。真实 UI zoom 仅验证了上述点,不能宣称完整 zoom 矩阵已通过。
|
||||
|
||||
决定:使用 CSS 视口空间作为发现/截图区域公共契约;精确单位 adapter 的实现还必须在 PR 1
|
||||
跑计划内组合回归,不用一个比例同时解决所有协议字段。
|
||||
|
||||
### 5.3 Frame 空间与 main 已知偏移
|
||||
|
||||
fixture local Canvas 原点为 (17,23)。
|
||||
|
||||
| 场景 | 实测 owner content 原点(target root) | 正确 top Canvas 原点 | main parser replay |
|
||||
|---|---|---|---|
|
||||
| 同 target frame,10px border | (50,610) | (67,633) | (57,623) |
|
||||
| 上述 frame 内 nested,另有 7px border | (87,707) | (104,730) | (87,713) |
|
||||
| OOPIF,10px border | (410,610) | (427,633) | 本表不混入非完整 replay 的 OOPIF 结果 |
|
||||
|
||||
根 target 的 DOMSnapshot 含三份 document(top/same/nested);OOPIF 在单独 target 中只返回自己的 document。
|
||||
probe 通过实际 `TargetInfo.type=iframe` 验证 OOPIF,而非仅根据 origin 推断进程隔离。
|
||||
|
||||
子 frame scrollY=10 后,runtime local y 从 23 变为 13,snapshot document y 仍为 23。
|
||||
owner scale(1.25) 后 content quad 宽度变为 375,child innerWidth 仍为 300。
|
||||
|
||||
决定:frame-local 原始矩形必须先减该 document 的滚动,再经过一次权威投影;同 target 的
|
||||
owner quad 已是 target-root 坐标,不能重复逐层加偏移。禁止 border-box 替代 content origin。
|
||||
|
||||
Box 种类也必须进入契约:带 5px border 的 120px Canvas,其 snapshot/border quad 宽 130,
|
||||
content quad 宽 120。视觉入口采用 border-box,实时验证同种 box;iframe 投影则采用 content quad。
|
||||
带 border/padding 的 overflow owner,snapshot.clientRects 对应 `[clientLeft,clientTop,clientWidth,clientHeight]`,
|
||||
本 fixture 为 `[5,5,68,38]`。裁剪边界应从 client box 派生,不能直接拿 ancestor border bounds。
|
||||
单轴 `overflow-x:clip;overflow-y:visible` 的 computed style 保持分轴,必须分轴传播。
|
||||
|
||||
CSS zoom 1.25 后,上述 owner 的 bounds 变为 `[745,455.625,97,59.5]`,clientRects 仍为
|
||||
`[5,5,68,38]`,offsetRects 为 `[596,365,78,48]`。直接按 border-width 比例推导 client box
|
||||
会混入不同空间与整数取整。首版将有 CSS zoom/DOM transform 的 overflow 祖先组合明确降级,
|
||||
不承诺精确 crop;普通未变换 overflow 及 iframe content-quad 投影保留支持。
|
||||
|
||||
## 6. Document 与 ref 时效
|
||||
|
||||
真实 CDP 实验:
|
||||
|
||||
| 操作 | frameId | loaderId | document backend | anchor |
|
||||
|---|---|---|---|---|
|
||||
| history.replaceState | 不变 | 不变 | 不变 | 仍属于当前树 |
|
||||
| 跨 document 导航 | 不变 | 变化 | 变化 | 旧目标不可继续视为当前内容 |
|
||||
| document.open/write/close | 不变 | 不变 | 本次实验不变 | 旧 Canvas 失去连接,documentElement 被替换 |
|
||||
|
||||
决定:身份取 target attachment epoch + frameId + loaderId + documentElement backend identity;
|
||||
工具执行时仍检查 anchor.isConnected 及其所属 documentElement。不能仅凭 loaderId 判断有效。
|
||||
DOMSnapshot 可提供根元素 backend identity,frame graph 需要保留 loaderId;当前 main graph
|
||||
没有该字段,这是后续数据契约的明确修改点。
|
||||
|
||||
12 次静态几何采样完全相同;0.5px 移动可被检测;真实 90% 缩放引入约 1e-5 CSS px 的
|
||||
数值误差。首版采用最大边误差 0.25 CSS px 作为保守工程阈值,超出要求重新 observe。
|
||||
这不是大规模统计得到的通用稳定性保证;改变阈值需更新其测试及行为文档。
|
||||
|
||||
既有 eN 被下一次观察重用后,单靠旧字符串不能检测它来自上一代。内部 generation 不消除
|
||||
该协议限制。本次不改变协议,只验证当前存储目标的身份/几何。Canvas 可以在身份与几何
|
||||
不变时重绘;截图始终表达执行时图像,不承诺观察时像素快照。
|
||||
|
||||
## 7. 既有路径回归基线:用于防止负优化
|
||||
|
||||
诊断复制当前 renderer 到临时目录,在三个明确循环点加计数;生产文件未改变。
|
||||
复现命令见 [fixture README](../../apps/extension/test-fixtures/visual-observation/README.md)。
|
||||
|
||||
| 输入 | Render stack visits | DOM context candidates | Same-container sibling visits |
|
||||
|---|---:|---:|---:|
|
||||
| 100 headings + 100 View buttons | 201 | 10,000 | 15,050 |
|
||||
| 200 headings + 200 View buttons | 401 | 40,000 | 60,100 |
|
||||
| 400 headings + 400 View buttons | 801 | 160,000 | 240,200 |
|
||||
| 2,000 transparent wrappers + control | 2,001 | 0 | 0 |
|
||||
| 4,000 transparent wrappers + control | 4,001 | 0 | 0 |
|
||||
|
||||
显式栈访问线性并通过深树输入;本组宽树上下文计数呈二次增长。这里记录的是现有路径的
|
||||
特征,供后续同输入比较是否发生回退,不是本次要求修复的问题,也不是 Canvas 新算法的模板。
|
||||
计数不是完整 CPU profile,更不能替代 Canvas 发现、裁剪、选择和截图各自的性能测试。
|
||||
|
||||
处理边界:
|
||||
|
||||
1. 既有语义、DOM 操作和截图路径对照 main 不发生性能回退;不把历史算法优化列为本次前置依赖。
|
||||
2. Canvas 新路径按自己的规模和访问模式论证算法;发现、裁剪、去重、选择和截图分别验收。
|
||||
3. 精确 crop key 去重的选择来自首版的覆盖等价契约和新增成本分析,与 #165 采用什么算法无关。
|
||||
4. PR 7/8 分别报告既有路径回归、无 Canvas 页面的额外成本、有 Canvas 页面的新增成本。
|
||||
新功能有合理的额外成本不等于旧算法负优化;重复执行旧工作则属于需要消除的回退。
|
||||
|
||||
## 8. 锁定首版策略与退化
|
||||
|
||||
| 决策 | 首版契约 | 理由/验收 |
|
||||
|---|---|---|
|
||||
| 去重 | 同 document + 同直接 parent + 四条 crop 边完全相等 | 无模糊聚类;误差导致重复比误合并更可控 |
|
||||
| 索引容量 | 50,000 keys;耗尽后其余候选继续未去重选择 | 内存有界,不因早期小图耗尽而停止后续发现 |
|
||||
| K | min(预算推导容量, 512) | 所有行仍逐条核算可变成本 |
|
||||
| 默认视觉预算 | 未给总 maxTokens 时 2048 estimated tokens | 有显式 maxTokens 时全部纳入同一预算 |
|
||||
| 摘要预留 | 固定提示行按实际估算器成本预留 | 不用不可验证的固定 token 常量表示某段文本 |
|
||||
| 几何时效 | 最大边差 > 0.25 CSS px 即重新观察 | 身份和裁剪策略变化独立拒绝 |
|
||||
| 图像 | max edge 2048、max pixels 4,000,000、最多两次 capture | 成功必须以实际尺寸满足后置条件 |
|
||||
| 外部几何并发 | 4 | 单操作缓存;操作结束释放 |
|
||||
| 新增纯计算取消 | 每 256 项形成工作块并让出事件循环 | 不仅是轮询无法及时更新的 signal |
|
||||
|
||||
这些数值是可审查的首版资源政策,不是声称已证明用户体验最优的测量结果。
|
||||
参数可以在对应模块 PR 中经新的证据调整,但必须一并更新契约及测试。
|
||||
|
||||
进一步收敛:首版精确区域支持轴向矩形、普通 overflow 链、正向轴对齐平移/缩放与 frame。
|
||||
旋转、透视、clip-path、mask、活动 pinch-zoom 进入明确的 geometry-unsupported 降级,
|
||||
不给看似精确的 region ref,提示现有视口截图途径。不会把包围矩形偷换成精确区域。
|
||||
|
||||
## 9. 已执行的检查与下一阶段门槛
|
||||
|
||||
已执行:
|
||||
|
||||
- main VOM tests:47 passed。
|
||||
- main 相关 extension tests:217 passed,含 frame/element geometry、observation、interaction、ref-store、VOM。
|
||||
- `pnpm ext:build` 成功,用户 reload 后采集有效页面基线。
|
||||
- 独立 Chrome browser contract probe:39 个断言通过;可复现命令见 fixture README。
|
||||
- main 合成 capture replay、显式栈/上下文操作计数。
|
||||
- 实际 Chrome UI 90% zoom 样本,恢复 100%。
|
||||
- 原始表格与合成嵌入式表格均人工查看截图,并与未截断 observe 对照。
|
||||
|
||||
PR 0 交付的是已明确边界的设计和基线,不是未来功能已经通过:新的 discovery、typed ref、
|
||||
freshness resolver、pixel planner 和 renderer 接线尚未实现。后续每个 PR 必须验证自己的
|
||||
生产实现,不能把本次浏览器事实验证替代实现验收。
|
||||
|
||||
人工证据不设置默认 CI 对内网站点的访问依赖。合成 probe 可按需执行;页面原文不进仓库。
|
||||
后续 PR 的职责和验收保持在主设计中,本文负责记录证据、决策及没有作出的保证。
|
||||
@@ -0,0 +1,675 @@
|
||||
# BrowserSkill 视觉观察:架构设计、边界约束与 PR 实施计划
|
||||
|
||||
> 日期:2026-09-08
|
||||
> 状态:PR 0 设计验证稿;基线、浏览器实验和参数决策见 [验证结果](visual-observation-evidence.md)。生产实现尚未开始。执行步骤和成功条件见 [验收清单](visual-observation-acceptance.md),无需另写报告。
|
||||
> 实施基线:从实施时确认的 main 新建分支;本次分析使用本地 main `3c5f838`。
|
||||
> 参考材料:当前 `fix/vom-canvas-observation` 分支、PR #165、此前的 visual-surface 架构蓝图。旧实现和旧测试是证据来源,不是不可改变的规格。
|
||||
> 本次范围:设计规格及可复现的合成 fixture/诊断工具;不修改生产代码,不继承旧分支提交。
|
||||
|
||||
## 1. 设计结论
|
||||
|
||||
本次工程的目标是:
|
||||
|
||||
**为 VOM 建立独立的视觉观察通道,共享可靠的采集与几何基础,通过明确的区域身份提供截图能力,并使处理成本随输入规模可预测增长。**
|
||||
|
||||
工作方式是先确定整体架构、行为契约、模块依赖、复杂度和验收矩阵,再分 PR 实现。分步实现不等于分步思考架构。任何实现阶段需要改变已定契约时,必须先更新设计及相关验收项,不得用局部特殊分支掩盖契约冲突。
|
||||
|
||||
本次不是一次目录整理,也不是通用视觉理解平台重写。设计完整覆盖当前能力的生命周期,但不为尚无需求和证据的未来能力预建实现。
|
||||
|
||||
### 1.1 必须成立的五个结果
|
||||
|
||||
1. observe 能表达“这里存在需要看图理解的 Canvas 内容”,不依赖该元素是否有可用 AX 语义。
|
||||
2. 视觉入口的 document、frame、target、节点及坐标归属明确。
|
||||
3. 视觉 ref 与 DOM 操作 ref 具有明确且不可矛盾的工具契约。
|
||||
4. 截图解析当前目标并验证时效,返回浏览器最终合成的当前图像及真实尺寸。
|
||||
5. 采集、索引、发现、选择、输出和截图均有复杂度或资源上限,并从首个实现 PR 开始验证。
|
||||
|
||||
### 1.2 不能用来证明成功的指标
|
||||
|
||||
- 代码文件拆得更多。
|
||||
- 旧分支全部测试通过,但没有重新审核测试中的行为假设。
|
||||
- 能识别某个特定 嵌入式 Canvas 表格 页面,却依赖站点 class、URL 或布局常量。
|
||||
- 单个简单页面运行很快,但没有最坏输入分析。
|
||||
- 文本出现 surface 标记,但其截图对应关系、时效和工具限制没有验证。
|
||||
|
||||
## 2. 问题本质与现状
|
||||
|
||||
### 2.1 main 的基础能力
|
||||
|
||||
当前 main 的主要路径为:
|
||||
|
||||
```text
|
||||
DOMSnapshot + 各 frame 的 AX
|
||||
→ FrameDocument
|
||||
→ semantic graph:构建、解析、结构整理
|
||||
→ VomScene
|
||||
→ VOM 文本及 DOM ref
|
||||
```
|
||||
|
||||
已经存在 frame document、语义图、几何模块、ref store、截图路径和显式栈渲染器。应复用这些职责,不在旁边建立第二套长期运行的采集、frame traversal 或语义树。
|
||||
|
||||
缺口是:元素的 DOM/AX 描述不等于 Canvas 内部的文字、图形和布局。补 role、名称或普通可点击 ref,并不能建立完整的视觉观察能力。
|
||||
|
||||
### 2.2 必须分别解决的问题
|
||||
|
||||
| 问题 | 要回答的具体问题 | 不属于该问题的内容 |
|
||||
|---|---|---|
|
||||
| 视觉发现 | 哪些 Canvas 构成可观察候选? | 它是否可点击、输出多少条 |
|
||||
| 空间定位 | 候选在顶层视口的区域是什么? | 它是否重要、属于哪个业务画布 |
|
||||
| 输出选择 | 有限预算下展示哪些入口? | 改写几何事实、推断业务实体 |
|
||||
| ref 身份 | 入口绑定什么目标、允许哪些工具使用? | 像素缩放、内容识别 |
|
||||
| 时效验证 | 当前目标是否仍符合此次观察契约? | 保证 Canvas 像素未发生重绘 |
|
||||
| 图像获取 | 如何获取当前合成区域并限制成本? | 重建 Canvas 内部对象 |
|
||||
|
||||
### 2.3 当前分支值得保留与需要重审的部分
|
||||
|
||||
值得保留:语义/视觉双通道、视觉附录、单节点截图锚点、frame 几何共享、像素预算、相关失败场景和 fixture。
|
||||
|
||||
需要重审:
|
||||
|
||||
- 将 `aria-hidden`、`inert` 与视觉隐藏合并排除。语义暴露、交互资格、视觉存在是不同维度。
|
||||
- 将旧区域与实时区域的交集当作时效验证。区域重合不能证明目标仍对应原观察。
|
||||
- 将几何重合和共同父节点解释成逻辑画布身份。
|
||||
- 将名称有无作为重要性的绝对优先条件,可能让大量有标签的小图排在无标签主画布之前。
|
||||
- 视觉附录依赖 DOM 渲染后的剩余预算,可能使视觉通道完全消失。
|
||||
- 在每个候选上重复祖先遍历,或假设引入空间索引便自动消除了最坏二次复杂度。
|
||||
|
||||
PR #165 优化了既有渲染及上下文查询路径。本次将它作为性能回归保护基线,不作为 Canvas 算法模板,也不据此要求重做原有算法。Canvas 的发现、裁剪、选择和截图具有不同的数据规模与访问模式,应分别设计和验证适合自己的算法。
|
||||
|
||||
PR 0 已在用户指定页面确认:表格实际绘制后,main 输出外围工具栏而没有表格列内容及视觉入口,且未截断。直接打开嵌入应用确认存在无标签 Canvas。内部页面资料仅留临时目录;入库使用匿名化结论和独立合成 fixture。
|
||||
|
||||
## 3. 范围收敛
|
||||
|
||||
### 3.1 本次纳入
|
||||
|
||||
- Canvas 元素锚定的视觉观察入口。
|
||||
- 顶层、同 target 子 frame、OOPIF 及嵌套 frame 的身份和坐标归属。
|
||||
- 已明确支持的视口及 overflow 裁剪。
|
||||
- 无标签、存在 AX、不可交互但有视觉内容等候选情况。
|
||||
- 有界输出选择及保守的重复截图入口去重。
|
||||
- 视觉 ref 的工具契约和 document/几何时效检查。
|
||||
- 截图区域解析、像素预算、真实输出尺寸验证和有界重试。
|
||||
- 深层 DOM、密集小图、重叠 Canvas 和多 frame 的性能约束。
|
||||
|
||||
### 3.2 本次不纳入
|
||||
|
||||
- OCR、图像语义识别、Canvas 内部对象树。
|
||||
- 画布内部点选、拖拽或内部对象 ref。
|
||||
- 地图 tile 拼接、跨节点逻辑画布聚合、多 anchor 区域。
|
||||
- 面向所有 SVG、视频、CSS 背景等介质的通用发现插件系统。
|
||||
- 任意页面遮挡、透明度、clip-path 和 mask 的精确像素级重建。
|
||||
- 改写所有现有语义算法、表单、hover probe 或 recording 协议。
|
||||
- 新的 ref 字符串协议、跨 observe 持久对象 ID、分页接口。
|
||||
|
||||
边界外的能力可以提出独立设计,不能通过放宽当前去重阈值或给单节点 target 塞可选字段实现。
|
||||
|
||||
## 4. 总体架构
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Browser[浏览器 / CDP] --> Capture[CaptureCoordinator]
|
||||
Capture --> Batch[RawCaptureBatch]
|
||||
Batch --> Geometry[GeometryContext]
|
||||
Batch --> Normalize[Normalize + DocumentIndex]
|
||||
Geometry --> Normalize
|
||||
Normalize --> Facts[ObservationFacts]
|
||||
Facts --> Semantic[现有 SemanticPipeline]
|
||||
Facts --> Discover[VisualDiscovery]
|
||||
Discover --> Candidates[VisualCandidates]
|
||||
Semantic --> SemanticScene[SemanticScene]
|
||||
Candidates --> Select[VisualSelection]
|
||||
SemanticScene --> Context[输出上下文 / 活跃层策略]
|
||||
Context --> Select
|
||||
SemanticScene --> Renderer[ObservationRenderer + BudgetPlan]
|
||||
Select --> Renderer
|
||||
Renderer --> Output[文本 + Typed Ref Descriptors]
|
||||
Output --> Registry[RefRegistry]
|
||||
Registry --> Resolve[当前目标及区域解析]
|
||||
Browser --> Resolve
|
||||
Resolve --> Pixel[PixelPlanner]
|
||||
Pixel --> Shot[CaptureExecutor]
|
||||
Browser --> Shot
|
||||
```
|
||||
|
||||
### 4.1 架构原则
|
||||
|
||||
1. 共享事实,分开解释。视觉存在不复用语义节点的保留/排除结论。
|
||||
2. 语义关系是视觉区域的可选上下文,不是视觉身份的根。
|
||||
3. 几何只有一套解释和投影实现,观察与工具执行使用不同生命周期的上下文。
|
||||
4. renderer 只处理数据;浏览器调用止于采集或工具执行边界。
|
||||
5. 预算选择只影响输出,不回写采集事实。
|
||||
6. ref 的 kind 决定工具适用性,不能由调用者传入另一份可矛盾能力数组。
|
||||
7. 模块失败必须可归因;不能将“没有候选”与“采集失败”混为一谈。
|
||||
8. 所有资源限制都具有可解释的退化行为,不只是一句“超过上限停止”。
|
||||
|
||||
## 5. 模块划分与实现位置
|
||||
|
||||
以下为目标职责和建议位置,不要求第一个 PR 搬动全部文件。迁移文件与修改算法尽量分开;模块完成切换后删除被替代路径。
|
||||
|
||||
```text
|
||||
apps/extension/src/tools/
|
||||
observation.ts # 工具入口与结果接线,逐步减去内部实现
|
||||
observation/
|
||||
capture-coordinator.ts # CDP 编排及批次生命周期
|
||||
capture-types.ts # 原始批次、归属、阶段状态
|
||||
normalize.ts # 原始节点 → 归一化事实
|
||||
document-index.ts # 按 document 建立共享索引
|
||||
facts.ts # 下游可读事实契约
|
||||
geometry/
|
||||
coordinate-types.ts
|
||||
frame-context.ts # frame/target 投影权威入口
|
||||
primitives.ts # 复用/迁入现有数学实现
|
||||
vom/
|
||||
capture.ts # 保留或拆出的 DOMSnapshot adapter
|
||||
frame-document.ts # 复用现有 frame 归属逻辑
|
||||
semantic-graph/ # 保留语义职责
|
||||
visual/
|
||||
types.ts
|
||||
discovery.ts
|
||||
selection.ts
|
||||
coverage-dedup.ts
|
||||
refs/
|
||||
target.ts
|
||||
resolve.ts
|
||||
screenshot/
|
||||
resolve-region.ts
|
||||
pixel-plan.ts
|
||||
capture.ts
|
||||
|
||||
apps/extension/src/session-manager/
|
||||
ref-store.ts # session 生命周期与注册存储
|
||||
|
||||
packages/vom/src/
|
||||
types.ts # 纯场景及渲染契约
|
||||
render.ts # 保留显式栈语义遍历
|
||||
render-budget.ts # 唯一输出预算核算
|
||||
visual-appendix.ts # 纯数据视觉条目输出
|
||||
```
|
||||
|
||||
不同时保留旧 `geometry.ts` 与新 primitives 的两套数学实现。上述目录可按现有工程规范调整,但职责与依赖不得随意调整。
|
||||
|
||||
### 5.1 CaptureCoordinator
|
||||
|
||||
输入:tab/target、已验证的 URL、取消信号、采集选项。
|
||||
输出:带来源、时间范围、document 身份和阶段状态的 RawCaptureBatch。
|
||||
|
||||
- 按 target 分组采集;DOMSnapshot 每 target 一次成功采集,重试单独计数且有界。
|
||||
- AX 按协议支持的 frame/target 组织,数量有预算。
|
||||
- 原始采集完成后才能运行会改变页面状态的 hover probe,保留已有行为顺序。
|
||||
- 明确记录无法采集的 frame,不能把失败 frame 的数据借用其他 target 补齐。
|
||||
- 不判断 Canvas 重要性,不投放 ref。
|
||||
|
||||
批次是来源和生命周期边界,不是浏览器原子事务。`Readonly` 或冻结对象不能证明各 CDP 响应来自同一瞬间。遇到可检测的导航/document 更换时应中止或有界重采集,不能拼接不同 document 的数据。
|
||||
|
||||
### 5.2 GeometryContext
|
||||
|
||||
输入:该次操作的 frame graph、原始 metrics、按需取得的 owner quad。
|
||||
输出:frame-local/target-local 到 top-viewport 的投影、裁剪及当前节点区域解析。
|
||||
|
||||
- 坐标类型同时携带空间及所属 frame/target;仅标记 `frame-local` 不足以防止错 frame 使用。
|
||||
- 原始 DOMSnapshot 单位、CSS 视口单位、截图像素单位分别适配。
|
||||
- Canvas 视觉入口采用元素 border-box 的可截图区域;实时解析明确请求同一种 box,不能沿用默认 content-quad 路径再将边框差异误判为移动。iframe 子视口投影仍采用 owner content quad,两者职责不同。
|
||||
- 普通 overflow 裁剪使用 ancestor client/scrollport box,不使用祖先 border bounds。adapter 保留 includeDOMRects 的 client offsets/size 及其单位;存在缩放时先转换到相同坐标空间,禁止直接相加不同单位。证据不足时明确降级,不猜测 border/padding 常量。
|
||||
- 一个操作内复用 metrics、owner quad 和 projection promise;不跨 observe 或 tool execution 复用 live geometry。
|
||||
- 合成变换可以预计算;裁剪边界复杂度须真实计入,不能靠 promise cache 就宣称全部计算线性。
|
||||
- 不允许通过 iframe border box 加减常量替代有证据支持的 content geometry。
|
||||
|
||||
### 5.3 Normalize / ObservationFacts / DocumentIndex
|
||||
|
||||
输入:原始批次及几何上下文。
|
||||
输出:按 document 隔离的只读事实和有界共享索引。
|
||||
|
||||
- Raw node 不能含语义不明的 top rect;normalized node 的派生字段由 normalization 创建。
|
||||
- 节点 key 包含 document/target 所属信息,不能仅用 backendNodeId 跨 target 建表。
|
||||
- 建立 node lookup、parent/children 等基础索引;派生索引必须有明确消费者和复杂度。
|
||||
- 用显式栈和 visiting/visited 状态处理任意深度及异常环。
|
||||
- orphan 保留“祖先信息不完整”状态;可以在已知范围内生成保守候选,但不能伪装成完整裁剪证据。
|
||||
- 视觉样式、AX 暴露、交互资格分开保存,禁止万能 `excluded` 字段承载全部规则。
|
||||
- 不给每个节点复制完整祖先数组或完整裁剪链;必要时共享结构或按需解析,计入内存预算。
|
||||
|
||||
### 5.4 SemanticPipeline
|
||||
|
||||
消费归一化事实,输出现有语义场景。通过窄 adapter 接入新事实契约;迁移完成后删除不再需要的 adapter。
|
||||
|
||||
- 不因为视觉发现需求改变 semantic keep/drop、名称恢复或可交互推断规则。
|
||||
- 不把视觉候选塞回语义树遍历。
|
||||
- PR #165 的显式栈、上下文索引及行为回归保留。
|
||||
- 偶然发现的既有语义性能问题只作范围外记录,不构成本次工作项或前置依赖。本次只要求不使原有路径发生性能回退。
|
||||
|
||||
### 5.5 VisualDiscovery
|
||||
|
||||
输入:ObservationFacts。
|
||||
输出:候选及发现状态,不含最终 ref,不依赖 token 预算。
|
||||
|
||||
候选表达一个 Canvas 节点锚定的截图区域。必须具有可解析地址、支持范围内的几何证据和非空候选区域。
|
||||
|
||||
- 无 label 不排除;有 AX 内容也不能证明像素内容已被完整描述。
|
||||
- `aria-hidden`、`inert`、`pointer-events:none` 不单独构成视觉排除原因。
|
||||
- 零尺寸、无渲染区域、已证实的完全透明/隐藏、支持范围内的完全裁剪可排除。
|
||||
- `visibility` 等继承与覆盖行为必须按实际 CSS 语义处理,不能把祖先 hidden 一律不可逆传播。
|
||||
- `hidden` 等属性不应替代最终渲染事实;存在特殊状态时按实验确认的规则处理。
|
||||
- agent 自身 overlay 排除应按所属 document/target 传播。
|
||||
- 常规轴向 overflow 累积在一致的坐标空间计算;变换后不能把 top-space 包围矩形当作精确 local-axis clip。
|
||||
- 不重建任意遮挡。几何区域是截图候选范围,不保证范围内每个像素均来自该 Canvas。
|
||||
|
||||
### 5.6 VisualSelection / CoverageDedup
|
||||
|
||||
输入:候选、输出上下文及预算规划所需参数。
|
||||
输出:选中候选、明确的遗漏/退化状态。
|
||||
|
||||
- 默认以区域覆盖量等可解释几何依据排序;label 是命名事实,不作为压倒面积的优先条件。
|
||||
- 活跃层优先级属于选择策略;被降低优先级不等于物理不可见。
|
||||
- 排序键预计算;同层按可见面积降序,再按 frameId、top y、left x、backendNodeId 升序决定顺序。字符串按代码单元字典序、数值按数值比较;同一精确去重组取该顺序最小者。不把跨批次 backend id 稳定性作为保证。
|
||||
- 去重只减少重复截图入口,不产生新的 logical surface。
|
||||
- 首版仅在同 document、同直接 DOM parent、四条裁剪边数值完全相同的候选之间去重,使用精确 key 的 Map;不使用近似 IoU、尺寸 band 或空间邻居比较。亚像素不同的范围保留独立入口。
|
||||
- 首版不输出无法证明的 `layers=N` 业务解释。去重成员只是覆盖证据,不是内部对象模型。
|
||||
- 精确去重索引达到 50,000 个 key 后,剩余候选以未去重形式继续进入有界选择,记录去重退化;不得直接丢掉剩余区域。索引已知 key 仍可命中去重。
|
||||
- 即使没有去重,top-K 选择和总输出成本也必须有界。
|
||||
|
||||
### 5.7 ObservationRenderer / BudgetPlan
|
||||
|
||||
输入:SemanticScene、视觉选择结果和输出预算。
|
||||
输出:文本及与已提交行严格对应的 typed ref descriptors。
|
||||
|
||||
- 唯一模块核算总文本预算;摘要、头部、条目和截断提示全部计入。新 observe 的统一账本为每个实际输出行分别 `ceil(line.length / 4)` 后求和,length 使用 JavaScript UTF-16 单位;不混用整段核算与逐行核算。
|
||||
- 保留语义显式栈和 DOM ref 相对顺序;视觉条目在语义阶段之后提交。
|
||||
- 有视觉候选时,规划阶段预留简短视觉提示;其余视觉条目使用明确分配的剩余预算。
|
||||
- 极小预算不保证语义与视觉同时完整,必须产生确定的截断结果。
|
||||
- 写入行与提交 ref 在逻辑上原子:未写入的条目不能留下可解析 ref。
|
||||
- 使用既有 token 估算方法时,预算表示“估算 token 上限”,不能宣称精确模型 tokenizer 上限。
|
||||
- 新 observe 预算规划若发现显式预算不足以容纳最小协议头,返回 insufficient-budget,由工具层映射为既有 invalid_params,附最低估算成本;不生成半截协议头或新 ref。snapshot 的旧预算语义不在本次顺带改变。这是 observe 极小预算的明确兼容性例外。逐行统一账本也可能改变临界预算的截断位置;足够预算下无 Canvas 的语义文本及顺序保持不变。
|
||||
- 首版固定视觉提示为 `[visual] Canvas regions present; use screenshot`,成本由同一估算器计算。有候选但条目不足时仍尝试保留提示;连提示也容不下则只返回合法的截断语义输出,truncated=true。未知计数不伪装成精确遗漏数。
|
||||
- 输出标签最多 160 个 Unicode code point,再经 JSON escaping 与整行成本核算;排序不依赖标签长度。原始采集输入量 B 仍单独计入,不因输出裁剪便宣称采集内存有界。
|
||||
- `snapshot` 继续使用既定静态语义输出;新视觉入口首先仅接入 `observe`。不要顺带改变 recording/其他调用方默认选项。
|
||||
|
||||
### 5.8 RefRegistry / Resolve
|
||||
|
||||
保存 session/tab 隔离的目标描述。DOM 与视觉目标通过判别联合区分;kind 为权威,外部展示能力可从 kind 派生。
|
||||
|
||||
```ts
|
||||
// 概念契约,具体类型名与仓库适配时确定。
|
||||
type RefTarget =
|
||||
| { kind: "dom"; address: NodeAddress }
|
||||
| {
|
||||
kind: "visual-region";
|
||||
anchor: NodeAddress;
|
||||
observed: ObservedGeometry;
|
||||
observationId: ObservationId;
|
||||
};
|
||||
|
||||
// NodeAddress 必须能表示 tab、CDP target、frame、document 身份及 backend node。
|
||||
// 坐标值同时保留所属空间及 owner,不使用无归属 Rect 穿越模块边界。
|
||||
```
|
||||
|
||||
工具适用性:
|
||||
|
||||
| 消费入口 | DOM ref | Visual ref |
|
||||
|---|---|---|
|
||||
| screenshot | 维持既有 DOM 截图策略 | 使用视觉区域策略 |
|
||||
| click/fill/hover/select 等 | 维持既有检查 | 拒绝,且拒绝发生在派发输入前 |
|
||||
| get_html | 维持既有行为 | 拒绝;不隐式降级为 anchor 的 HTML |
|
||||
| 人工帮助定位 | 维持既有行为 | 若当前入口必须解释为 DOM 目标则拒绝;未来支持视觉区域需另立契约 |
|
||||
|
||||
所有 ref 消费者都必须通过类型化 resolver,不能依赖调用者记得传可选 capability 参数。
|
||||
|
||||
现有 `eN` 重用存在明确限制:如果新的 observe 已将 `e1` 重新绑定,单凭客户端传来的旧字符串 `e1` 无法判断它来自上一代。内部 generation 不会自动解决该问题。观察身份采用 target attach epoch + frameId + loaderId + documentElement backend identity,并在执行时验证 anchor 仍连接到匹配的 documentElement;loaderId 单独不足以覆盖 document.open。
|
||||
|
||||
本次保留当前“最新观察映射”的外部语义;只承诺能检测当前已存条目的 document/节点/几何失效,不承诺识别所有跨代旧字符串。彻底解决需独立协议设计。
|
||||
|
||||
### 5.9 ScreenshotExecution
|
||||
|
||||
分成三部分:
|
||||
|
||||
1. `resolve-region`:按目标种类解析当前身份与几何、验证 document 和区域变化。
|
||||
2. `pixel-plan`:纯数值函数,输出 clip scale 等计划。
|
||||
3. `capture`:overlay suppression、CDP 截图、真实尺寸读取、有界重试、取消及 dialog 集成。
|
||||
|
||||
- Visual ref 不自动滚动,不通过旧、新区域交集掩盖位移。
|
||||
- 首版对超过既定容差的位移、尺寸或裁剪变化返回“需要重新观察”;容差在 PR 0 由测量误差证据锁定。
|
||||
- 截图返回执行时浏览器最终合成图,不是观察时像素快照。
|
||||
- document 身份验证不能替代几何验证;几何相同也不能证明 Canvas 内容未重绘。
|
||||
- 像素规划采用明确 metrics 适配值;真实 PNG 尺寸是输出预算后置条件的依据。
|
||||
- 尺寸头校验不宣称完整 PNG 内容验证;缺失/不合法数据与预算超限分开归因。
|
||||
- 成功返回必须满足像素上限;有界重试仍失败则返回失败,不能猜测尺寸或无界重拍。
|
||||
|
||||
## 6. 算法与资源预算
|
||||
|
||||
### 6.0 两类独立的性能责任
|
||||
|
||||
**既有路径:防止回退。** 在相同输入、选项和环境下对照 main,检查原有语义处理、DOM ref、普通交互与截图的工作量、内存和实际耗时。不得因接入 Canvas 而重复运行语义解析、重建已有索引、增加原有节点的上下文查询,或撤销 #165 的优化。本次不要求把既有算法统一改成某个复杂度。
|
||||
|
||||
**新增路径:独立设计。** 围绕 Canvas 数量、frame 数量、裁剪复杂度、输出预算及截图像素量分析新成本;不照搬旧 renderer 的算法,也不以“旧算法使用了索引/显式栈”代替新问题的算法论证。下文复杂度与资源要求约束新增或本次实际改动的职责。
|
||||
|
||||
性能验收分三组,分别报告,不能互相替代:
|
||||
|
||||
| 验收组 | 对照方式 | 必须说明的结果 |
|
||||
|---|---|---|
|
||||
| 既有语义/操作路径 | 同一输入在 main 与新分支运行,隔离新增视觉工作 | 既有阶段未引入额外遍历、重复请求或可归因的性能回退 |
|
||||
| 无 Canvas 页面 | 实际完整 observe 对照 | 仅保留必要的候选检测开销;确认无候选后不建视觉索引、不执行视觉几何查询或截图 |
|
||||
| 有 Canvas 页面 | 单独测量视觉各阶段,再测完整 observe | 新成本与 C/F/P/K/像素等规模相符,资源有界;不把合理的新功能成本误记为旧算法退化 |
|
||||
|
||||
实际耗时需要多次同环境对照并结合操作数判断,不能仅凭一个容易波动的耗时值下结论。新增字段采集或共享模块调整的额外开销也必须归因并报告,不能藏在“原有成本”中。
|
||||
|
||||
### 6.1 规模定义
|
||||
|
||||
| 符号 | 含义 |
|
||||
|---|---|
|
||||
| B | 原始协议数据及字符串总量 |
|
||||
| N | 采集 DOM 节点数 |
|
||||
| A | AX 节点数 |
|
||||
| F / T | frame 数 / CDP target 数 |
|
||||
| C | 视觉候选数 |
|
||||
| K | 最大选中输出条目数 |
|
||||
| Q | 去重实际候选比较次数 |
|
||||
| P | 几何计算量,如顶点处理、裁剪操作总数 |
|
||||
| H | 语义上下文查询实际处理的数据总量 |
|
||||
| L | 最终输出文本长度 |
|
||||
|
||||
不能忽略 B、P、H 后笼统宣称整个管线 `O(N)`。原始数据解码至少与输入量相关,复杂裁剪和语义上下文的总成本需要显式计算。
|
||||
|
||||
### 6.2 模块性能契约
|
||||
|
||||
| 阶段 | 复杂度/上限目标 | 禁止的实现 |
|
||||
|---|---|---|
|
||||
| 原始解码 | 与 B 线性相关 | 为每个节点重新解码共享列或字符串 |
|
||||
| 文档索引 | `O(N + A + F)`,不含协议字节解码 | 每个候选全文扫描 |
|
||||
| 可合成祖先状态 | `O(N)` 建立,候选常数次查询 | 每 Canvas 完整祖先 walk |
|
||||
| Frame 几何 | 变换/证据按 frame 复用,额外成本计入 P | 每 node 查询 metrics/quad |
|
||||
| 发现 | 索引后 `O(C)`,额外几何成本计入 P | 发现函数内重新查 CDP |
|
||||
| top-K | `O(C log(K+1) + K log(K+1))` | 无需要地重复全量排序 |
|
||||
| 去重 | 精确 key Map,摊销 O(C),索引至多 50,000 个 key;首版 Q=0 | 引入模糊匹配和同 bucket 两两比较 |
|
||||
| 新增视觉渲染与既有渲染接线 | 新增工作按选中条目及 L 核算;既有 H 对照 main 不回退 | 为视觉条目重新执行语义遍历或扫描全部节点 |
|
||||
| 截图 | 调用和重试次数有界,成功图像像素数有界 | 无限重试、猜测输出尺寸 |
|
||||
|
||||
K 从预算允许的最小条目成本推导,并受异常安全上限约束;实际每行仍核算成本。面积优先 top-K 不应被描述为严格求解“可变长度条目预算最优装包”,本次不实现此类优化问题。
|
||||
|
||||
### 6.3 首版资源参数及锁定规则
|
||||
|
||||
以下是 PR 0 锁定的首版策略参数。它们是有明确退化语义的工程预算,不是性能最优值的统计结论;调整须更新对应契约测试:
|
||||
|
||||
- Visual 截图最大边:2048 px;最大面积:4,000,000 pixels。
|
||||
- 单次视觉截图最多 2 次 capture 尝试。
|
||||
- Frame 几何外部请求最大并发:4。
|
||||
- K 异常安全上限 512;正常数量由预算与条目成本决定。
|
||||
- 几何变化阈值:top-viewport CSS px 的四条边最大差值超过 0.25 即要求重新 observe;document、anchor、裁剪策略改变不受该容差豁免。
|
||||
- 没有显式 maxTokens 时,视觉通道估算 token 预算 2048;有显式预算时,优先为固定摘要行按当前估算器计算实际预留量,条目用剩余预算,全部计入同一总预算。
|
||||
- 精确覆盖去重索引最多 50,000 个 key;耗尽后继续未去重 top-K,标记 dedup-degraded;不得停止扫描剩余候选。
|
||||
- 新增纯计算阶段每 256 个节点/候选形成可取消工作块;需要实际让出事件循环,而不是只读一个尚无机会更新的 AbortSignal。
|
||||
|
||||
每个限额均须有单位、适用阶段、计数口径和耗尽行为。数字集中在所属模块策略配置中,不散落为多个含义相近的常量。若 PR 0 未锁定参数及退化契约,对应实现 PR 不得以随意常量合入。
|
||||
|
||||
### 6.4 性能测试方法
|
||||
|
||||
- 节点访问、祖先推导、候选比较、CDP 请求数、最大并发及中间集合大小均做可断言计数。
|
||||
- 对新增树处理使用深树/宽树测试,证明无调用栈溢出或重复全量扫描;既有 renderer 的对应输入用于 main 前后回归对照,不要求本次消除其历史复杂度。
|
||||
- 规模翻倍时检查操作数增长是否符合分析,避免只看一次墙钟时间。
|
||||
- 精确去重索引至少测试:全部重合、亚像素差异、同 crop 不同 parent、全部互不相交、容量耗尽;未来若引入空间索引,必须另增退化测试。
|
||||
- 大量有名小图与无名主画布并存,验证排序不会因标签策略丢失主要区域。
|
||||
- 保留 #165 回归;为新增模块增加性能契约,而不是声称旧测试覆盖整个链路。
|
||||
- 真实耗时基准用于观察额外成本和实际收益,需记录环境;不使用耗时阈值代替算法验证。按验收清单的性能步骤 预热 3 次、交替采样 20 次,分别报告旧阶段和新增成本;旧阶段中位数增加超过 max(基线 10%, 2ms) 时独立复测,两轮均超过且可归因于改动则 FAIL,无法排除噪声则 BLOCKED。
|
||||
|
||||
## 7. 失败、降级与一致性
|
||||
|
||||
内部状态至少区分:
|
||||
|
||||
```text
|
||||
capture-unavailable
|
||||
document-changed
|
||||
frame-ownership-unresolved
|
||||
geometry-unavailable
|
||||
geometry-unsupported
|
||||
outside-supported-clip
|
||||
discovery-partial
|
||||
selection-truncated
|
||||
dedup-degraded
|
||||
target-detached
|
||||
geometry-changed
|
||||
image-invalid
|
||||
pixel-budget-exceeded
|
||||
```
|
||||
|
||||
这些是内部分类,不要求逐项扩展外部 RPC code。外部沿用既有错误分类并给出准确 message/必要 detail;不可把所有时效或几何失败叫权限问题。
|
||||
|
||||
原则:
|
||||
|
||||
- 视觉分支失败不应使已获得的语义结果无故丢失。
|
||||
- 没有候选、没有预算、没有采集成功必须可区分。
|
||||
- exact omitted count 只在可计算时输出;原始候选数与去重后的区域数不能相加冒充精确数量。
|
||||
- 捕获批次不保证页面静止;可检测的身份变化必须处理,不可检测的像素重绘不作虚假保证。
|
||||
- 取消在阶段边界和有界工作块之间检查,不能在大型纯计算循环结束后才响应。
|
||||
|
||||
## 8. 兼容性与影响面约束
|
||||
|
||||
1. 从 main 新建实现分支;不将当前分支整体 cherry-pick 作为迁移方式。
|
||||
2. 没有视觉候选的页面,语义文本、DOM ref 相对顺序和操作行为维持原有契约。
|
||||
例外为上述 observe 显式预算小于最小协议头的 invalid_params 行为及统一账本导致的临界预算截断位置变化;足够预算不改变旧语义。
|
||||
3. 有视觉候选的 observe 允许新增视觉文本,并按新预算规划改变截断位置;这是明确行为变化。
|
||||
4. `snapshot`、recording、hover probe 默认行为不随视觉通道接入而隐式改变。
|
||||
5. 修正已证明错误的坐标允许改变输出位置,不以错误 golden 为兼容目标。
|
||||
6. 几何共享改动必须覆盖已有 DOM 点击/截图路径回归;不以 Canvas 测试代替。
|
||||
7. CLI/协议外形优先保持;确需新增外部字段、错误子码或参数时,必须在对应 PR 明确列出全部消费者。
|
||||
8. 不进行无关格式化、目录大搬迁、依赖升级或语义启发式修改。
|
||||
9. 不长期保留两套 capture decoder、metrics adapter、frame resolver、token accounting 或 pixel planner。
|
||||
10. 不把站点特征、fixture 标签、CSS class 或 URL 放入生产决策。
|
||||
|
||||
## 9. 设计验证清单与关闭方式
|
||||
|
||||
| 待验证项 | 证据 | 必须形成的决定 |
|
||||
|---|---|---|
|
||||
| 嵌入式 Canvas 表格 原始失败 | main 的观察输出、对应截图及结构证据 | 失败归属、最小复现和成功标准 |
|
||||
| DOMSnapshot / metrics 单位 | DPR、zoom、scroll、frame 矩阵 | 原始单位适配和转换入口(实验结论见证据文档) |
|
||||
| frame / target 坐标 | 同 target、OOPIF、nested、带边框/变换样例 | content quad 来源、坐标归属及支持范围 |
|
||||
| document 身份 | 导航、同 frame document 替换、target 重建 | 使用哪些现有标识/事件,何时失效 |
|
||||
| CSS 视觉规则 | visibility 覆盖、opacity、hidden、单轴 overflow | 可合成状态和不支持情况 |
|
||||
| 时效容差 | 静止页面测量噪声、可控移动/缩放/裁剪变化 | 容差单位、比较规则和失败行为 |
|
||||
| 预算 | 调用方 max_tokens 使用方式、密集画布样例 | 提示预留、K、限额与截断规则 |
|
||||
| 去重 | 重合、近重合、不同父节点、相邻 tile | 首版等价规则和 Q 上限 |
|
||||
|
||||
验证结果与最终决定见 [PR 0 证据及决策](visual-observation-evidence.md)。首版精确几何支持收敛到已验证的轴向矩形、正向轴对齐平移/缩放、普通 frame/overflow 链。旋转、透视、clip-path、mask 和活动 pinch-zoom 不承诺精确区域 ref:返回 geometry-unsupported 状态并提示使用现有视口截图。后续扩展必须独立验证。浏览器 UI zoom 与 CSS zoom 分开验证,不能用 pinch emulation 代替 UI zoom。
|
||||
|
||||
进一步限定:CSS zoom/DOM transform 作用于 overflow 裁剪祖先的组合,首版也进入 geometry-unsupported。
|
||||
实验表明 snapshot.bounds 已缩放,而 clientRects/offsetRects 仍有未缩放和整数取整语义,不能靠一个宽度比准确恢复所有边。
|
||||
普通 Canvas 的 CSS zoom bounds 与 iframe content quad 缩放仍可使用;这不意味着缩放后的任意 DOM scrollport 都已获得精确支持。
|
||||
Facts 必须保留 client box、transform/zoom 及 clip-path/mask 等支持性证据,discovery 返回状态而不是另写近似投影。
|
||||
|
||||
## 10. PR 划分与依赖
|
||||
|
||||
PR 0 是设计与证据入口;PR 1—6 建立模块;PR 7 接入公开 observe;PR 8 完成跨模块验收和迁移收尾。编号是顺序计划,不是实际 GitHub PR 编号。
|
||||
|
||||
```text
|
||||
PR 0 设计规格与实验
|
||||
└─ PR 1 几何与身份契约
|
||||
└─ PR 2 采集批次与共享事实
|
||||
└─ PR 3 视觉发现
|
||||
└─ PR 4 选择、去重与预算
|
||||
|
||||
PR 1 + PR 2 ──→ PR 5 类型化 ref
|
||||
PR 1 + PR 5 ──→ PR 6 视觉截图执行
|
||||
|
||||
PR 2 + PR 3 + PR 4 + PR 5 + PR 6
|
||||
└─ PR 7 observe 组合与接线
|
||||
└─ PR 8 整体验收与迁移收尾
|
||||
```
|
||||
|
||||
独立模块可以使用人工输入测试,不必等待真实上游接线。实现可依赖前序已合入 main 的 PR,避免多个大型长期分支互相漂移。阶段未接线的模块必须有紧邻的消费计划,不保留无期限死代码或对用户暴露半成品。
|
||||
|
||||
### PR 0:规格收敛、最小复现与基线
|
||||
|
||||
完成:
|
||||
|
||||
- 审核现有分支 fixture,区分正确规格与需要推翻的预期。
|
||||
- 记录 嵌入式 Canvas 表格 原始失败及 main 基线,建立独立于站点的最小 fixture。
|
||||
- 完成第 9 节实验,锁定核心数据类型、支持范围、容差和资源参数。
|
||||
- 固定 #165 的相关回归结果和操作数基线;不夸大旧测试覆盖范围。
|
||||
- 将行为契约映射到后续每个模块的测试清单。
|
||||
|
||||
不做:生产算法替换、全目录迁移、正式视觉通道接入。
|
||||
|
||||
验收:核心身份、坐标、裁剪、预算和失效规则不存在未指定行为;未支持情况有明确策略。若仍有未知项,先调整范围而不是让后续 PR 猜测。
|
||||
|
||||
### PR 1:统一几何与节点身份契约
|
||||
|
||||
完成:
|
||||
|
||||
- 引入有 owner 的坐标和地址类型,区分 frame/target/top/CSS/image 空间。
|
||||
- 统一 metrics 解释、frame projection 和 owner geometry 查询。
|
||||
- 共享单次操作内的 promise cache;保证采集和执行使用独立上下文。
|
||||
- 给既有 DOM 几何入口做窄适配;移除已被替代的投影路径。
|
||||
|
||||
不做:Canvas 发现、排序、ref 外部行为变化。
|
||||
|
||||
验收:scroll、border、同 target、OOPIF、nested、支持范围内的 transform/DPR/zoom 矩阵通过;已有 DOM 点击与截图回归通过;请求计数和几何成本可解释。
|
||||
|
||||
### PR 2:采集批次、归一化事实和共享索引
|
||||
|
||||
完成:
|
||||
|
||||
- 将原始解码、采集编排、normalization 明确分开。
|
||||
- 按 target 去重采集,记录 document 身份、时间范围和阶段失败。
|
||||
- 建立 ObservationFacts、document 隔离 lookup/children 索引和正确的祖先状态结构。
|
||||
- 现有 SemanticPipeline 通过 adapter 消费新事实;保持 hover 在静态采集后发生。
|
||||
- 删除不再使用的可变中间态和重复解码/归属推断路径。
|
||||
|
||||
不做:变更语义保留规则、视觉输出和截图政策。
|
||||
|
||||
验收:语义回归及 frame ownership 通过;deep/wide/orphan/cycle 有明确结果;解码、索引、采集数量符合预算;无每节点完整祖先复制。
|
||||
|
||||
### PR 3:纯数据视觉候选发现
|
||||
|
||||
完成:
|
||||
|
||||
- 实现 VisualCandidate 及 discoverVisualCandidates。
|
||||
- 落实无标签、有 AX、aria-hidden/inert/pointer-events、透明/隐藏、裁剪规则。
|
||||
- 候选关联 document/anchor/geometry,语义上下文只作为可选信息。
|
||||
- 返回发现失败或不完整状态,避免与无候选混淆。
|
||||
|
||||
不做:ref 分配、截图、token 排序、逻辑画布合并。
|
||||
|
||||
验收:每条发现规则都有正反 fixture;输入同一份事实输出确定;索引后的候选发现不重复扫 DOM 或走完整祖先链;不增加 CDP 调用。
|
||||
|
||||
### PR 4:有界选择、保守去重与输出预算计划
|
||||
|
||||
完成:
|
||||
|
||||
- 明确几何优先排序及确定性 tie-breaker,预计算比较键。
|
||||
- 实现有界 top-K,K 受文本预算和安全限额约束。
|
||||
- 实现精确覆盖 key 去重及索引容量耗尽后“继续未去重选择”的退化行为;首版 Q=0,不进行模糊两两比较。
|
||||
- 建立纯 BudgetPlan 和视觉摘要/条目格式,统一预算核算契约。
|
||||
- 明确候选数、输出数、遗漏数及去重退化信息的口径。
|
||||
|
||||
不做:多 anchor target、业务画布推断、生产 observe 接线。
|
||||
|
||||
验收:密集小图、无名主画布、全部重合、同 bucket 不等价、极小预算均有确定结果;比较次数、堆大小、输出成本有界。
|
||||
|
||||
### PR 5:类型化 ref 注册与工具解析
|
||||
|
||||
完成:
|
||||
|
||||
- RefStore 保存判别联合,绑定 document 及 observation 信息。
|
||||
- kind 派生工具适用性,删除可独立构造的矛盾 capability 状态。
|
||||
- 审核并迁移截图、交互、get_html、人工帮助等全部 ref 消费者。
|
||||
- 保持 eN 外部语义,文档明确跨代字符串识别限制。
|
||||
|
||||
不做:引入新版 ref 协议、自动把视觉 ref 当 DOM anchor 使用。
|
||||
|
||||
验收:跨 session/tab 隔离;document 失效;全部工具矩阵通过;拒绝在输入派发前发生;旧 DOM ref 行为保持。
|
||||
|
||||
### PR 6:视觉截图的区域解析、像素规划与执行
|
||||
|
||||
完成:
|
||||
|
||||
- 分离 resolve-region、pixel-plan、capture executor。
|
||||
- 实现 visual 不滚动、身份验证、几何容差及重新观察错误。
|
||||
- 实现真实输出尺寸后置条件、有界重试和准确失败归因。
|
||||
- 复用 overlay suppression、取消和 dialog 机制。
|
||||
|
||||
不做:Canvas 内部截图、像素冻结、多节点 union、内部点交互。
|
||||
|
||||
验收:移动/缩放/裁剪变化、节点移除、document 替换、DPR/zoom、无效图像、超预算和取消路径通过;成功图像符合像素预算;现有 DOM 截图回归通过。
|
||||
|
||||
### PR 7:观察结果组合与 observe 接线
|
||||
|
||||
完成:
|
||||
|
||||
- 将 Facts 的语义/视觉两路结果交给统一输出规划和 renderer。
|
||||
- 保留 #165 显式栈和 DOM ref 相对顺序,追加视觉条目。
|
||||
- 仅对已写入条目提交 ref,失败/截断状态正确传递。
|
||||
- 视觉提示参与预先预算,避免 DOM 耗尽预算后完全无提示。
|
||||
- 仅在 observe 默认启用新视觉通道;必要的 CLI/skill 使用说明同步更新。
|
||||
|
||||
不做:顺带修改 snapshot、recording 或 hover 默认策略。
|
||||
|
||||
验收:从人工 facts 到文本/ref 的集成测试、真实单 Canvas observe → screenshot 闭环通过;无 Canvas 页面回归通过;极小预算无悬空 ref。
|
||||
|
||||
### PR 8:跨模块验收、性能证据与收尾
|
||||
|
||||
完成:
|
||||
|
||||
- 在真实浏览器运行完整 fixture 矩阵和 嵌入式 Canvas 表格 原始场景,保存可复现结果。
|
||||
- 验证整个管线的请求数量、集合规模、比较次数、深树安全和真实耗时。
|
||||
- 检查新模块是否存在重复解释、长期 adapter、无消费者代码或语义影响扩散。
|
||||
- 收尾剩余迁移文档、调试信息说明、支持范围和已知限制。
|
||||
|
||||
不做:把此前未测试的性能正确性全部拖到本 PR,也不接纳新的功能范围。
|
||||
|
||||
验收:第 11 节全部通过;若发现某模块契约错误,修正在该模块并补对应验收,不在最终接线处加站点特判。
|
||||
|
||||
## 11. 总体验收矩阵
|
||||
|
||||
本节是范围索引,具体怎么做、什么算成功见 [验收清单](visual-observation-acceptance.md)。新会话按当前 PR 执行并直接说明结果即可,无需报告模板或逐项登记。修改契约时同步清单,避免两套标准。
|
||||
|
||||
| 维度 | 必测场景 | 主要责任 |
|
||||
|---|---|---|
|
||||
| 发现 | 无名、有 AX、aria-hidden、inert、pointer-events:none | PR 3 |
|
||||
| 视觉状态 | 零尺寸、透明祖先、visibility 覆盖、支持范围内 hidden 状态 | PR 0/2/3 |
|
||||
| 裁剪 | viewport、单轴/嵌套 overflow、支持范围内变换 | PR 1/2/3 |
|
||||
| frame | 同 target、OOPIF、nested、border、缺失 owner | PR 1/2 |
|
||||
| 身份 | target 冲突、节点移除、导航、document 替换 | PR 2/5/6 |
|
||||
| 输出 | DOM 顺序、视觉提示、modal 策略、极小预算、遗漏口径 | PR 4/7 |
|
||||
| ref | 工具矩阵、跨 tab/session、无悬空 ref | PR 5/7 |
|
||||
| 截图 | 几何变化、DPR/zoom、实际像素、重试、overlay | PR 6 |
|
||||
| 算法 | 深树、宽树、高密度、全部重合、索引退化 | 各模块 PR |
|
||||
| 回归 | #165、普通 DOM 点击/截图、snapshot、recording、hover | PR 1/2/7/8 |
|
||||
| 产品结果 | main 原始失败与新链路的 嵌入式 Canvas 表格 场景对照 | PR 0/8 |
|
||||
|
||||
## 12. 相比旧蓝图的明确调整
|
||||
|
||||
| 旧方向 | 本文调整 | 原因 |
|
||||
|---|---|---|
|
||||
| 全链路 transaction 先行 | 建立来源明确的 capture batch;不承诺原子快照 | 生命周期模型不能代替一致性证据 |
|
||||
| candidate → stack → logical surface 必经分层 | candidate + 输出覆盖去重;logical surface 不进入本次 | 限制身份模型及截图复杂度 |
|
||||
| 统一祖先 excluded | 视觉、语义、交互状态分开 | 避免加速错误规则 |
|
||||
| 各阶段一律兼容 adapter | 仅消费者迁移需要时使用,并随切换删除 | 避免临时抽象堆积 |
|
||||
| 性能作为后续优化阶段 | 每个模块设计/实现均有复杂度与计数验收 | 防止功能完成后再补算法 |
|
||||
| renderer 最后尝试附录 | 先规划预算,再分阶段输出 | 避免视觉入口系统性消失 |
|
||||
| observed rect 与 live rect 求交 | 身份和几何分别验证 | 避免成功返回错误截图片段 |
|
||||
| generation 暗示 freshness 已解决 | 明确现有 eN 重用限制 | 防止无法实现的旧字符串识别承诺 |
|
||||
| 当前测试全部视为兼容基准 | 先审核期望,再保留正确回归 | 测试不能替代行为规格 |
|
||||
|
||||
## 13. 完成定义与设计变更纪律
|
||||
|
||||
工程完成必须同时满足:
|
||||
|
||||
- 五条核心结果全部成立,真实 嵌入式 Canvas 表格 场景及独立 fixture 均有证据。
|
||||
- 每个模块只有一个权威实现,依赖方向与本文一致。
|
||||
- 坐标和节点地址跨边界时身份明确,失败可归因。
|
||||
- 视觉发现不依赖语义树保留结果,不混淆视觉与交互属性。
|
||||
- 性能上限及退化可测试;复杂度没有遗漏协议输入量和几何工作量。
|
||||
- 现有语义、DOM 操作及 #165 成果受到明确回归保护。
|
||||
- 当前支持范围、近似/拒绝行为和 ref freshness 限制已记录。
|
||||
- 不存在为了通过原始站点而增加的生产特判。
|
||||
|
||||
每个实现 PR 的描述必须写明:解决哪个独立问题、改变哪个契约、哪些行为允许变化、验证哪些正确性与性能要求、删除哪条旧路径。若没有删除旧路径,应说明保留原因和明确的后续消费/移除 PR。
|
||||
|
||||
新的案例若属于已定规则范围,应修复负责该规则的模块;若要求改变目标身份、几何支持或对外协议,应先形成设计变更。不得在 observation 接线、renderer 或截图执行末端不断补跨层例外。
|
||||
|
||||
## 14. 参考与证据来源
|
||||
|
||||
- 本地 main:`3c5f838`;当前分析分支:`fix/vom-canvas-observation`。
|
||||
- PR #165 本地合并提交:`5ed8287`;实现提交:`641d934`。涉及 renderer 上下文索引、显式栈等优化。
|
||||
- 主要代码范围:`apps/extension/src/tools/observation.ts`、`tools/vom/capture.ts`、`tools/vom/frame-capture.ts`、`tools/vom/semantic-graph/`、`tools/frame-geometry.ts`、`session-manager/ref-store.ts`、`packages/vom/src/render.ts`。
|
||||
- 当前分支参考:`tools/vom/rendered-surfaces/`、`tools/visual-screenshot-plan.ts` 及相关 fixture/test。
|
||||
- 原 visual-surface 架构蓝图作为历史参考,不是本设计的执行规格。
|
||||
- 浏览器语义参考:[aria-hidden](https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Reference/Attributes/aria-hidden)、[inert](https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Global_attributes/inert)、[visibility](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Properties/visibility)。本次 CDP 单位、frame 和身份实验结论见验证结果文档;支持范围之外的行为不作推断。
|
||||
|
||||
本文中的路径为代码职责定位,最终目录命名可适配工程规范;行为契约、问题边界、复杂度预算和验收要求不能因此省略。
|
||||
@@ -489,9 +489,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " +
|
||||
"understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " +
|
||||
"alongside them. Read-only: never submits input.";
|
||||
"probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
@@ -563,8 +561,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
name: "interact.click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_inspect observation, or a CSS selector. A visual Surface requires the " +
|
||||
"captureId and image coordinates returned by browser_inspect action=screenshot.",
|
||||
"last browser_inspect observation, or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -581,24 +578,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
type: "integer",
|
||||
description: "Number of consecutive presses (double-click = 2).",
|
||||
},
|
||||
modifiers: {
|
||||
type: "array",
|
||||
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
|
||||
description: "Keyboard modifiers held during the click.",
|
||||
},
|
||||
captureId: {
|
||||
type: "string",
|
||||
description:
|
||||
"Short-lived Surface capture id returned by browser_inspect action=screenshot.",
|
||||
},
|
||||
imageX: {
|
||||
type: "number",
|
||||
description: "X coordinate in capture-image pixels; requires captureId and imageY.",
|
||||
},
|
||||
imageY: {
|
||||
type: "number",
|
||||
description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
@@ -609,7 +588,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
tabId: { type: "integer", required: true },
|
||||
x: { type: "number", required: true },
|
||||
y: { type: "number", required: true },
|
||||
captureId: { type: "string" },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
@@ -621,44 +599,17 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const pointArgCount =
|
||||
Number(args.captureId !== undefined) +
|
||||
Number(args.imageX !== undefined) +
|
||||
Number(args.imageY !== undefined);
|
||||
if (pointArgCount !== 0 && pointArgCount !== 3) {
|
||||
throw new Error("captureId, imageX, and imageY must be given together");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
||||
cmdArgs.push("--modifiers", args.modifiers.join(","));
|
||||
}
|
||||
if (args.captureId !== undefined) {
|
||||
cmdArgs.push(
|
||||
"--capture",
|
||||
args.captureId,
|
||||
"--image-x",
|
||||
String(args.imageX),
|
||||
"--image-y",
|
||||
String(args.imageY),
|
||||
);
|
||||
}
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
||||
tab_id: number;
|
||||
x: number;
|
||||
y: number;
|
||||
capture_id?: string;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
x: reply.x,
|
||||
y: reply.y,
|
||||
...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
@@ -803,17 +754,12 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
name: "inspect.screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"This is the supported operation for visual surface refs emitted by inspect.observe. Only use " +
|
||||
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
|
||||
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
|
||||
"not guess content or coordinates. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
ref: {
|
||||
type: "string",
|
||||
description:
|
||||
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
|
||||
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
@@ -827,8 +773,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
byteSize: { type: "integer", required: true },
|
||||
captureId: { type: "string" },
|
||||
captureExpiresAt: { type: "integer" },
|
||||
image: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
@@ -852,11 +796,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
value.image !== undefined
|
||||
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
||||
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
||||
const captureText =
|
||||
value.captureId !== undefined
|
||||
? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
|
||||
: "";
|
||||
const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
|
||||
const blocks: ContentBlock[] = [{ type: "text", text }];
|
||||
if (value.image !== undefined) {
|
||||
blocks.push({
|
||||
type: "image",
|
||||
@@ -895,7 +835,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
height: number;
|
||||
path: string;
|
||||
byte_size: number;
|
||||
capture?: { id: string; expires_at: number };
|
||||
};
|
||||
writtenPath = reply.path;
|
||||
const data = await readFile(reply.path);
|
||||
@@ -909,9 +848,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
||||
width: reply.width,
|
||||
height: reply.height,
|
||||
byteSize: reply.byte_size,
|
||||
...(reply.capture !== undefined
|
||||
? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
|
||||
: {}),
|
||||
...(ref !== undefined
|
||||
? {
|
||||
image: {
|
||||
|
||||
@@ -478,12 +478,12 @@ describe("interaction tools", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
};
|
||||
|
||||
it("interact.click maps target, button, click count, and modifiers", async () => {
|
||||
it("interact.click maps target, button, and click count", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const click = tools.get("interact.click");
|
||||
const value = (await click?.execute(
|
||||
{ target: "@e1", button: "right", clickCount: 2, modifiers: ["ctrl", "shift"] },
|
||||
{ target: "@e1", button: "right", clickCount: 2 },
|
||||
makeExec(),
|
||||
)) as { session: string; x: number; y: number };
|
||||
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
|
||||
@@ -495,43 +495,6 @@ describe("interaction tools", () => {
|
||||
"right",
|
||||
"--click-count",
|
||||
"2",
|
||||
"--modifiers",
|
||||
"ctrl,shift",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("interact.click forwards a complete Surface capture coordinate transaction", async () => {
|
||||
const { tools, calls } = setup({
|
||||
...responses,
|
||||
click: {
|
||||
tab_id: 7,
|
||||
used_ref: "e1",
|
||||
x: 110,
|
||||
y: 70,
|
||||
capture_id: "sc_test",
|
||||
image_x: 200,
|
||||
image_y: 100,
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const click = tools.get("interact.click");
|
||||
const value = await click?.execute(
|
||||
{ target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
|
||||
makeExec(),
|
||||
);
|
||||
|
||||
expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
|
||||
expect(calls[1].args).toEqual([
|
||||
"click",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"200",
|
||||
"--image-y",
|
||||
"100",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
@@ -960,32 +923,6 @@ describe("inspect.screenshot", () => {
|
||||
expect(rendered?.every((block) => block.type === "text")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns Surface capture metadata alongside the screenshot", async () => {
|
||||
const path = pngFile();
|
||||
const { tools } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: {
|
||||
tab_id: 7,
|
||||
width: 800,
|
||||
height: 600,
|
||||
format: "png",
|
||||
path,
|
||||
byte_size: 8,
|
||||
capture: { id: "sc_test", expires_at: 1700000030000 },
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
|
||||
|
||||
expect(value).toMatchObject({
|
||||
captureId: "sc_test",
|
||||
captureExpiresAt: 1700000030000,
|
||||
});
|
||||
const rendered = screenshot?.output.render({ session: "s1" }, value as never);
|
||||
expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
|
||||
});
|
||||
|
||||
it("commits the image through the attachment store on an image-capable route", async () => {
|
||||
const path = pngFile();
|
||||
const services = {
|
||||
|
||||
@@ -29,139 +29,6 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("renders visual surfaces as screenshot-only refs", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 2,
|
||||
backendNodeId: 99,
|
||||
frameId: "child",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
label: "Data grid",
|
||||
memberCount: 2,
|
||||
},
|
||||
];
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain("[visual surfaces]");
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
backendNodeId: 99,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("renders an unnamed visual surface with a stable generic label", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ref: "e1",
|
||||
name: "canvas visual surface",
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("appends visual refs after DOM refs so canvas density cannot renumber controls", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 80, h: 50 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const withoutVisuals = renderVom({ ...input, visualSurfaces: [] });
|
||||
const withVisuals = renderVom(input);
|
||||
|
||||
expect(withoutVisuals.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||
expect(withVisuals.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||
expect(withVisuals.refs[1]).toMatchObject({ ref: "e2", backendNodeId: 99, kind: "surface" });
|
||||
expect(withVisuals.text.indexOf('@e1 button "Submit"')).toBeLessThan(
|
||||
withVisuals.text.indexOf("[visual surfaces]"),
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds visual output and prefers explicitly labeled surfaces", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({
|
||||
parentId: 1,
|
||||
backendNodeId: 100 + index,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas" as const,
|
||||
visibleRect: { x: index * 10, y: 20, w: 8, h: 8 },
|
||||
...(index === 11 ? { label: "Important status" } : {}),
|
||||
memberCount: 1,
|
||||
}));
|
||||
|
||||
const out = renderVom(input);
|
||||
const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface");
|
||||
|
||||
expect(surfaceRefs).toHaveLength(8);
|
||||
expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" });
|
||||
expect(out.text).toContain("… 4 additional visual surfaces omitted");
|
||||
expect(out.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("does not let a visual surface exhaust the budget before later DOM content", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "First", tag: "button" }),
|
||||
node({ id: 3, parentId: 1, role: "button", name: "Later", tag: "button" }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const out = renderVom(input, { maxTokens: 30 });
|
||||
|
||||
expect(out.text).toContain('@e2 button "Later"');
|
||||
expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(0);
|
||||
expect(out.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
@@ -15,8 +15,6 @@ export type {
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomRef,
|
||||
VomRefCapability,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
+2
-124
@@ -9,7 +9,6 @@ import type {
|
||||
VomOptions,
|
||||
VomResult,
|
||||
VomScene,
|
||||
VomVisualSurface,
|
||||
} from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
@@ -88,7 +87,6 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000;
|
||||
const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
|
||||
const MAX_HANDLE_CONTEXT_ITEMS = 3;
|
||||
const MAX_SURFACE_ITEMS = 12;
|
||||
const MAX_VISUAL_SURFACES = 8;
|
||||
const MAX_SCOPE_LINES = 40;
|
||||
const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
|
||||
const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
@@ -795,108 +793,6 @@ interface RenderState {
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
visualSurfaces: VomVisualSurface[];
|
||||
}
|
||||
|
||||
function visualSurfaceArea(surface: VomVisualSurface): number {
|
||||
return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
|
||||
}
|
||||
|
||||
function compareVisualSurfaces(a: VomVisualSurface, b: VomVisualSurface): number {
|
||||
const aLabeled = cleaned(a.label) !== undefined;
|
||||
const bLabeled = cleaned(b.label) !== undefined;
|
||||
return (
|
||||
Number(bLabeled) - Number(aLabeled) ||
|
||||
visualSurfaceArea(b) - visualSurfaceArea(a) ||
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.visibleRect.y - b.visibleRect.y ||
|
||||
a.visibleRect.x - b.visibleRect.x ||
|
||||
a.backendNodeId - b.backendNodeId
|
||||
);
|
||||
}
|
||||
|
||||
function visualSurfaceLine(surface: VomVisualSurface, ref: string): string {
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||
const { x, y, w, h } = surface.visibleRect;
|
||||
const bounds = [x, y, w, h].map(Math.round).join(",");
|
||||
return ` @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
||||
}
|
||||
|
||||
function selectVisualSurfaces(
|
||||
surfaces: readonly VomVisualSurface[],
|
||||
nodesById: ReadonlyMap<number, VomNode>,
|
||||
): { selected: VomVisualSurface[]; eligibleCount: number } {
|
||||
const selected: VomVisualSurface[] = [];
|
||||
let eligibleCount = 0;
|
||||
|
||||
for (const surface of surfaces) {
|
||||
if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
|
||||
eligibleCount += 1;
|
||||
|
||||
const insertionIndex = selected.findIndex(
|
||||
(candidate) => compareVisualSurfaces(surface, candidate) < 0,
|
||||
);
|
||||
if (insertionIndex < 0) {
|
||||
if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface);
|
||||
continue;
|
||||
}
|
||||
selected.splice(insertionIndex, 0, surface);
|
||||
if (selected.length > MAX_VISUAL_SURFACES) selected.pop();
|
||||
}
|
||||
|
||||
return { selected, eligibleCount };
|
||||
}
|
||||
|
||||
function emitVisualAppendix(state: RenderState): void {
|
||||
if (state.stopped) return;
|
||||
const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById);
|
||||
if (eligibleCount === 0) return;
|
||||
|
||||
const header = "[visual surfaces]";
|
||||
let headerEmitted = false;
|
||||
let emitted = 0;
|
||||
for (const surface of selected) {
|
||||
const ref = `e${state.nextRef}`;
|
||||
const line = visualSurfaceLine(surface, ref);
|
||||
const nextTokens =
|
||||
state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
|
||||
if (nextTokens > state.maxTokens) break;
|
||||
if (!headerEmitted) {
|
||||
state.lines.push(header);
|
||||
state.tokens += estimateTokens(header);
|
||||
headerEmitted = true;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens += estimateTokens(line);
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
state.refs.push({
|
||||
ref,
|
||||
backendNodeId: surface.backendNodeId,
|
||||
frameId: surface.frameId,
|
||||
visibleRect: surface.visibleRect,
|
||||
role: "surface",
|
||||
name: label,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
emitted += 1;
|
||||
}
|
||||
|
||||
const omitted = eligibleCount - emitted;
|
||||
if (omitted > 0) {
|
||||
state.truncated = true;
|
||||
if (headerEmitted) {
|
||||
const summary = ` … ${omitted} additional visual surfaces omitted`;
|
||||
const nextTokens = state.tokens + estimateTokens(summary);
|
||||
if (nextTokens <= state.maxTokens) {
|
||||
state.lines.push(summary);
|
||||
state.tokens = nextTokens;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
||||
@@ -977,8 +873,6 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
|
||||
...(node.role ? { role: node.role } : {}),
|
||||
...(name ? { name } : {}),
|
||||
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||
kind: "dom",
|
||||
capabilities: ["interact", "screenshot"],
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
@@ -1000,7 +894,6 @@ function renderNodes(
|
||||
initialLines: string[],
|
||||
surfaces: CondSurface[] = [],
|
||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||
visualSurfaces: VomVisualSurface[] = [],
|
||||
): RenderState {
|
||||
const children = buildChildren(nodes);
|
||||
const state: RenderState = {
|
||||
@@ -1020,7 +913,6 @@ function renderNodes(
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
visualSurfaces,
|
||||
};
|
||||
|
||||
renderTree(children, state);
|
||||
@@ -1232,20 +1124,8 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
"@layers 2 focus=L1",
|
||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||
];
|
||||
const visibleSurface = (parentId: number | null): boolean =>
|
||||
parentId === null || included.has(parentId);
|
||||
const state = renderNodes(
|
||||
visibleNodes,
|
||||
options,
|
||||
header,
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||
);
|
||||
const occlusionLine = renderPageOcclusionLine(hiddenCount);
|
||||
state.lines.push(occlusionLine);
|
||||
state.tokens += estimateTokens(occlusionLine);
|
||||
emitVisualAppendix(state);
|
||||
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
@@ -1272,9 +1152,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
],
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces,
|
||||
);
|
||||
emitVisualAppendix(state);
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
|
||||
@@ -61,8 +61,6 @@ export interface VomScene {
|
||||
rootFrameId?: string;
|
||||
surfaces?: CondSurface[];
|
||||
activeScopeBlocks?: ActiveScopeBlock[];
|
||||
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
|
||||
visualSurfaces?: VomVisualSurface[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
@@ -96,22 +94,6 @@ export interface VomRef {
|
||||
ref: string;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
/** Observation-time top-viewport crop for screenshot-only surfaces. */
|
||||
visibleRect?: Rect;
|
||||
kind?: "dom" | "surface";
|
||||
capabilities?: VomRefCapability[];
|
||||
}
|
||||
|
||||
export type VomRefCapability = "interact" | "screenshot";
|
||||
|
||||
export interface VomVisualSurface {
|
||||
parentId: number | null;
|
||||
backendNodeId: number;
|
||||
frameId: string;
|
||||
renderingKind: "canvas";
|
||||
visibleRect: Rect;
|
||||
label?: string;
|
||||
memberCount: number;
|
||||
}
|
||||
|
||||
export interface RenderedRef extends VomRef {
|
||||
|
||||
@@ -73,28 +73,6 @@ CSS-only hover menu looks like from here. It hovers a bounded set of likely trig
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
When `bsk observe` renders
|
||||
`@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered
|
||||
canvas content that is not represented by the text observation, not an interactable DOM control.
|
||||
Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or
|
||||
image-reading capability, do not take a screenshot and pretend to know its contents, and never guess
|
||||
coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
||||
|
||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain
|
||||
the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a
|
||||
point action and the image provides a clear target, bind that exact screenshot to the click command:
|
||||
|
||||
```bash
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing,
|
||||
navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture
|
||||
makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates.
|
||||
A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press
|
||||
workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and
|
||||
click/fill/hover/select reject them.
|
||||
|
||||
Escalate page reading only as needed:
|
||||
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
@@ -147,7 +125,6 @@ Required flags that are easy to get wrong:
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
bsk upload <ref> --file <path> bsk download <ref> --out <path>
|
||||
bsk click <surface-ref> --capture <id> --image-x <px> --image-y <px>
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
|
||||
Reference in new issue
Block a user