mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): update vom algorithm, fix overlay problem with background state machine
This commit is contained in:
@@ -274,6 +274,35 @@ describe("ToolDispatcher", () => {
|
||||
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
sent.length = 0;
|
||||
|
||||
deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).not.toHaveBeenCalled();
|
||||
|
||||
deliver({
|
||||
...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
|
||||
id: "r-control",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
});
|
||||
|
||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
parsePngDimensions,
|
||||
@@ -1412,7 +1413,7 @@ describe("buildVomScene", () => {
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
@@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => {
|
||||
expect(ctx.refStore.resolve("e2")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps snapshots static without conditional surface probing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleSnapshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "User tab" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Read" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const deps = makeDeps([root, button]);
|
||||
deps.tabsApi.get = vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
|
||||
);
|
||||
|
||||
const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.tab_id).toBe(9);
|
||||
expect(res.text).toContain('@e1 button "Read"');
|
||||
expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {});
|
||||
});
|
||||
|
||||
it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const deps = makeDeps([]);
|
||||
deps.tabsApi.get = vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
|
||||
);
|
||||
|
||||
const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "agent_window_scope" },
|
||||
});
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resets the RefStore on every fresh snapshot", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { lookupSession, parseBufferedReadBounds } from "../shared";
|
||||
import {
|
||||
cdpBlockedUrlReason,
|
||||
enforceCdpAccessibleTarget,
|
||||
lookupSession,
|
||||
parseBufferedReadBounds,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
} from "../shared";
|
||||
|
||||
function fakeAgentWindow() {
|
||||
return {
|
||||
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("CDP target URL guard", () => {
|
||||
it("allows ordinary pages and about:blank", () => {
|
||||
expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
|
||||
expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
|
||||
expect(cdpBlockedUrlReason("about:blank")).toBeNull();
|
||||
});
|
||||
|
||||
it("blocks browser and extension internal pages", () => {
|
||||
expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
|
||||
expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
|
||||
expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
|
||||
});
|
||||
|
||||
it("returns a structured permission error before page CDP access", () => {
|
||||
expect(
|
||||
enforceCdpAccessibleTarget(
|
||||
{
|
||||
tabId: 7,
|
||||
windowId: 100,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
},
|
||||
"snapshot",
|
||||
),
|
||||
).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "restricted_tab_url" },
|
||||
message: expect.stringContaining("snapshot cannot access tab 7"),
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back from a restricted default active tab to an accessible agent tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const tabsApi = {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
|
||||
if (query.active) {
|
||||
return [
|
||||
{
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
} as chrome.tabs.Tab,
|
||||
];
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
} as chrome.tabs.Tab,
|
||||
{
|
||||
id: 8,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: false,
|
||||
url: "https://example.test/",
|
||||
} as chrome.tabs.Tab,
|
||||
];
|
||||
}),
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
|
||||
).resolves.toMatchObject({
|
||||
tabId: 8,
|
||||
url: "https://example.test/",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not replace an explicitly requested restricted tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
async () =>
|
||||
({
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
}) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
|
||||
).resolves.toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "restricted_tab_url" },
|
||||
});
|
||||
expect(tabsApi.query).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
NavigateForwardParams,
|
||||
NavigateParams,
|
||||
NetworkParams,
|
||||
ObserveParams,
|
||||
PressParams,
|
||||
ProtocolFrame,
|
||||
RecordAwaitParams,
|
||||
@@ -42,6 +43,7 @@ import {
|
||||
type CdpRunner,
|
||||
chromeTabsCaptureApi,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
@@ -86,6 +88,8 @@ export interface DispatcherDeps {
|
||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||
*/
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
@@ -114,6 +118,7 @@ export class ToolDispatcher {
|
||||
private readonly sessions: SessionManager;
|
||||
private readonly cdp?: DispatcherCdpRunner;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
@@ -130,6 +135,7 @@ export class ToolDispatcher {
|
||||
this.sessions = deps.sessions;
|
||||
this.cdp = deps.cdp;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
}
|
||||
@@ -193,6 +199,8 @@ export class ToolDispatcher {
|
||||
let body: ResponseFrame;
|
||||
let startedSession: string | null = null;
|
||||
try {
|
||||
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
@@ -304,6 +312,12 @@ export class ToolDispatcher {
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.observe":
|
||||
return handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.get_html":
|
||||
return handleGetHtml(
|
||||
this.sessions,
|
||||
@@ -457,6 +471,33 @@ function isRpcError(v: unknown): v is RpcError {
|
||||
);
|
||||
}
|
||||
|
||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
switch (req.method) {
|
||||
case "tool.tab_create":
|
||||
case "tool.tab_close":
|
||||
case "tool.tab_select":
|
||||
case "tool.tab_borrow":
|
||||
case "tool.tab_return":
|
||||
case "tool.navigate":
|
||||
case "tool.navigate_back":
|
||||
case "tool.navigate_forward":
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.evaluate":
|
||||
case "tool.observe":
|
||||
case "tool.request_help":
|
||||
case "tool.record_start": {
|
||||
const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||
return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null;
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Read-only observation handlers — `tool.screenshot`, `tool.snapshot`,
|
||||
// and `tool.get_html` (design §7). Each handler resolves the target
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
@@ -15,6 +15,8 @@ import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
GetHtmlParams,
|
||||
GetHtmlResult,
|
||||
ObserveParams,
|
||||
ObserveResult,
|
||||
RpcError,
|
||||
ScreenshotParams,
|
||||
ScreenshotResult,
|
||||
@@ -26,11 +28,13 @@ import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
@@ -167,7 +171,13 @@ export async function handleScreenshot(
|
||||
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
"screenshot",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
||||
const withShotDialogs = <T extends object>(result: T) =>
|
||||
@@ -181,6 +191,7 @@ export async function handleScreenshot(
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(captured)) return captured;
|
||||
return withShotDialogs({
|
||||
@@ -664,6 +675,7 @@ function inputStateFor(
|
||||
value: string | undefined,
|
||||
): VomNode["inputState"] {
|
||||
if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined;
|
||||
if (capturedNode.formState) return capturedNode.formState;
|
||||
const formValue = capturedNode.formValue;
|
||||
if (formValue !== undefined) {
|
||||
if (formValue === "") return "empty";
|
||||
@@ -1099,7 +1111,13 @@ export async function handleGetHtml(
|
||||
const ctxOrErr = lookupSession(manager, params, "get_html");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
"get_html",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
@@ -1108,6 +1126,7 @@ export async function handleGetHtml(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
let html: string;
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
@@ -1178,28 +1197,44 @@ async function fallbackCapturedViewModel(
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId);
|
||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
toolName: "snapshot" | "observe",
|
||||
effect: ToolEffect,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "snapshot");
|
||||
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, toolName);
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
toolName,
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
||||
target.tabId,
|
||||
@@ -1207,7 +1242,7 @@ export async function handleSnapshot(
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
@@ -1232,3 +1267,19 @@ export async function handleSnapshot(
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
|
||||
}
|
||||
|
||||
export async function handleObserve(
|
||||
manager: SessionManager,
|
||||
params: ObserveParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
|
||||
}
|
||||
|
||||
@@ -369,7 +369,7 @@ async function rearmRecording(
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// content-script counter, and a single stop decrement left the control mask
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
@@ -561,9 +561,9 @@ export async function handleRecordStart(
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
// Register the recording *before* navigate so content-script syncAgentOverlay
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
// Register the recording *before* navigate so the destination content script's
|
||||
// overlay.ready flow can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing the control mask ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
|
||||
@@ -32,6 +32,13 @@ export const chromeTabsApi: ChromeTabsApi = {
|
||||
query: (q) => chrome.tabs.query(q),
|
||||
};
|
||||
|
||||
export interface ResolvedTargetTab {
|
||||
tabId: number;
|
||||
windowId: number;
|
||||
active: boolean;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export type { DialogCursor };
|
||||
|
||||
/**
|
||||
@@ -44,6 +51,7 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -146,7 +154,7 @@ export async function resolveTargetTab(
|
||||
ctx: SessionContext,
|
||||
tabId: number | undefined,
|
||||
api: ChromeTabsApi,
|
||||
): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> {
|
||||
): Promise<ResolvedTargetTab | RpcError> {
|
||||
if (tabId !== undefined) {
|
||||
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
|
||||
return {
|
||||
@@ -176,7 +184,7 @@ export async function resolveTargetTab(
|
||||
message: `tab ${tabId} not found in session scope`,
|
||||
};
|
||||
}
|
||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true };
|
||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url };
|
||||
}
|
||||
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
@@ -186,7 +194,12 @@ export async function resolveTargetTab(
|
||||
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
|
||||
};
|
||||
}
|
||||
return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true };
|
||||
return {
|
||||
tabId: first.id,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: first.active === true,
|
||||
url: first.url,
|
||||
};
|
||||
}
|
||||
|
||||
export function isRpcError(v: unknown): v is RpcError {
|
||||
@@ -202,6 +215,91 @@ export function isRpcError(v: unknown): v is RpcError {
|
||||
/** Re-export so M6/M7 tools keep a stable import path. */
|
||||
export { normaliseRef };
|
||||
|
||||
export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation";
|
||||
|
||||
const CDP_BLOCKED_PROTOCOLS = new Set([
|
||||
"chrome:",
|
||||
"chrome-extension:",
|
||||
"devtools:",
|
||||
"edge:",
|
||||
"brave:",
|
||||
"vivaldi:",
|
||||
"opera:",
|
||||
]);
|
||||
|
||||
export function cdpBlockedUrlReason(url: string | undefined): string | null {
|
||||
if (!url) return null;
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol;
|
||||
if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Page CDP tools cannot inspect browser/extension internal pages. Check this
|
||||
* at the tool boundary so Chrome's low-level "Cannot access..." errors do not
|
||||
* leak as ambiguous page-read failures.
|
||||
*/
|
||||
export function enforceCdpAccessibleTarget(
|
||||
target: ResolvedTargetTab,
|
||||
toolName: string,
|
||||
): RpcError | null {
|
||||
const reason = cdpBlockedUrlReason(target.url);
|
||||
if (!reason) return null;
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"restricted_tab_url",
|
||||
`${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`,
|
||||
);
|
||||
}
|
||||
|
||||
function resolvedTargetFromChromeTab(
|
||||
tab: chrome.tabs.Tab,
|
||||
fallbackWindowId: number,
|
||||
): ResolvedTargetTab | null {
|
||||
if (typeof tab.id !== "number") return null;
|
||||
return {
|
||||
tabId: tab.id,
|
||||
windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId,
|
||||
active: tab.active === true,
|
||||
url: tab.url,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a target for page CDP reads. Explicit tab_ids stay exact and are
|
||||
* rejected when browser security blocks the URL. For default targeting, skip a
|
||||
* restricted Agent Window active tab if another CDP-accessible tab exists in
|
||||
* the same Agent Window.
|
||||
*/
|
||||
export async function resolveCdpAccessibleTargetTab(
|
||||
manager: SessionManager,
|
||||
ctx: SessionContext,
|
||||
tabId: number | undefined,
|
||||
api: ChromeTabsApi,
|
||||
toolName: string,
|
||||
): Promise<ResolvedTargetTab | RpcError> {
|
||||
const target = await resolveTargetTab(manager, ctx, tabId, api);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
const restricted = enforceCdpAccessibleTarget(target, toolName);
|
||||
if (!restricted) return target;
|
||||
if (tabId !== undefined) return restricted;
|
||||
|
||||
const tabs = await api.query({ windowId: ctx.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId);
|
||||
if (!candidate) continue;
|
||||
if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate;
|
||||
}
|
||||
return restricted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
|
||||
* MUST refuse to touch a tab outside the session's Agent Window
|
||||
@@ -224,3 +322,17 @@ export function enforceAgentWindow(
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified target-scope policy by tool effect. Passive reads may inspect user
|
||||
* tabs; any tool that dispatches page input must stay inside the Agent Window.
|
||||
*/
|
||||
export function enforceToolTargetScope(
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number; windowId: number },
|
||||
effect: ToolEffect,
|
||||
toolName: string,
|
||||
): RpcError | null {
|
||||
if (effect === "passive_read") return null;
|
||||
return enforceAgentWindow(ctx, target, toolName);
|
||||
}
|
||||
|
||||
@@ -59,6 +59,23 @@ function makeCdp(snapshot: unknown) {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
controls: [
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
childFrames: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
@@ -80,6 +97,89 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
it("batch-enriches form controls without per-node object resolution", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
const input = result.nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let runtimeCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
runtimeCalls += 1;
|
||||
return runtimeCalls === 1
|
||||
? {
|
||||
result: {
|
||||
value: [
|
||||
{
|
||||
triggerSel: ".menu",
|
||||
affectedSub: " .items",
|
||||
label: "Products",
|
||||
x: 50,
|
||||
y: 20,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
: { result: { value: ["Shoes", "Bags"] } };
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: -10, y: -10 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
@@ -349,8 +449,11 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
@@ -557,8 +660,187 @@ describe("captureViewModel", () => {
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
|
||||
const input = subNodes.find((n) => n.backendNodeId === 101);
|
||||
expect(input?.tag).toBe("input");
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
});
|
||||
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [100, 101],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[20, 20, 120, 60]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 20, 300, 200],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [20, 21, 23],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [2] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[5, 6, 100, 80],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [30, 31],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[1, 2, 40, 20]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
|
||||
@@ -16,9 +16,14 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
@@ -31,6 +36,7 @@ export interface CapturedNode {
|
||||
*/
|
||||
cursor?: string;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
@@ -53,6 +59,10 @@ export interface CapturedViewModel {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
@@ -102,48 +112,112 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) {
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface CapturedFrameFormState {
|
||||
controls: CapturedFormState[];
|
||||
childFrames: CapturedFrameFormState[];
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
const controls = [];
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
controls.push(controlState(el));
|
||||
remaining -= 1;
|
||||
}
|
||||
const childFrames = [];
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
||||
}
|
||||
return { controls, childFrames };
|
||||
};
|
||||
return collect(document);
|
||||
})()`;
|
||||
}
|
||||
|
||||
function flattenFrameFormStates(
|
||||
frame: CapturedFrameFormState | undefined,
|
||||
out: CapturedFormState[][] = [],
|
||||
): CapturedFormState[][] {
|
||||
if (!frame) return out;
|
||||
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
||||
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
||||
flattenFrameFormStates(child, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
||||
let index = 0;
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(
|
||||
tabId,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: node.backendNodeId,
|
||||
},
|
||||
);
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) continue;
|
||||
const result = await cdp.send<{
|
||||
result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
returnByValue: true,
|
||||
functionDeclaration: `function(){
|
||||
return {
|
||||
value: typeof this.value === "string" ? this.value : "",
|
||||
defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "",
|
||||
placeholder: typeof this.placeholder === "string" ? this.placeholder : ""
|
||||
};
|
||||
}`,
|
||||
});
|
||||
const value = result.result?.value;
|
||||
if (!value) continue;
|
||||
node.formValue = value.value ?? "";
|
||||
node.formDefaultValue = value.defaultValue ?? "";
|
||||
node.formPlaceholder = value.placeholder ?? "";
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the node.
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
const state = states[index];
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
@@ -169,6 +243,14 @@ interface ParseDocumentResult {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
@@ -183,6 +265,49 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
@@ -235,14 +360,24 @@ function hoverCssScanExpression(): string {
|
||||
|
||||
const candidates = [];
|
||||
const seenLabels = new Set();
|
||||
const isUnsafeTrigger = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return true;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||
if (el.isContentEditable) return true;
|
||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
||||
return false;
|
||||
};
|
||||
for (const pair of pairs) {
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof Element)) continue;
|
||||
if (isUnsafeTrigger(el)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
if (!label || seenLabels.has(label)) continue;
|
||||
seenLabels.add(label);
|
||||
@@ -304,6 +439,20 @@ async function wait(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
await cdp
|
||||
.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: -10,
|
||||
y: -10,
|
||||
})
|
||||
.catch(() =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
||||
// best effort
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
@@ -328,7 +477,6 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
||||
expression: hoverCollectExpression(candidate),
|
||||
returnByValue: true,
|
||||
});
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
|
||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
@@ -341,6 +489,8 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
@@ -391,15 +541,13 @@ export async function collectOverlayExcludedBackendIds(
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param scrollX - horizontal scroll offset to subtract (CSS px)
|
||||
* @param scrollY - vertical scroll offset to subtract (CSS px)
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
scrollX: number,
|
||||
scrollY: number,
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
@@ -477,6 +625,7 @@ function parseDocumentNodes(
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
@@ -485,12 +634,13 @@ function parseDocumentNodes(
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
rect = {
|
||||
x: b[0] / dpr - scrollX,
|
||||
y: b[1] / dpr - scrollY,
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -508,9 +658,11 @@ function parseDocumentNodes(
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
@@ -521,7 +673,81 @@ function parseDocumentNodes(
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function parseChildFrameDocuments(
|
||||
documents: SnapshotDocument[],
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
const childDoc = documents[childDocIndex];
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
|
||||
const childContext: FrameContext = {
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
documents,
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
@@ -552,42 +778,26 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
|
||||
};
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
const topContext: FrameContext = {
|
||||
ownerFrameBackendNodeId: null,
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
// Build a map from node-array-index → sub-document-index using the sparse
|
||||
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
if (documents.length > 1) {
|
||||
const cdi = doc0.nodes?.contentDocumentIndex;
|
||||
if (cdi?.index && cdi.value) {
|
||||
for (let k = 0; k < cdi.index.length; k++) {
|
||||
const nodeArrayIdx = cdi.index[k];
|
||||
const docIdx = cdi.value[k];
|
||||
const subDoc = documents[docIdx];
|
||||
if (!subDoc) continue;
|
||||
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
|
||||
if (iframeBid === undefined) continue;
|
||||
// Sub-documents have their own scroll offsets; bounds are relative
|
||||
// to the iframe's own coordinate space (not the main viewport).
|
||||
const subScrollX = subDoc.scrollOffsetX ?? 0;
|
||||
const subScrollY = subDoc.scrollOffsetY ?? 0;
|
||||
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
|
||||
iframeNodes.set(iframeBid, subParsed.nodes);
|
||||
for (const id of subParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlState(cdp, tabId, nodes);
|
||||
for (const subNodes of iframeNodes.values()) {
|
||||
await enrichFormControlState(cdp, tabId, subNodes);
|
||||
}
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
|
||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user