feat(vom): update vom algorithm, fix overlay problem with background state machine

This commit is contained in:
Ljy-0827
2026-07-27 13:46:33 +08:00
parent e14ae9cea0
commit 4fb2e331a6
36 changed files with 2276 additions and 452 deletions
@@ -119,6 +119,88 @@ describe("ChromiumCdp", () => {
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 9 }, "Page.enable", {});
});
it("validates the attached top-frame URL for page reads", async () => {
const { api } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (_target, method: string) => {
if (method === "Page.getFrameTree") {
return { frameTree: { frame: { url: "https://example.test/page" } } };
}
return {};
},
);
const cdp = new ChromiumCdp(api);
await expect(
cdp.ensureAttachedToUrl(9, "https://example.test/page#section"),
).resolves.toBeUndefined();
expect(api.detach).not.toHaveBeenCalled();
});
it("reattaches when the cached CDP target points at an extension page", async () => {
const { api } = fakeApi();
let frameReads = 0;
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (_target, method: string) => {
if (method === "Page.getFrameTree") {
frameReads += 1;
return {
frameTree: {
frame: {
url:
frameReads === 1
? "chrome-extension://other-extension/page.html"
: "https://example.test/page",
},
},
};
}
return {};
},
);
const cdp = new ChromiumCdp(api);
await expect(cdp.ensureAttachedToUrl(9, "https://example.test/page")).resolves.toBeUndefined();
expect(api.attach).toHaveBeenCalledTimes(2);
expect(api.detach).toHaveBeenCalledWith({ tabId: 9 });
});
it("rejects when reattach still points at the wrong page", async () => {
const { api } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (_target, method: string) => {
if (method === "Page.getFrameTree") {
return { frameTree: { frame: { url: "https://wrong.test/" } } };
}
return {};
},
);
const cdp = new ChromiumCdp(api);
await expect(cdp.ensureAttachedToUrl(9, "https://example.test/")).rejects.toThrow(
/CDP target URL mismatch/,
);
expect(api.attach).toHaveBeenCalledTimes(2);
});
it("clears the attach cache after an internal-page access error", async () => {
const { api } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (_target, method: string) => {
if (method === "DOM.getDocument") {
throw new Error("Cannot access a chrome-extension:// URL of different extension");
}
return {};
},
);
const cdp = new ChromiumCdp(api);
await expect(cdp.ensureAttached(9)).resolves.toBeUndefined();
expect(cdp.isAttached(9)).toBe(true);
await expect(cdp.send(9, "DOM.getDocument")).rejects.toThrow(/chrome-extension/);
expect(cdp.isAttached(9)).toBe(false);
});
it("enables console capture domains best-effort during attach", async () => {
const { api } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -93,6 +93,15 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
const MAX_NETWORK_BUFFER = 200;
const MAX_NETWORK_FIELD_LENGTH = 4096;
const MAX_NETWORK_REQUEST_META = 1024;
const CDP_INTERNAL_PROTOCOLS = new Set([
"chrome:",
"chrome-extension:",
"devtools:",
"edge:",
"brave:",
"vivaldi:",
"opera:",
]);
interface ParsedDialogOpening {
type: JavaScriptDialogType;
@@ -173,6 +182,31 @@ export class ChromiumCdp {
await attach;
}
/**
* Attach and verify Chrome's actual CDP target still points at the page URL
* Chrome tabs reported to the tool resolver. A stale debugger target can
* otherwise surface another extension/internal URL while the tab record looks
* like a normal page.
*/
async ensureAttachedToUrl(tabId: number, expectedUrl: string | undefined): Promise<void> {
await this.ensureAttached(tabId);
if (!expectedUrl) return;
const first = await this.readTopFrameUrl(tabId).catch((err) => {
if (isInternalTargetAccessError(err)) return null;
throw err;
});
if (first && cdpUrlsCompatible(first, expectedUrl)) return;
await this.forceReattach(tabId);
const second = await this.readTopFrameUrl(tabId);
if (cdpUrlsCompatible(second, expectedUrl)) return;
throw new Error(
`CDP target URL mismatch for tab ${tabId}: expected ${expectedUrl}, attached to ${second}`,
);
}
/**
* Send a CDP command and decode the result as `T`. Throws on any
* `chrome.runtime.lastError`.
@@ -185,6 +219,9 @@ export class ChromiumCdp {
const result = await this.api.sendCommand({ tabId }, method, params ?? {});
return result as T;
} catch (err) {
if (isInternalTargetAccessError(err)) {
this.clearAttachedState(tabId);
}
throw normalizeError(err);
}
}
@@ -359,6 +396,31 @@ export class ChromiumCdp {
}
}
private async forceReattach(tabId: number): Promise<void> {
this.clearAttachedState(tabId);
try {
await this.api.detach({ tabId });
} catch {
// Chrome may already have detached; attach below is the source of truth.
}
await this.ensureAttached(tabId);
}
private clearAttachedState(tabId: number): void {
this.attachInFlight.delete(tabId);
this.attachedTabs.delete(tabId);
this.clearDialogState(tabId);
this.clearConsoleState(tabId);
this.clearNetworkState(tabId);
}
private async readTopFrameUrl(tabId: number): Promise<string> {
const frameTree = await this.api.sendCommand({ tabId }, "Page.getFrameTree", {});
const url = topFrameUrl(frameTree);
if (!url) throw new Error(`Page.getFrameTree returned no top-frame URL for tab ${tabId}`);
return url;
}
private bindDialogHandler(): void {
if (this.dialogSubscription) return;
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
@@ -906,3 +968,49 @@ function normalizeError(err: unknown): Error {
}
return new Error("unknown chrome.debugger error");
}
function isInternalTargetAccessError(err: unknown): boolean {
const message = normalizeError(err).message;
return (
/Cannot access a chrome-extension:\/\/ URL/i.test(message) ||
/Cannot access a chrome:\/\/ URL/i.test(message) ||
/Cannot access contents of url/i.test(message)
);
}
function topFrameUrl(frameTree: unknown): string | null {
const raw = (frameTree ?? {}) as Record<string, unknown>;
const tree = (raw.frameTree ?? {}) as Record<string, unknown>;
const frame = (tree.frame ?? {}) as Record<string, unknown>;
return typeof frame.url === "string" && frame.url.length > 0 ? frame.url : null;
}
function cdpUrlsCompatible(actualUrl: string, expectedUrl: string): boolean {
const actual = normalizeComparableUrl(actualUrl);
const expected = normalizeComparableUrl(expectedUrl);
if (!actual || !expected) return false;
if (isInternalCdpUrl(actual)) return false;
return actual === expected;
}
function normalizeComparableUrl(value: string): string | null {
let parsed: URL;
try {
parsed = new URL(value);
} catch {
return null;
}
parsed.hash = "";
return parsed.toString();
}
function isInternalCdpUrl(value: string): boolean {
let parsed: URL;
try {
parsed = new URL(value);
} catch {
return false;
}
if (CDP_INTERNAL_PROTOCOLS.has(parsed.protocol)) return true;
return parsed.protocol === "about:" && parsed.pathname !== "blank";
}
@@ -1,153 +0,0 @@
import { useTranslation } from "@browser-skill/i18n/react";
import { RiStopCircleLine } from "@remixicon/react";
import { useEffect, useState } from "react";
import logoUrl from "../../assets/logo.png";
export interface ControlOverlayProps {
visible: boolean;
interrupting: boolean;
automationBypass: boolean;
onInterrupt: () => void;
}
export function ControlOverlay({
visible,
interrupting,
automationBypass,
onInterrupt,
}: ControlOverlayProps) {
const { t } = useTranslation("extension");
const [show, setShow] = useState(false);
useEffect(() => {
if (visible) {
const raf = requestAnimationFrame(() => setShow(true));
return () => cancelAnimationFrame(raf);
}
setShow(false);
}, [visible]);
if (!visible) return null;
const pointerEvents = automationBypass ? "none" : "auto";
return (
<>
<style>{`
@keyframes bsk-breathe {
0%, 100% {
box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25);
}
50% {
box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5);
}
}
`}</style>
<div
data-slot="control-overlay"
style={{
position: "fixed",
inset: 0,
zIndex: 2147483646,
pointerEvents: "none",
animation: "bsk-breathe 3s ease-in-out infinite",
opacity: show ? 1 : 0,
transition: "opacity 300ms ease-out",
}}
/>
<div
data-slot="control-overlay-blocker"
onPointerDown={(event) => {
if (automationBypass) return;
event.preventDefault();
event.stopPropagation();
}}
onClick={(event) => {
if (automationBypass) return;
event.preventDefault();
event.stopPropagation();
}}
style={{
position: "fixed",
inset: 0,
zIndex: 2147483646,
pointerEvents,
background: "transparent",
opacity: show ? 1 : 0,
transition: "opacity 300ms ease-out",
}}
/>
<div
data-slot="control-overlay-pill"
style={{
position: "fixed",
bottom: 32,
left: "50%",
transform: "translateX(-50%)",
zIndex: 2147483647,
// Always receive clicks so Interrupt works even while the page
// blocker is in automation-bypass mode (CDP pass-through).
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: 12,
backgroundColor: "#fff",
borderRadius: 9999,
padding: "10px 10px 10px 20px",
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
opacity: show ? 1 : 0,
transition: "opacity 300ms ease-out",
fontFamily:
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
}}
>
<img
src={logoUrl}
alt="browser-skill"
style={{ width: 24, height: 24, borderRadius: 4, flexShrink: 0 }}
/>
<span
style={{
fontSize: 16,
fontWeight: 500,
color: "#333",
whiteSpace: "nowrap",
userSelect: "none",
}}
>
{t("controlOverlay.status")}
</span>
<button
type="button"
data-slot="control-overlay-stop-all"
disabled={interrupting}
onClick={onInterrupt}
style={{
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: 6,
border: "none",
borderRadius: 9999,
padding: "8px 20px 8px 16px",
fontSize: 15,
fontWeight: 600,
color: "#fff",
backgroundColor: interrupting ? "#9ca3af" : "#f97316",
cursor: interrupting ? "default" : "pointer",
opacity: interrupting ? 0.7 : 1,
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
whiteSpace: "nowrap",
lineHeight: 1,
}}
>
<RiStopCircleLine size={18} color="#fff" />
{interrupting ? t("controlOverlay.interrupting") : t("controlOverlay.interrupt")}
</button>
</div>
</>
);
}
@@ -1,47 +0,0 @@
import { cleanup, render } from "@testing-library/react";
import { afterEach, describe, expect, it } from "vitest";
import { ControlOverlay } from "../ControlOverlay";
describe("ControlOverlay", () => {
afterEach(() => {
cleanup();
});
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
const { container } = render(
<ControlOverlay
visible={true}
interrupting={false}
automationBypass={true}
onInterrupt={() => {}}
/>,
);
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
const pill = container.querySelector("[data-slot='control-overlay-pill']");
expect(pill).toBeTruthy();
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
expect(stopBtn).toBeTruthy();
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
});
it("uses pointer-events auto on blocker when automationBypass is false", () => {
const { container } = render(
<ControlOverlay
visible={true}
interrupting={false}
automationBypass={false}
onInterrupt={() => {}}
/>,
);
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
});
});
@@ -116,4 +116,18 @@ describe("OverlayController", () => {
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
expect(controller.snapshot().controlVisible).toBe(false);
});
it("hides control while paused and shows it again only after authoritative control mode", () => {
const controller = new OverlayController();
controller.applyAgentControlMode("sess-1", "paused");
expect(controller.snapshot().activeSessionId).toBe("sess-1");
expect(controller.snapshot().controlVisible).toBe(false);
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
controller.applyAgentControlMode("sess-1", "control");
expect(controller.snapshot().activeSessionId).toBe("sess-1");
expect(controller.snapshot().controlVisible).toBe(true);
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
});
});
@@ -1,3 +1,4 @@
import type { OverlayMode } from "@/lib/overlay-bridge";
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
import type { HelpRequestData } from "./HelpRequestOverlay";
import type { RecordRequestData } from "./RecordOverlay";
@@ -9,6 +10,7 @@ export interface OverlayState {
controlVisible: boolean;
interrupting: boolean;
activeSessionId: string | null;
controlMode: OverlayMode;
automationBypassCount: number;
/**
* After record Finish/Stop clears `activeRecord`, the session is still
@@ -31,6 +33,7 @@ export class OverlayController {
activeRecord: null,
interrupting: false,
activeSessionId: null,
controlMode: "hidden",
automationBypassCount: 0,
suppressControlAfterRecord: false,
};
@@ -44,7 +47,7 @@ export class OverlayController {
}
isControlVisible(): boolean {
return this.state.activeSessionId !== null;
return this.state.activeSessionId !== null && this.state.controlMode === "control";
}
addBorrowRequest(request: BorrowRequestData): void {
@@ -61,6 +64,25 @@ export class OverlayController {
activateAgentSession(sessionId: string): void {
this.state.activeSessionId = sessionId;
this.state.interrupting = false;
this.state.controlMode = "control";
}
applyAgentControlMode(sessionId: string | null, mode: OverlayMode): void {
if (mode === "control" && sessionId) {
this.state.activeSessionId = sessionId;
this.state.interrupting = false;
this.state.controlMode = "control";
return;
}
if ((mode === "interrupting" || mode === "paused") && sessionId) {
this.state.activeSessionId = sessionId;
this.state.interrupting = mode === "interrupting";
this.state.controlMode = mode;
return;
}
this.state.activeSessionId = null;
this.state.interrupting = false;
this.state.controlMode = "hidden";
}
setInterrupting(interrupting: boolean): void {
@@ -118,6 +140,7 @@ export class OverlayController {
activeRecord: null,
interrupting: false,
activeSessionId: null,
controlMode: "hidden",
automationBypassCount: 0,
suppressControlAfterRecord: false,
};
+102 -6
View File
@@ -9,12 +9,16 @@ import {
} from "@/lib/instance-id";
import { startKeepalive } from "@/lib/keepalive";
import {
OVERLAY_AGENT_STATE,
OVERLAY_AUTOMATION_BYPASS,
OVERLAY_MSG_INTERRUPT,
OVERLAY_MSG_READY,
OVERLAY_MSG_WHO_AM_I,
type OverlayAgentStateMessage,
type OverlayInterruptRequest,
type OverlayInterruptResponse,
type OverlayMessage,
type OverlayMode,
} from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
@@ -44,6 +48,91 @@ export default defineBackground(() => {
const sessions = new SessionManager();
const cdp = new ChromiumCdp();
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
let overlayGeneration = 0;
const controlModes = new Map<string, OverlayMode>();
function setControlMode(sessionId: string, mode: OverlayMode): void {
if (controlModes.get(sessionId) === mode) return;
controlModes.set(sessionId, mode);
overlayGeneration += 1;
const ctx = sessions.get(sessionId);
if (ctx) void pushOverlayStateForWindow(ctx.agentWindowId);
}
function overlayStateForWindow(windowId?: number): OverlayAgentStateMessage {
const ctx = typeof windowId === "number" ? sessions.findByWindowId(windowId) : null;
if (!ctx) {
return {
type: OVERLAY_AGENT_STATE,
sessionId: null,
mode: "hidden",
generation: overlayGeneration,
};
}
return {
type: OVERLAY_AGENT_STATE,
sessionId: ctx.sessionId,
mode: controlModes.get(ctx.sessionId) ?? "control",
generation: overlayGeneration,
};
}
async function pushOverlayStateToTab(
tabId: number,
state: OverlayAgentStateMessage,
): Promise<void> {
try {
await chrome.tabs.sendMessage(tabId, state);
} catch {
// Restricted pages and not-yet-loaded content scripts cannot receive messages.
}
}
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
const state = overlayStateForWindow(windowId);
const tabs = await chrome.tabs.query({ windowId });
await Promise.all(
tabs.map((tab) =>
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
),
);
}
function pushAllAgentOverlayStates(): void {
const windowIds = new Set(sessions.list().map((ctx) => ctx.agentWindowId));
for (const windowId of windowIds) {
void pushOverlayStateForWindow(windowId);
}
}
function onOverlaySessionStateChanged(): void {
void sessionsLive.syncFromManager();
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
for (const sessionId of controlModes.keys()) {
if (!liveSessionIds.has(sessionId)) controlModes.delete(sessionId);
}
overlayGeneration += 1;
pushAllAgentOverlayStates();
}
function onBrowserControlResumed(sessionId: string): void {
const ctx = sessions.get(sessionId);
if (!ctx) return;
setControlMode(sessionId, "control");
}
function pushOverlayStateForAgentWindow(windowId: number): void {
if (!sessions.findByWindowId(windowId)) return;
void pushOverlayStateForWindow(windowId);
}
chrome.tabs.onActivated.addListener((activeInfo) => {
pushOverlayStateForAgentWindow(activeInfo.windowId);
});
chrome.tabs.onUpdated.addListener((_tabId, changeInfo, tab) => {
if (changeInfo.status !== "complete") return;
if (typeof tab.windowId !== "number") return;
pushOverlayStateForAgentWindow(tab.windowId);
});
// Re-sync the storage.session flag on SW startup so a previous SW's
// stale `true` does not keep waking us on every page load until the
// first mutation (review M4/M5 round 3 m-R3-1).
@@ -52,9 +141,8 @@ export default defineBackground(() => {
transport,
sessions,
cdp,
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
approveBorrow: (ctx) =>
requestBorrowConfirmation(ctx.tabId, {
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
@@ -124,9 +212,7 @@ export default defineBackground(() => {
manager: sessions,
transport,
cdp,
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
onSessionsChanged: onOverlaySessionStateChanged,
});
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
@@ -197,9 +283,19 @@ export default defineBackground(() => {
return false;
}
if (msg.kind === OVERLAY_MSG_READY) {
sendResponse(overlayStateForWindow(sender.tab?.windowId));
return false;
}
if (msg.kind === OVERLAY_MSG_INTERRUPT) {
const req = msg as OverlayInterruptRequest;
const ctx = sessions.get(req.sessionId);
if (ctx) setControlMode(req.sessionId, "interrupting");
void handleOverlayInterrupt(transport, req.sessionId).then((reply) => {
if (reply.ok && sessions.get(req.sessionId)) {
setControlMode(req.sessionId, "paused");
}
sendResponse(reply);
});
return true; // keep channel open
+303 -79
View File
@@ -3,7 +3,6 @@ import { I18nextProvider } from "@browser-skill/i18n/react";
import React from "react";
import ReactDOM from "react-dom/client";
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
@@ -27,13 +26,12 @@ import {
} from "@/lib/help-bridge";
import {
isOverlayAgentOverlayResetMessage,
isOverlayAgentStateMessage,
OVERLAY_AUTOMATION_BYPASS,
OVERLAY_HOST_MARKER_ATTR,
OVERLAY_HOST_NAME,
OVERLAY_MSG_WHO_AM_I,
OVERLAY_MSG_READY,
type OverlayAgentOverlayResetMessage,
type OverlayAgentStateMessage,
type OverlayAutomationBypassMessage,
type OverlayWhoAmIResponse,
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
@@ -42,12 +40,12 @@ import {
type RecordStartAck,
type RecordStopAck,
} from "@/lib/record-bridge";
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
import type {
BorrowCancelMessage,
BorrowRequestMessage,
BorrowResponseMessage,
} from "@/tools/borrow-confirmation";
import logoUrl from "../../assets/logo.png";
// Run at document_end so the overlay does not block first paint. Only attach
// in the top-level frame so iframes do not double-render overlays.
@@ -65,34 +63,264 @@ export default defineContentScript({
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
let overlayContainer: HTMLElement | null = null;
let controlRoot: HTMLElement | null = null;
let activeAgentState: OverlayAgentStateMessage | null = null;
let hostLossReported = false;
let remountInProgress = false;
const ui = await createShadowRootUi(ctx, {
name: OVERLAY_HOST_NAME,
name: "browser-skill-overlay",
position: "inline",
anchor: "html",
css: overlayCss,
onMount(container, _shadow, shadowHost) {
shadowHost.setAttribute("aria-hidden", "true");
shadowHost.setAttribute(OVERLAY_HOST_MARKER_ATTR, "");
shadowHost.setAttribute("data-bsk-overlay", "");
overlayHost = shadowHost;
overlayContainer = container;
hostLossReported = false;
const app = document.createElement("div");
app.className = "bsk-overlay-root";
container.append(app);
reactRoot = ReactDOM.createRoot(app);
renderOverlay();
renderAll();
void requestOverlayState();
return reactRoot;
},
onRemove(root) {
overlayHost = null;
overlayContainer = null;
controlRoot = null;
root?.unmount();
reactRoot = null;
},
});
function renderOverlay() {
function removeControlOverlay(): void {
controlRoot?.remove();
controlRoot = null;
}
function setControlSurfaceActive(active: boolean): void {
const host = overlayHost;
const container = overlayContainer;
if (!host || !container) return;
if (active) {
Object.assign(host.style, {
position: "fixed",
inset: "0",
zIndex: "2147483647",
pointerEvents: "auto",
});
Object.assign(container.style, {
position: "fixed",
inset: "0",
pointerEvents: "none",
});
return;
}
host.style.removeProperty("position");
host.style.removeProperty("inset");
host.style.removeProperty("z-index");
host.style.removeProperty("pointer-events");
container.style.removeProperty("position");
container.style.removeProperty("inset");
container.style.removeProperty("pointer-events");
}
function ensureControlOverlayRoot(): HTMLElement | null {
if (!overlayContainer) return null;
if (!controlRoot) {
controlRoot = document.createElement("div");
controlRoot.className = "bsk-control-overlay";
controlRoot.style.opacity = "0";
controlRoot.style.transition = "opacity 300ms ease-out";
overlayContainer.append(controlRoot);
window.requestAnimationFrame(() => {
if (controlRoot) controlRoot.style.opacity = "1";
});
}
return controlRoot;
}
function createStopCircleIcon(): SVGSVGElement {
const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
svg.setAttribute("width", "18");
svg.setAttribute("height", "18");
svg.setAttribute("viewBox", "0 0 24 24");
svg.setAttribute("fill", "currentColor");
svg.setAttribute("aria-hidden", "true");
svg.style.color = "#fff";
svg.style.flexShrink = "0";
const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
path.setAttribute(
"d",
"M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10Zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16ZM9 9h6v6H9V9Z",
);
svg.append(path);
return svg;
}
function renderControlOverlay(): void {
const overlayState = overlays.snapshot();
if (!shouldShowAgentControlOverlay(overlayState)) {
removeControlOverlay();
setControlSurfaceActive(false);
return;
}
const root = ensureControlOverlayRoot();
if (!root) return;
const pointerEvents = overlayState.automationBypassCount > 0 ? "none" : "auto";
setControlSurfaceActive(true);
root.innerHTML = "";
Object.assign(root.style, {
position: "fixed",
inset: "0",
pointerEvents: "none",
});
const style = document.createElement("style");
style.textContent = `
@keyframes bsk-breathe {
0%, 100% { box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25); }
50% { box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5); }
}
`;
const border = document.createElement("div");
border.setAttribute("data-slot", "control-overlay");
Object.assign(border.style, {
position: "fixed",
inset: "0",
zIndex: "2147483646",
pointerEvents: "none",
animation: "bsk-breathe 3s ease-in-out infinite",
});
const blocker = document.createElement("div");
blocker.setAttribute("data-slot", "control-overlay-blocker");
Object.assign(blocker.style, {
position: "fixed",
inset: "0",
zIndex: "2147483646",
pointerEvents,
background: "transparent",
});
blocker.addEventListener("pointerdown", (event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
});
blocker.addEventListener("click", (event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
});
blocker.addEventListener(
"wheel",
(event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
},
{ passive: false },
);
blocker.addEventListener(
"touchmove",
(event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
},
{ passive: false },
);
const pill = document.createElement("div");
pill.setAttribute("data-slot", "control-overlay-pill");
Object.assign(pill.style, {
position: "fixed",
bottom: "32px",
left: "50%",
transform: "translateX(-50%)",
zIndex: "2147483647",
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: "12px",
borderRadius: "999px",
backgroundColor: "#fff",
padding: "10px 10px 10px 20px",
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
fontFamily:
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
});
const logo = document.createElement("img");
logo.src = logoUrl;
logo.alt = "browser-skill";
Object.assign(logo.style, {
width: "24px",
height: "24px",
borderRadius: "4px",
flexShrink: "0",
});
const label = document.createElement("span");
label.textContent = i18n.t("controlOverlay.status", { ns: "extension" });
Object.assign(label.style, {
fontSize: "16px",
fontWeight: "500",
color: "#333",
whiteSpace: "nowrap",
userSelect: "none",
});
const button = document.createElement("button");
button.type = "button";
button.setAttribute("data-slot", "control-overlay-stop-all");
button.disabled = overlayState.interrupting;
Object.assign(button.style, {
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: "6px",
border: "none",
borderRadius: "999px",
padding: "8px 20px 8px 16px",
fontSize: "15px",
fontWeight: "600",
color: "#fff",
backgroundColor: overlayState.interrupting ? "#9ca3af" : "#f97316",
cursor: overlayState.interrupting ? "default" : "pointer",
opacity: overlayState.interrupting ? "0.7" : "1",
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
whiteSpace: "nowrap",
lineHeight: "1",
});
button.append(
createStopCircleIcon(),
document.createTextNode(
overlayState.interrupting
? i18n.t("controlOverlay.interrupting", { ns: "extension" })
: i18n.t("controlOverlay.interrupt", { ns: "extension" }),
),
);
button.addEventListener("click", (event) => {
event.preventDefault();
event.stopPropagation();
handleInterrupt();
});
pill.append(logo, label, button);
root.append(style, border, blocker, pill);
}
function renderReactOverlays(): void {
const overlayState = overlays.snapshot();
reactRoot?.render(
React.createElement(
@@ -104,12 +332,6 @@ export default defineContentScript({
React.createElement(BorrowConfirmationOverlay, {
requests: overlayState.borrowRequests,
}),
React.createElement(ControlOverlay, {
visible: shouldShowAgentControlOverlay(overlayState),
interrupting: overlayState.interrupting,
automationBypass: overlayState.automationBypassCount > 0,
onInterrupt: handleInterrupt,
}),
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
),
@@ -117,6 +339,23 @@ export default defineContentScript({
);
}
function renderAll(): void {
renderReactOverlays();
renderControlOverlay();
}
function clearCurrentAgentSession(): void {
const sessionId = overlays.snapshot().activeSessionId;
if (!sessionId) return;
resetAgentOverlayState(sessionId);
}
function applyOverlayState(state: OverlayAgentStateMessage): void {
activeAgentState = state;
overlays.applyAgentControlMode(state.sessionId, state.mode);
renderAll();
}
function resetAgentOverlayState(sessionId: string) {
const previousHelp = overlays.resetAgentOverlays(sessionId);
if (previousHelp) {
@@ -125,7 +364,7 @@ export default defineContentScript({
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderOverlay();
renderAll();
}
function handleInterrupt() {
@@ -137,12 +376,8 @@ export default defineContentScript({
return;
}
overlays.setInterrupting(true);
renderOverlay();
renderAll();
void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => {
// Always retract the mask after the round trip resolves
// (success, failure, or timeout). Cancellation is fire-and-
// forget on the daemon side; the user must not be stuck
// behind a transient issue. The Agent Window stays open.
resetAgentOverlayState(sessionId);
if (!reply.ok) {
console.warn("[bsk overlay] interrupt did not get a clean ack from daemon");
@@ -157,6 +392,7 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
@@ -183,11 +419,11 @@ export default defineContentScript({
});
},
});
renderOverlay();
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderOverlay();
renderAll();
},
},
sendResponse as unknown as
@@ -205,7 +441,12 @@ export default defineContentScript({
) {
const bypassMsg = message as OverlayAutomationBypassMessage;
overlays.setAutomationBypass(bypassMsg.enabled);
renderOverlay();
renderAll();
return false;
}
if (isOverlayAgentStateMessage(message)) {
applyOverlayState(message);
return false;
}
@@ -216,7 +457,7 @@ export default defineContentScript({
if (message.type === "borrow-cancel") {
overlays.removeBorrowRequest(message.requestId);
renderOverlay();
renderAll();
return false;
}
@@ -224,7 +465,7 @@ export default defineContentScript({
const state = overlays.snapshot();
if (state.activeHelp && state.activeHelp.id === message.requestId) {
overlays.clearAgentHelpRequest(message.requestId);
renderOverlay();
renderAll();
}
return false;
}
@@ -244,7 +485,7 @@ export default defineContentScript({
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
renderOverlay();
renderAll();
sendResponse({ type: HELP_ACK, ok: true });
return false;
}
@@ -256,7 +497,7 @@ export default defineContentScript({
responded = true;
sendResponse({ type: "borrow-response", allowed });
overlays.removeBorrowRequest(message.requestId);
renderOverlay();
renderAll();
};
overlays.addBorrowRequest({
@@ -267,7 +508,7 @@ export default defineContentScript({
onAllow: () => respond(true),
onDeny: () => respond(false),
});
renderOverlay();
renderAll();
return true;
}
@@ -286,7 +527,7 @@ export default defineContentScript({
...(note ? { note } : {}),
};
overlays.clearAgentHelpRequest(requestId);
renderOverlay();
renderAll();
await chrome.runtime.sendMessage(msg).catch((err) => {
console.debug("[bsk overlay] help finish failed", err);
});
@@ -305,7 +546,7 @@ export default defineContentScript({
});
}
async function queryActiveHelpWithRetry(): Promise<boolean> {
async function queryActiveHelpWithRetry(): Promise<void> {
for (let attempt = 0; attempt < 6; attempt += 1) {
try {
const helpQuery = (await chrome.runtime.sendMessage({
@@ -313,38 +554,21 @@ export default defineContentScript({
})) as HelpQueryResponse | undefined;
if (helpQuery?.active && helpQuery.request) {
mountHelpRequest(helpQuery.request);
renderOverlay();
return true;
renderAll();
return;
}
} catch (err) {
console.debug("[bsk overlay] help query failed", err);
}
await new Promise((resolve) => window.setTimeout(resolve, 150));
}
return false;
}
async function syncAgentOverlay(): Promise<void> {
if (!(await anySessionLive())) return;
async function queryActiveRecord(): Promise<void> {
try {
const helpActive = await queryActiveHelpWithRetry();
const reply = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_WHO_AM_I,
})) as OverlayWhoAmIResponse | undefined;
if (!reply?.sessionId) return;
// Query / re-arm recording *before* activating the control overlay so
// record start (navigate → content load) does not flash
// 「Agent 正在控制」before RecordOverlay mounts.
let recordQuery: { active?: boolean; requestId?: string } | undefined;
try {
recordQuery = (await chrome.runtime.sendMessage({
type: RECORD_QUERY,
})) as { active?: boolean; requestId?: string } | undefined;
} catch (err) {
console.debug("[bsk overlay] record query failed", err);
}
const recordQuery = (await chrome.runtime.sendMessage({
type: RECORD_QUERY,
})) as { active?: boolean; requestId?: string } | undefined;
if (
recordQuery?.active &&
typeof recordQuery.requestId === "string" &&
@@ -360,26 +584,38 @@ export default defineContentScript({
});
},
});
renderAll();
}
if (!helpActive && overlays.snapshot().activeHelp === null) {
void queryActiveHelpWithRetry();
}
overlays.activateAgentSession(reply.sessionId);
renderOverlay();
} catch (err) {
console.debug("[bsk overlay] syncAgentOverlay failed", err);
console.debug("[bsk overlay] record query failed", err);
}
}
async function refreshAuxiliaryOverlayState(): Promise<void> {
await Promise.all([queryActiveHelpWithRetry(), queryActiveRecord()]);
}
async function requestOverlayState(): Promise<void> {
try {
const state = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_READY,
})) as OverlayAgentStateMessage | undefined;
if (state && isOverlayAgentStateMessage(state)) {
applyOverlayState(state);
}
void refreshAuxiliaryOverlayState();
} catch (err) {
console.debug("[bsk overlay] overlay.ready failed", err);
}
}
const onPageShow = (event: PageTransitionEvent) => {
if (event.persisted) void syncAgentOverlay();
if (event.persisted) void requestOverlayState();
};
ui.mount();
chrome.runtime.onMessage.addListener(onMessage);
void syncAgentOverlay();
void requestOverlayState();
window.addEventListener("pageshow", onPageShow);
@@ -391,7 +627,8 @@ export default defineContentScript({
remountInProgress = true;
try {
ui.mount();
void syncAgentOverlay();
if (activeAgentState) applyOverlayState(activeAgentState);
void requestOverlayState();
} finally {
remountInProgress = false;
}
@@ -414,16 +651,3 @@ export default defineContentScript({
});
},
});
async function anySessionLive(): Promise<boolean> {
if (!chrome.storage?.session?.get) return true;
try {
const result = (await chrome.storage.session.get({
[SESSIONS_LIVE_FLAG_KEY]: false,
})) as Record<string, unknown> | undefined;
return Boolean(result?.[SESSIONS_LIVE_FLAG_KEY]);
} catch (err) {
console.debug("[bsk overlay] sessions-live flag read failed", err);
return true;
}
}
+1 -1
View File
@@ -5,7 +5,7 @@
*
* Mirrors the borrow-confirmation message style (see
* `@/tools/borrow-confirmation`). The background asks a specific tab to
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
* enter "help mode" (render HelpRequestOverlay + hide the control mask);
* the content script acks display, then later reports Done / Cancel.
*/
+39 -4
View File
@@ -3,9 +3,8 @@
* between the content-script control overlay and the background SW.
*
* Content script → background:
* - `{ kind: "overlay.who_am_i", tabId, windowId }` → background
* replies `{ sessionId: string | null }` so the overlay knows
* whether to render itself.
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
* overlay state for the sender's window.
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
* daemon (via a `session.user_interrupt` WS event) to cancel
* every inflight + queued tool call for that session with
@@ -14,6 +13,7 @@
*/
export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i";
export const OVERLAY_MSG_READY = "overlay.ready";
export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt";
/**
@@ -57,6 +57,22 @@ export interface OverlayWhoAmIResponse {
sessionId: string | null;
}
export interface OverlayReadyRequest {
kind: typeof OVERLAY_MSG_READY;
}
export type OverlayMode = "control" | "interrupting" | "paused" | "hidden";
/** Background → content: complete, authoritative control-overlay state. */
export const OVERLAY_AGENT_STATE = "bh-agent-overlay-state";
export interface OverlayAgentStateMessage {
type: typeof OVERLAY_AGENT_STATE;
sessionId: string | null;
mode: OverlayMode;
generation: number;
}
export interface OverlayInterruptRequest {
kind: typeof OVERLAY_MSG_INTERRUPT;
sessionId: string;
@@ -90,4 +106,23 @@ export function isOverlayAgentOverlayResetMessage(
return candidate.type === OVERLAY_AGENT_OVERLAY_RESET && typeof candidate.sessionId === "string";
}
export type OverlayMessage = OverlayWhoAmIRequest | OverlayInterruptRequest;
export function isOverlayAgentStateMessage(message: unknown): message is OverlayAgentStateMessage {
if (!message || typeof message !== "object") return false;
const candidate = message as {
type?: unknown;
sessionId?: unknown;
mode?: unknown;
generation?: unknown;
};
return (
candidate.type === OVERLAY_AGENT_STATE &&
(typeof candidate.sessionId === "string" || candidate.sessionId === null) &&
(candidate.mode === "control" ||
candidate.mode === "interrupting" ||
candidate.mode === "paused" ||
candidate.mode === "hidden") &&
typeof candidate.generation === "number"
);
}
export type OverlayMessage = OverlayWhoAmIRequest | OverlayReadyRequest | OverlayInterruptRequest;
@@ -274,6 +274,35 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
const onBrowserControlResumed = vi.fn();
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
dispatcher.start();
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
await flushMicrotasks();
sent.length = 0;
deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
await flushMicrotasks();
expect(onBrowserControlResumed).not.toHaveBeenCalled();
deliver({
...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
id: "r-control",
});
await flushMicrotasks();
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
});
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
const { transport, deliver } = fakeTransport();
const sessions = new SessionManager({
@@ -7,6 +7,7 @@ import {
buildVomScene,
type CdpAxNode,
handleGetHtml,
handleObserve,
handleScreenshot,
handleSnapshot,
parsePngDimensions,
@@ -1412,7 +1413,7 @@ describe("buildVomScene", () => {
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
});
it("enriches names and active scope signals from AX properties", () => {
@@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => {
expect(ctx.refStore.resolve("e2")).toBeNull();
});
it("keeps snapshots static without conditional surface probing", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Example" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 200,
};
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
const i = (s: string) => strings.indexOf(s);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 20, 120, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
if (method === "Runtime.evaluate") return { result: { value: [] } };
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleSnapshot(
sm,
{ session_id: "aa11" },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(send).not.toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
});
it("enables conditional surface probing for semantic observe", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Example" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 200,
};
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
const i = (s: string) => strings.indexOf(s);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 20, 120, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
if (method === "Runtime.evaluate") return { result: { value: [] } };
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleObserve(
sm,
{ session_id: "aa11" },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
});
it("allows passive snapshots of explicit user-window tabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "User tab" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Read" },
backendDOMNodeId: 200,
};
const deps = makeDeps([root, button]);
deps.tabsApi.get = vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
);
const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.tab_id).toBe(9);
expect(res.text).toContain('@e1 button "Read"');
expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {});
});
it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const deps = makeDeps([]);
deps.tabsApi.get = vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
);
const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps);
expect(res).toMatchObject({
code: "permission_denied",
data: { reason: "agent_window_scope" },
});
expect(deps.send).not.toHaveBeenCalled();
});
it("resets the RefStore on every fresh snapshot", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -1,6 +1,12 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { lookupSession, parseBufferedReadBounds } from "../shared";
import {
cdpBlockedUrlReason,
enforceCdpAccessibleTarget,
lookupSession,
parseBufferedReadBounds,
resolveCdpAccessibleTargetTab,
} from "../shared";
function fakeAgentWindow() {
return {
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
});
});
});
describe("CDP target URL guard", () => {
it("allows ordinary pages and about:blank", () => {
expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
expect(cdpBlockedUrlReason("about:blank")).toBeNull();
});
it("blocks browser and extension internal pages", () => {
expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
});
it("returns a structured permission error before page CDP access", () => {
expect(
enforceCdpAccessibleTarget(
{
tabId: 7,
windowId: 100,
active: true,
url: "chrome-extension://other-extension/page.html",
},
"snapshot",
),
).toMatchObject({
code: "permission_denied",
data: { reason: "restricted_tab_url" },
message: expect.stringContaining("snapshot cannot access tab 7"),
});
});
it("falls back from a restricted default active tab to an accessible agent tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const tabsApi = {
get: vi.fn(),
query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
if (query.active) {
return [
{
id: 7,
windowId: ctx.agentWindowId,
active: true,
url: "chrome-extension://other-extension/page.html",
} as chrome.tabs.Tab,
];
}
return [
{
id: 7,
windowId: ctx.agentWindowId,
active: true,
url: "chrome-extension://other-extension/page.html",
} as chrome.tabs.Tab,
{
id: 8,
windowId: ctx.agentWindowId,
active: false,
url: "https://example.test/",
} as chrome.tabs.Tab,
];
}),
};
await expect(
resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
).resolves.toMatchObject({
tabId: 8,
url: "https://example.test/",
});
});
it("does not replace an explicitly requested restricted tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const tabsApi = {
get: vi.fn(
async () =>
({
id: 7,
windowId: ctx.agentWindowId,
active: true,
url: "chrome-extension://other-extension/page.html",
}) as chrome.tabs.Tab,
),
query: vi.fn(),
};
await expect(
resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
).resolves.toMatchObject({
code: "permission_denied",
data: { reason: "restricted_tab_url" },
});
expect(tabsApi.query).not.toHaveBeenCalled();
});
});
+41
View File
@@ -11,6 +11,7 @@ import type {
NavigateForwardParams,
NavigateParams,
NetworkParams,
ObserveParams,
PressParams,
ProtocolFrame,
RecordAwaitParams,
@@ -42,6 +43,7 @@ import {
type CdpRunner,
chromeTabsCaptureApi,
handleGetHtml,
handleObserve,
handleScreenshot,
handleSnapshot,
} from "./observation";
@@ -86,6 +88,8 @@ export interface DispatcherDeps {
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
*/
onSessionsChanged?: () => void;
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
onBrowserControlResumed?: (sessionId: string) => void;
/** User approval for `tool.tab_borrow` (overlay in content script). */
approveBorrow?: BorrowConfirmationApprover;
/** i18n notification copy for `tool.request_help` (resolved per-call). */
@@ -114,6 +118,7 @@ export class ToolDispatcher {
private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner;
private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
private readonly helpNotificationCopy?: () => { title: string; body: string };
private subscription: { dispose(): void } | null = null;
@@ -130,6 +135,7 @@ export class ToolDispatcher {
this.sessions = deps.sessions;
this.cdp = deps.cdp;
this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow;
this.helpNotificationCopy = deps.helpNotificationCopy;
}
@@ -193,6 +199,8 @@ export class ToolDispatcher {
let body: ResponseFrame;
let startedSession: string | null = null;
try {
const sessionId = sessionIdForBrowserControlMethod(req);
if (sessionId) this.onBrowserControlResumed?.(sessionId);
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
if (isRpcError(result)) {
body = { id: req.id, error: result };
@@ -304,6 +312,12 @@ export class ToolDispatcher {
req.params as SnapshotParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
);
case "tool.observe":
return handleObserve(
this.sessions,
req.params as ObserveParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
);
case "tool.get_html":
return handleGetHtml(
this.sessions,
@@ -457,6 +471,33 @@ function isRpcError(v: unknown): v is RpcError {
);
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) {
case "tool.tab_create":
case "tool.tab_close":
case "tool.tab_select":
case "tool.tab_borrow":
case "tool.tab_return":
case "tool.navigate":
case "tool.navigate_back":
case "tool.navigate_forward":
case "tool.reload":
case "tool.click":
case "tool.fill":
case "tool.press":
case "tool.select":
case "tool.evaluate":
case "tool.observe":
case "tool.request_help":
case "tool.record_start": {
const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null;
}
default:
return null;
}
}
/**
* Resolves never; rejects with `AbortLikeError` as soon as the signal
* fires (or immediately if it is already aborted). Used by the
+64 -13
View File
@@ -1,5 +1,5 @@
// Read-only observation handlers — `tool.screenshot`, `tool.snapshot`,
// and `tool.get_html` (design §7). Each handler resolves the target
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and
// returns a payload that mirrors the bsk-protocol Rust structs.
@@ -15,6 +15,8 @@ import type { SessionManager } from "@/session-manager/manager";
import type {
GetHtmlParams,
GetHtmlResult,
ObserveParams,
ObserveResult,
RpcError,
ScreenshotParams,
ScreenshotResult,
@@ -26,11 +28,13 @@ import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
import { rpcError } from "./errors";
import {
type ChromeTabsApi,
enforceToolTargetScope,
isRpcError,
lookupSession,
resolveTargetTab,
resolveCdpAccessibleTargetTab,
type CdpRunner as SharedCdpRunner,
normaliseRef as sharedNormaliseRef,
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import {
@@ -167,7 +171,13 @@ export async function handleScreenshot(
const ctxOrErr = lookupSession(manager, params, "screenshot");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
const target = await resolveCdpAccessibleTargetTab(
manager,
ctx,
params.tab_id,
deps.tabsApi,
"screenshot",
);
if (isRpcError(target)) return target;
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
const withShotDialogs = <T extends object>(result: T) =>
@@ -181,6 +191,7 @@ export async function handleScreenshot(
const node = resolveSnapshotRef(ctx, ref, target.tabId);
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
if (isRpcError(captured)) return captured;
return withShotDialogs({
@@ -664,6 +675,7 @@ function inputStateFor(
value: string | undefined,
): VomNode["inputState"] {
if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined;
if (capturedNode.formState) return capturedNode.formState;
const formValue = capturedNode.formValue;
if (formValue !== undefined) {
if (formValue === "") return "empty";
@@ -1099,7 +1111,13 @@ export async function handleGetHtml(
const ctxOrErr = lookupSession(manager, params, "get_html");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
const target = await resolveCdpAccessibleTargetTab(
manager,
ctx,
params.tab_id,
deps.tabsApi,
"get_html",
);
if (isRpcError(target)) return target;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
@@ -1108,6 +1126,7 @@ export async function handleGetHtml(
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
let html: string;
if (params.ref) {
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
@@ -1178,28 +1197,44 @@ async function fallbackCapturedViewModel(
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId);
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
} catch {
return fallbackCapturedViewModel(cdp, tabId);
}
}
export async function handleSnapshot(
async function handleVomObservation(
manager: SessionManager,
params: SnapshotParams,
params: SnapshotParams | ObserveParams,
toolName: "snapshot" | "observe",
effect: ToolEffect,
conditionalSurfaceProbe: boolean,
deps: SnapshotDeps = getDefaultDeps(),
): Promise<SnapshotResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "snapshot");
): Promise<SnapshotResult | ObserveResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, toolName);
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
const target = await resolveCdpAccessibleTargetTab(
manager,
ctx,
params.tab_id,
deps.tabsApi,
toolName,
);
if (isRpcError(target)) return target;
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
target.tabId,
@@ -1207,7 +1242,7 @@ export async function handleSnapshot(
{},
);
const axNodes = result.nodes ?? [];
const captured = await captureForVom(deps.cdp, target.tabId);
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
const scene = buildVomScene(axNodes, captured);
const rendered = renderVom(scene, {
maxDepth: params.max_depth,
@@ -1232,3 +1267,19 @@ export async function handleSnapshot(
};
}
}
export async function handleSnapshot(
manager: SessionManager,
params: SnapshotParams,
deps: SnapshotDeps = getDefaultDeps(),
): Promise<SnapshotResult | RpcError> {
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
}
export async function handleObserve(
manager: SessionManager,
params: ObserveParams,
deps: SnapshotDeps = getDefaultDeps(),
): Promise<ObserveResult | RpcError> {
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
}
+4 -4
View File
@@ -369,7 +369,7 @@ async function rearmRecording(
deps: RecordDeps,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// content-script counter, and a single stop decrement left the control mask
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
@@ -561,9 +561,9 @@ export async function handleRecordStart(
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
// Register the recording *before* navigate so content-script syncAgentOverlay
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制").
// Register the recording *before* navigate so the destination content script's
// overlay.ready flow can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing the control mask ("Agent 正在控制").
const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: Trace) => void;
let rejectFinish!: (err: Error) => void;
+115 -3
View File
@@ -32,6 +32,13 @@ export const chromeTabsApi: ChromeTabsApi = {
query: (q) => chrome.tabs.query(q),
};
export interface ResolvedTargetTab {
tabId: number;
windowId: number;
active: boolean;
url?: string;
}
export type { DialogCursor };
/**
@@ -44,6 +51,7 @@ export type { DialogCursor };
*/
export interface CdpRunner {
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void;
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
dispose(): void;
@@ -146,7 +154,7 @@ export async function resolveTargetTab(
ctx: SessionContext,
tabId: number | undefined,
api: ChromeTabsApi,
): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> {
): Promise<ResolvedTargetTab | RpcError> {
if (tabId !== undefined) {
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
return {
@@ -176,7 +184,7 @@ export async function resolveTargetTab(
message: `tab ${tabId} not found in session scope`,
};
}
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true };
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url };
}
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
const first = tabs.find((t) => typeof t.id === "number");
@@ -186,7 +194,12 @@ export async function resolveTargetTab(
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
};
}
return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true };
return {
tabId: first.id,
windowId: ctx.agentWindowId,
active: first.active === true,
url: first.url,
};
}
export function isRpcError(v: unknown): v is RpcError {
@@ -202,6 +215,91 @@ export function isRpcError(v: unknown): v is RpcError {
/** Re-export so M6/M7 tools keep a stable import path. */
export { normaliseRef };
export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation";
const CDP_BLOCKED_PROTOCOLS = new Set([
"chrome:",
"chrome-extension:",
"devtools:",
"edge:",
"brave:",
"vivaldi:",
"opera:",
]);
export function cdpBlockedUrlReason(url: string | undefined): string | null {
if (!url) return null;
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return null;
}
if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol;
if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:";
return null;
}
/**
* Page CDP tools cannot inspect browser/extension internal pages. Check this
* at the tool boundary so Chrome's low-level "Cannot access..." errors do not
* leak as ambiguous page-read failures.
*/
export function enforceCdpAccessibleTarget(
target: ResolvedTargetTab,
toolName: string,
): RpcError | null {
const reason = cdpBlockedUrlReason(target.url);
if (!reason) return null;
return rpcError(
"permission_denied",
"restricted_tab_url",
`${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`,
);
}
function resolvedTargetFromChromeTab(
tab: chrome.tabs.Tab,
fallbackWindowId: number,
): ResolvedTargetTab | null {
if (typeof tab.id !== "number") return null;
return {
tabId: tab.id,
windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId,
active: tab.active === true,
url: tab.url,
};
}
/**
* Resolve a target for page CDP reads. Explicit tab_ids stay exact and are
* rejected when browser security blocks the URL. For default targeting, skip a
* restricted Agent Window active tab if another CDP-accessible tab exists in
* the same Agent Window.
*/
export async function resolveCdpAccessibleTargetTab(
manager: SessionManager,
ctx: SessionContext,
tabId: number | undefined,
api: ChromeTabsApi,
toolName: string,
): Promise<ResolvedTargetTab | RpcError> {
const target = await resolveTargetTab(manager, ctx, tabId, api);
if (isRpcError(target)) return target;
const restricted = enforceCdpAccessibleTarget(target, toolName);
if (!restricted) return target;
if (tabId !== undefined) return restricted;
const tabs = await api.query({ windowId: ctx.agentWindowId });
for (const tab of tabs) {
const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId);
if (!candidate) continue;
if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate;
}
return restricted;
}
/**
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
* MUST refuse to touch a tab outside the session's Agent Window
@@ -224,3 +322,17 @@ export function enforceAgentWindow(
}
return null;
}
/**
* Unified target-scope policy by tool effect. Passive reads may inspect user
* tabs; any tool that dispatches page input must stay inside the Agent Window.
*/
export function enforceToolTargetScope(
ctx: SessionContext,
target: { tabId: number; windowId: number },
effect: ToolEffect,
toolName: string,
): RpcError | null {
if (effect === "passive_read") return null;
return enforceAgentWindow(ctx, target, toolName);
}
@@ -59,6 +59,23 @@ function makeCdp(snapshot: unknown) {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "Runtime.evaluate") {
return {
result: {
value: {
controls: [
{
state: "filled",
sensitive: true,
defaultValue: "",
placeholder: "secret",
},
],
childFrames: [],
},
},
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
@@ -80,6 +97,89 @@ describe("captureViewModel", () => {
expect(input?.attrs).toEqual({ type: "password" });
});
it("does not run conditional surface probes by default", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
expect(result.surfaceProbes).toEqual([]);
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
});
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formDefaultValue: "",
formPlaceholder: "secret",
});
expect(input?.formValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => {
let runtimeCalls = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
runtimeCalls += 1;
return runtimeCalls === 1
? {
result: {
value: [
{
triggerSel: ".menu",
affectedSub: " .items",
label: "Products",
x: 50,
y: 20,
},
],
},
}
: { result: { value: ["Shoes", "Bags"] } };
}
if (method === "Input.dispatchMouseEvent") return {};
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(result.surfaceProbes).toEqual([
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
]);
expect(cdp.send).toHaveBeenCalledWith(
4,
"Input.dispatchMouseEvent",
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
);
expect(cdp.send).toHaveBeenCalledWith(
4,
"Input.dispatchMouseEvent",
expect.objectContaining({ type: "mouseMoved", x: -10, y: -10 }),
);
});
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
// DOMSnapshot inlines an open shadow root's content as descendants of the
// host. The agent's WXT overlay host carries a fixed full-viewport
@@ -349,8 +449,11 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
const div = nodes.find((n) => n.backendNodeId === 12);
// local rect preserves viewport-relative scroll math.
expect(div?.localRect?.y).toBe(-200);
// exported rect is clipped to the top-level viewport before VOM consumes it.
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
@@ -557,8 +660,187 @@ describe("captureViewModel", () => {
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
});
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
const S = [
"html",
"body",
"iframe",
"input",
"position",
"static",
"pointer-events",
"auto",
"type",
"text",
];
const i = (s: string) => S.indexOf(s);
const snapshot = {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
backendNodeId: [10, 11, 13],
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [1] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[950, 760, 200, 100],
],
paintOrders: [0, 1],
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
backendNodeId: [100, 101],
attributes: [[], [i("type"), i("text")]],
},
layout: {
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
bounds: [[20, 20, 120, 60]],
paintOrders: [0],
},
},
],
};
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
});
it("recursively normalizes nested iframe sub-documents", async () => {
const S = [
"html",
"body",
"iframe",
"input",
"position",
"static",
"pointer-events",
"auto",
"type",
"text",
];
const i = (s: string) => S.indexOf(s);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
backendNodeId: [10, 11, 13],
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [1] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[10, 20, 300, 200],
],
paintOrders: [0, 1],
},
},
{
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
backendNodeId: [20, 21, 23],
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [2] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 300, 200],
[5, 6, 100, 80],
],
paintOrders: [0, 1],
},
},
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
backendNodeId: [30, 31],
attributes: [[], [i("type"), i("text")]],
},
layout: {
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
bounds: [[1, 2, 40, 20]],
paintOrders: [0],
},
},
],
};
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
expect(input?.ownerFrameBackendNodeId).toBe(23);
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
});
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
+286 -76
View File
@@ -16,9 +16,14 @@ const STYLE_COL = Object.fromEntries(
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
@@ -31,6 +36,7 @@ export interface CapturedNode {
*/
cursor?: string;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
@@ -53,6 +59,10 @@ export interface CapturedViewModel {
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
conditionalSurfaceProbe?: boolean;
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
@@ -102,48 +112,112 @@ function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) {
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface CapturedFrameFormState {
controls: CapturedFormState[];
childFrames: CapturedFrameFormState[];
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
defaultValue,
placeholder,
...(sensitive ? {} : { value: rawValue }),
};
};
const collect = (doc) => {
const controls = [];
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
controls.push(controlState(el));
remaining -= 1;
}
const childFrames = [];
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
}
return { controls, childFrames };
};
return collect(document);
})()`;
}
function flattenFrameFormStates(
frame: CapturedFrameFormState | undefined,
out: CapturedFormState[][] = [],
): CapturedFormState[][] {
if (!frame) return out;
out.push(Array.isArray(frame.controls) ? frame.controls : []);
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
flattenFrameFormStates(child, out);
}
return out;
}
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
let index = 0;
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
let objectId: string | undefined;
try {
const resolved = await cdp.send<{ object?: { objectId?: string } }>(
tabId,
"DOM.resolveNode",
{
backendNodeId: node.backendNodeId,
},
);
objectId = resolved.object?.objectId;
if (!objectId) continue;
const result = await cdp.send<{
result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } };
}>(tabId, "Runtime.callFunctionOn", {
objectId,
returnByValue: true,
functionDeclaration: `function(){
return {
value: typeof this.value === "string" ? this.value : "",
defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "",
placeholder: typeof this.placeholder === "string" ? this.placeholder : ""
};
}`,
});
const value = result.result?.value;
if (!value) continue;
node.formValue = value.value ?? "";
node.formDefaultValue = value.defaultValue ?? "";
node.formPlaceholder = value.placeholder ?? "";
} catch {
// Best-effort enrichment. DOMSnapshot/AX data still carries the node.
} finally {
if (objectId) {
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
}
const state = states[index];
index += 1;
if (!state) continue;
node.formState = state.state;
node.formDefaultValue = state.defaultValue ?? "";
node.formPlaceholder = state.placeholder ?? "";
if (!state.sensitive && state.value !== undefined) {
node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
try {
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
returnByValue: true,
});
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
for (let i = 0; i < frameNodeGroups.length; i += 1) {
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
}
} catch {
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
}
}
interface RuntimeEvaluateReply {
result?: {
value?: unknown;
@@ -169,6 +243,14 @@ interface ParseDocumentResult {
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
ownerFrameBackendNodeId: number | null;
originInTop: { x: number; y: number };
clipRectInTop: Rect;
scrollX: number;
scrollY: number;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
@@ -183,6 +265,49 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
return dpr >= 1 ? dpr : 1;
}
function viewportRect(viewport: Viewport): Rect {
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
}
function intersectRects(a: Rect, b: Rect): Rect | null {
const x1 = Math.max(a.x, b.x);
const y1 = Math.max(a.y, b.y);
const x2 = Math.min(a.x + a.w, b.x + b.w);
const y2 = Math.min(a.y + a.h, b.y + b.h);
if (x2 <= x1 || y2 <= y1) return null;
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
const translated = {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
return intersectRects(translated, context.clipRectInTop);
}
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
return {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return;
if (typeof node.backendNodeId === "number") {
@@ -235,14 +360,24 @@ function hoverCssScanExpression(): string {
const candidates = [];
const seenLabels = new Set();
const isUnsafeTrigger = (el) => {
if (!(el instanceof HTMLElement)) return true;
const tag = el.tagName.toLowerCase();
if (["input", "textarea", "select", "option"].includes(tag)) return true;
if (el.isContentEditable) return true;
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
return false;
};
for (const pair of pairs) {
let elements;
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof Element)) continue;
if (isUnsafeTrigger(el)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
if (!label || seenLabels.has(label)) continue;
seenLabels.add(label);
@@ -304,6 +439,20 @@ async function wait(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
await cdp
.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: -10,
y: -10,
})
.catch(() =>
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
// best effort
}),
);
}
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
const started = Date.now();
try {
@@ -328,7 +477,6 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
expression: hoverCollectExpression(candidate),
returnByValue: true,
});
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
const subItems = (runtimeValue<string[]>(collected) ?? [])
.map((item) => item.replace(/\s+/g, " ").trim())
.filter(Boolean);
@@ -341,6 +489,8 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
});
} catch {
continue;
} finally {
await clearHover(cdp, tabId);
}
}
return results;
@@ -391,15 +541,13 @@ export async function collectOverlayExcludedBackendIds(
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param scrollX - horizontal scroll offset to subtract (CSS px)
* @param scrollY - vertical scroll offset to subtract (CSS px)
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
dpr: number,
scrollX: number,
scrollY: number,
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
@@ -477,6 +625,7 @@ function parseDocumentNodes(
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
@@ -485,12 +634,13 @@ function parseDocumentNodes(
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
rect = {
x: b[0] / dpr - scrollX,
y: b[1] / dpr - scrollY,
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
w: b[2] / dpr,
h: b[3] / dpr,
};
rect = rectInTop(localRect, context);
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
@@ -508,9 +658,11 @@ function parseDocumentNodes(
nodes.push({
backendNodeId,
parentBackendNodeId,
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
@@ -521,7 +673,81 @@ function parseDocumentNodes(
return { nodes, excludedBackendNodeIds };
}
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
interface ParseFrameDocumentsResult {
iframeNodes: CapturedIframeNodes;
excludedBackendNodeIds: Set<number>;
}
function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
const iframeNodes: CapturedIframeNodes = new Map();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
const childDoc = documents[childDocIndex];
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
if (!iframeNode?.localRect) continue;
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
if (!childClip) {
iframeNodes.set(iframeBackendId, []);
continue;
}
const childContext: FrameContext = {
ownerFrameBackendNodeId: iframeBackendId,
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
clipRectInTop: childClip,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes);
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments(
documents,
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
nextVisited,
);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return { iframeNodes, excludedBackendNodeIds };
}
export async function captureViewModel(
cdp: CdpRunner,
tabId: number,
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
const dpr = devicePixelRatio(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
@@ -552,42 +778,26 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
};
}
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
const topContext: FrameContext = {
ownerFrameBackendNodeId: null,
originInTop: { x: 0, y: 0 },
clipRectInTop: viewportRect(viewport),
scrollX,
scrollY,
};
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
// Build a map from node-array-index → sub-document-index using the sparse
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
const iframeNodes: CapturedIframeNodes = new Map();
if (documents.length > 1) {
const cdi = doc0.nodes?.contentDocumentIndex;
if (cdi?.index && cdi.value) {
for (let k = 0; k < cdi.index.length; k++) {
const nodeArrayIdx = cdi.index[k];
const docIdx = cdi.value[k];
const subDoc = documents[docIdx];
if (!subDoc) continue;
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
if (iframeBid === undefined) continue;
// Sub-documents have their own scroll offsets; bounds are relative
// to the iframe's own coordinate space (not the main viewport).
const subScrollX = subDoc.scrollOffsetX ?? 0;
const subScrollY = subDoc.scrollOffsetY ?? 0;
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
iframeNodes.set(iframeBid, subParsed.nodes);
for (const id of subParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
}
}
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
await enrichFormControlState(cdp, tabId, nodes);
for (const subNodes of iframeNodes.values()) {
await enrichFormControlState(cdp, tabId, subNodes);
}
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
}
+4
View File
@@ -30,6 +30,7 @@ export type RpcErrorReason =
| "option_not_found"
| "single_select_value_count"
| "tab_not_active"
| "restricted_tab_url"
| "borrow_conflict";
export interface RpcErrorData {
@@ -313,6 +314,9 @@ export interface SnapshotResult {
dialogs?: JavaScriptDialogInfo[];
}
export type ObserveParams = SnapshotParams;
export type ObserveResult = SnapshotResult;
export interface GetHtmlParams {
session_id: string;
tab_id?: number;