mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): update vom algorithm, fix overlay problem with background state machine
This commit is contained in:
@@ -119,6 +119,88 @@ describe("ChromiumCdp", () => {
|
||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 9 }, "Page.enable", {});
|
||||
});
|
||||
|
||||
it("validates the attached top-frame URL for page reads", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { url: "https://example.test/page" } } };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(
|
||||
cdp.ensureAttachedToUrl(9, "https://example.test/page#section"),
|
||||
).resolves.toBeUndefined();
|
||||
expect(api.detach).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reattaches when the cached CDP target points at an extension page", async () => {
|
||||
const { api } = fakeApi();
|
||||
let frameReads = 0;
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.getFrameTree") {
|
||||
frameReads += 1;
|
||||
return {
|
||||
frameTree: {
|
||||
frame: {
|
||||
url:
|
||||
frameReads === 1
|
||||
? "chrome-extension://other-extension/page.html"
|
||||
: "https://example.test/page",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(cdp.ensureAttachedToUrl(9, "https://example.test/page")).resolves.toBeUndefined();
|
||||
expect(api.attach).toHaveBeenCalledTimes(2);
|
||||
expect(api.detach).toHaveBeenCalledWith({ tabId: 9 });
|
||||
});
|
||||
|
||||
it("rejects when reattach still points at the wrong page", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { url: "https://wrong.test/" } } };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(cdp.ensureAttachedToUrl(9, "https://example.test/")).rejects.toThrow(
|
||||
/CDP target URL mismatch/,
|
||||
);
|
||||
expect(api.attach).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("clears the attach cache after an internal-page access error", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "DOM.getDocument") {
|
||||
throw new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(cdp.ensureAttached(9)).resolves.toBeUndefined();
|
||||
expect(cdp.isAttached(9)).toBe(true);
|
||||
await expect(cdp.send(9, "DOM.getDocument")).rejects.toThrow(/chrome-extension/);
|
||||
expect(cdp.isAttached(9)).toBe(false);
|
||||
});
|
||||
|
||||
it("enables console capture domains best-effort during attach", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
|
||||
@@ -93,6 +93,15 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const CDP_INTERNAL_PROTOCOLS = new Set([
|
||||
"chrome:",
|
||||
"chrome-extension:",
|
||||
"devtools:",
|
||||
"edge:",
|
||||
"brave:",
|
||||
"vivaldi:",
|
||||
"opera:",
|
||||
]);
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -173,6 +182,31 @@ export class ChromiumCdp {
|
||||
await attach;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach and verify Chrome's actual CDP target still points at the page URL
|
||||
* Chrome tabs reported to the tool resolver. A stale debugger target can
|
||||
* otherwise surface another extension/internal URL while the tab record looks
|
||||
* like a normal page.
|
||||
*/
|
||||
async ensureAttachedToUrl(tabId: number, expectedUrl: string | undefined): Promise<void> {
|
||||
await this.ensureAttached(tabId);
|
||||
if (!expectedUrl) return;
|
||||
|
||||
const first = await this.readTopFrameUrl(tabId).catch((err) => {
|
||||
if (isInternalTargetAccessError(err)) return null;
|
||||
throw err;
|
||||
});
|
||||
if (first && cdpUrlsCompatible(first, expectedUrl)) return;
|
||||
|
||||
await this.forceReattach(tabId);
|
||||
const second = await this.readTopFrameUrl(tabId);
|
||||
if (cdpUrlsCompatible(second, expectedUrl)) return;
|
||||
|
||||
throw new Error(
|
||||
`CDP target URL mismatch for tab ${tabId}: expected ${expectedUrl}, attached to ${second}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a CDP command and decode the result as `T`. Throws on any
|
||||
* `chrome.runtime.lastError`.
|
||||
@@ -185,6 +219,9 @@ export class ChromiumCdp {
|
||||
const result = await this.api.sendCommand({ tabId }, method, params ?? {});
|
||||
return result as T;
|
||||
} catch (err) {
|
||||
if (isInternalTargetAccessError(err)) {
|
||||
this.clearAttachedState(tabId);
|
||||
}
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
@@ -359,6 +396,31 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async forceReattach(tabId: number): Promise<void> {
|
||||
this.clearAttachedState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch {
|
||||
// Chrome may already have detached; attach below is the source of truth.
|
||||
}
|
||||
await this.ensureAttached(tabId);
|
||||
}
|
||||
|
||||
private clearAttachedState(tabId: number): void {
|
||||
this.attachInFlight.delete(tabId);
|
||||
this.attachedTabs.delete(tabId);
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
}
|
||||
|
||||
private async readTopFrameUrl(tabId: number): Promise<string> {
|
||||
const frameTree = await this.api.sendCommand({ tabId }, "Page.getFrameTree", {});
|
||||
const url = topFrameUrl(frameTree);
|
||||
if (!url) throw new Error(`Page.getFrameTree returned no top-frame URL for tab ${tabId}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
@@ -906,3 +968,49 @@ function normalizeError(err: unknown): Error {
|
||||
}
|
||||
return new Error("unknown chrome.debugger error");
|
||||
}
|
||||
|
||||
function isInternalTargetAccessError(err: unknown): boolean {
|
||||
const message = normalizeError(err).message;
|
||||
return (
|
||||
/Cannot access a chrome-extension:\/\/ URL/i.test(message) ||
|
||||
/Cannot access a chrome:\/\/ URL/i.test(message) ||
|
||||
/Cannot access contents of url/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
function topFrameUrl(frameTree: unknown): string | null {
|
||||
const raw = (frameTree ?? {}) as Record<string, unknown>;
|
||||
const tree = (raw.frameTree ?? {}) as Record<string, unknown>;
|
||||
const frame = (tree.frame ?? {}) as Record<string, unknown>;
|
||||
return typeof frame.url === "string" && frame.url.length > 0 ? frame.url : null;
|
||||
}
|
||||
|
||||
function cdpUrlsCompatible(actualUrl: string, expectedUrl: string): boolean {
|
||||
const actual = normalizeComparableUrl(actualUrl);
|
||||
const expected = normalizeComparableUrl(expectedUrl);
|
||||
if (!actual || !expected) return false;
|
||||
if (isInternalCdpUrl(actual)) return false;
|
||||
return actual === expected;
|
||||
}
|
||||
|
||||
function normalizeComparableUrl(value: string): string | null {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(value);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
parsed.hash = "";
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
function isInternalCdpUrl(value: string): boolean {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(value);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (CDP_INTERNAL_PROTOCOLS.has(parsed.protocol)) return true;
|
||||
return parsed.protocol === "about:" && parsed.pathname !== "blank";
|
||||
}
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import { RiStopCircleLine } from "@remixicon/react";
|
||||
import { useEffect, useState } from "react";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface ControlOverlayProps {
|
||||
visible: boolean;
|
||||
interrupting: boolean;
|
||||
automationBypass: boolean;
|
||||
onInterrupt: () => void;
|
||||
}
|
||||
|
||||
export function ControlOverlay({
|
||||
visible,
|
||||
interrupting,
|
||||
automationBypass,
|
||||
onInterrupt,
|
||||
}: ControlOverlayProps) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (visible) {
|
||||
const raf = requestAnimationFrame(() => setShow(true));
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}
|
||||
setShow(false);
|
||||
}, [visible]);
|
||||
|
||||
if (!visible) return null;
|
||||
|
||||
const pointerEvents = automationBypass ? "none" : "auto";
|
||||
|
||||
return (
|
||||
<>
|
||||
<style>{`
|
||||
@keyframes bsk-breathe {
|
||||
0%, 100% {
|
||||
box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25);
|
||||
}
|
||||
50% {
|
||||
box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5);
|
||||
}
|
||||
}
|
||||
`}</style>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay"
|
||||
style={{
|
||||
position: "fixed",
|
||||
inset: 0,
|
||||
zIndex: 2147483646,
|
||||
pointerEvents: "none",
|
||||
animation: "bsk-breathe 3s ease-in-out infinite",
|
||||
opacity: show ? 1 : 0,
|
||||
transition: "opacity 300ms ease-out",
|
||||
}}
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-blocker"
|
||||
onPointerDown={(event) => {
|
||||
if (automationBypass) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onClick={(event) => {
|
||||
if (automationBypass) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
style={{
|
||||
position: "fixed",
|
||||
inset: 0,
|
||||
zIndex: 2147483646,
|
||||
pointerEvents,
|
||||
background: "transparent",
|
||||
opacity: show ? 1 : 0,
|
||||
transition: "opacity 300ms ease-out",
|
||||
}}
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-pill"
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
zIndex: 2147483647,
|
||||
// Always receive clicks so Interrupt works even while the page
|
||||
// blocker is in automation-bypass mode (CDP pass-through).
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
backgroundColor: "#fff",
|
||||
borderRadius: 9999,
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
opacity: show ? 1 : 0,
|
||||
transition: "opacity 300ms ease-out",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
}}
|
||||
>
|
||||
<img
|
||||
src={logoUrl}
|
||||
alt="browser-skill"
|
||||
style={{ width: 24, height: 24, borderRadius: 4, flexShrink: 0 }}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
}}
|
||||
>
|
||||
{t("controlOverlay.status")}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
data-slot="control-overlay-stop-all"
|
||||
disabled={interrupting}
|
||||
onClick={onInterrupt}
|
||||
style={{
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
border: "none",
|
||||
borderRadius: 9999,
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: 15,
|
||||
fontWeight: 600,
|
||||
color: "#fff",
|
||||
backgroundColor: interrupting ? "#9ca3af" : "#f97316",
|
||||
cursor: interrupting ? "default" : "pointer",
|
||||
opacity: interrupting ? 0.7 : 1,
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: 1,
|
||||
}}
|
||||
>
|
||||
<RiStopCircleLine size={18} color="#fff" />
|
||||
{interrupting ? t("controlOverlay.interrupting") : t("controlOverlay.interrupt")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { cleanup, render } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { ControlOverlay } from "../ControlOverlay";
|
||||
|
||||
describe("ControlOverlay", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
interrupting={false}
|
||||
automationBypass={true}
|
||||
onInterrupt={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
|
||||
|
||||
const pill = container.querySelector("[data-slot='control-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
|
||||
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
|
||||
expect(stopBtn).toBeTruthy();
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
it("uses pointer-events auto on blocker when automationBypass is false", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
interrupting={false}
|
||||
automationBypass={false}
|
||||
onInterrupt={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
});
|
||||
@@ -116,4 +116,18 @@ describe("OverlayController", () => {
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
});
|
||||
|
||||
it("hides control while paused and shows it again only after authoritative control mode", () => {
|
||||
const controller = new OverlayController();
|
||||
|
||||
controller.applyAgentControlMode("sess-1", "paused");
|
||||
expect(controller.snapshot().activeSessionId).toBe("sess-1");
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
|
||||
controller.applyAgentControlMode("sess-1", "control");
|
||||
expect(controller.snapshot().activeSessionId).toBe("sess-1");
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { OverlayMode } from "@/lib/overlay-bridge";
|
||||
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
import type { RecordRequestData } from "./RecordOverlay";
|
||||
@@ -9,6 +10,7 @@ export interface OverlayState {
|
||||
controlVisible: boolean;
|
||||
interrupting: boolean;
|
||||
activeSessionId: string | null;
|
||||
controlMode: OverlayMode;
|
||||
automationBypassCount: number;
|
||||
/**
|
||||
* After record Finish/Stop clears `activeRecord`, the session is still
|
||||
@@ -31,6 +33,7 @@ export class OverlayController {
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
controlMode: "hidden",
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
@@ -44,7 +47,7 @@ export class OverlayController {
|
||||
}
|
||||
|
||||
isControlVisible(): boolean {
|
||||
return this.state.activeSessionId !== null;
|
||||
return this.state.activeSessionId !== null && this.state.controlMode === "control";
|
||||
}
|
||||
|
||||
addBorrowRequest(request: BorrowRequestData): void {
|
||||
@@ -61,6 +64,25 @@ export class OverlayController {
|
||||
activateAgentSession(sessionId: string): void {
|
||||
this.state.activeSessionId = sessionId;
|
||||
this.state.interrupting = false;
|
||||
this.state.controlMode = "control";
|
||||
}
|
||||
|
||||
applyAgentControlMode(sessionId: string | null, mode: OverlayMode): void {
|
||||
if (mode === "control" && sessionId) {
|
||||
this.state.activeSessionId = sessionId;
|
||||
this.state.interrupting = false;
|
||||
this.state.controlMode = "control";
|
||||
return;
|
||||
}
|
||||
if ((mode === "interrupting" || mode === "paused") && sessionId) {
|
||||
this.state.activeSessionId = sessionId;
|
||||
this.state.interrupting = mode === "interrupting";
|
||||
this.state.controlMode = mode;
|
||||
return;
|
||||
}
|
||||
this.state.activeSessionId = null;
|
||||
this.state.interrupting = false;
|
||||
this.state.controlMode = "hidden";
|
||||
}
|
||||
|
||||
setInterrupting(interrupting: boolean): void {
|
||||
@@ -118,6 +140,7 @@ export class OverlayController {
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
controlMode: "hidden",
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
|
||||
@@ -9,12 +9,16 @@ import {
|
||||
} from "@/lib/instance-id";
|
||||
import { startKeepalive } from "@/lib/keepalive";
|
||||
import {
|
||||
OVERLAY_AGENT_STATE,
|
||||
OVERLAY_AUTOMATION_BYPASS,
|
||||
OVERLAY_MSG_INTERRUPT,
|
||||
OVERLAY_MSG_READY,
|
||||
OVERLAY_MSG_WHO_AM_I,
|
||||
type OverlayAgentStateMessage,
|
||||
type OverlayInterruptRequest,
|
||||
type OverlayInterruptResponse,
|
||||
type OverlayMessage,
|
||||
type OverlayMode,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
@@ -44,6 +48,91 @@ export default defineBackground(() => {
|
||||
const sessions = new SessionManager();
|
||||
const cdp = new ChromiumCdp();
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
|
||||
function setControlMode(sessionId: string, mode: OverlayMode): void {
|
||||
if (controlModes.get(sessionId) === mode) return;
|
||||
controlModes.set(sessionId, mode);
|
||||
overlayGeneration += 1;
|
||||
const ctx = sessions.get(sessionId);
|
||||
if (ctx) void pushOverlayStateForWindow(ctx.agentWindowId);
|
||||
}
|
||||
|
||||
function overlayStateForWindow(windowId?: number): OverlayAgentStateMessage {
|
||||
const ctx = typeof windowId === "number" ? sessions.findByWindowId(windowId) : null;
|
||||
if (!ctx) {
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: null,
|
||||
mode: "hidden",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: ctx.sessionId,
|
||||
mode: controlModes.get(ctx.sessionId) ?? "control",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function pushOverlayStateToTab(
|
||||
tabId: number,
|
||||
state: OverlayAgentStateMessage,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, state);
|
||||
} catch {
|
||||
// Restricted pages and not-yet-loaded content scripts cannot receive messages.
|
||||
}
|
||||
}
|
||||
|
||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||
const state = overlayStateForWindow(windowId);
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
await Promise.all(
|
||||
tabs.map((tab) =>
|
||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function pushAllAgentOverlayStates(): void {
|
||||
const windowIds = new Set(sessions.list().map((ctx) => ctx.agentWindowId));
|
||||
for (const windowId of windowIds) {
|
||||
void pushOverlayStateForWindow(windowId);
|
||||
}
|
||||
}
|
||||
|
||||
function onOverlaySessionStateChanged(): void {
|
||||
void sessionsLive.syncFromManager();
|
||||
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||
for (const sessionId of controlModes.keys()) {
|
||||
if (!liveSessionIds.has(sessionId)) controlModes.delete(sessionId);
|
||||
}
|
||||
overlayGeneration += 1;
|
||||
pushAllAgentOverlayStates();
|
||||
}
|
||||
|
||||
function onBrowserControlResumed(sessionId: string): void {
|
||||
const ctx = sessions.get(sessionId);
|
||||
if (!ctx) return;
|
||||
setControlMode(sessionId, "control");
|
||||
}
|
||||
|
||||
function pushOverlayStateForAgentWindow(windowId: number): void {
|
||||
if (!sessions.findByWindowId(windowId)) return;
|
||||
void pushOverlayStateForWindow(windowId);
|
||||
}
|
||||
chrome.tabs.onActivated.addListener((activeInfo) => {
|
||||
pushOverlayStateForAgentWindow(activeInfo.windowId);
|
||||
});
|
||||
chrome.tabs.onUpdated.addListener((_tabId, changeInfo, tab) => {
|
||||
if (changeInfo.status !== "complete") return;
|
||||
if (typeof tab.windowId !== "number") return;
|
||||
pushOverlayStateForAgentWindow(tab.windowId);
|
||||
});
|
||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
@@ -52,9 +141,8 @@ export default defineBackground(() => {
|
||||
transport,
|
||||
sessions,
|
||||
cdp,
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
approveBorrow: (ctx) =>
|
||||
requestBorrowConfirmation(ctx.tabId, {
|
||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||
@@ -124,9 +212,7 @@ export default defineBackground(() => {
|
||||
manager: sessions,
|
||||
transport,
|
||||
cdp,
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
});
|
||||
|
||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||
@@ -197,9 +283,19 @@ export default defineBackground(() => {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_READY) {
|
||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_INTERRUPT) {
|
||||
const req = msg as OverlayInterruptRequest;
|
||||
const ctx = sessions.get(req.sessionId);
|
||||
if (ctx) setControlMode(req.sessionId, "interrupting");
|
||||
void handleOverlayInterrupt(transport, req.sessionId).then((reply) => {
|
||||
if (reply.ok && sessions.get(req.sessionId)) {
|
||||
setControlMode(req.sessionId, "paused");
|
||||
}
|
||||
sendResponse(reply);
|
||||
});
|
||||
return true; // keep channel open
|
||||
|
||||
@@ -3,7 +3,6 @@ import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
@@ -27,13 +26,12 @@ import {
|
||||
} from "@/lib/help-bridge";
|
||||
import {
|
||||
isOverlayAgentOverlayResetMessage,
|
||||
isOverlayAgentStateMessage,
|
||||
OVERLAY_AUTOMATION_BYPASS,
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
OVERLAY_HOST_NAME,
|
||||
OVERLAY_MSG_WHO_AM_I,
|
||||
OVERLAY_MSG_READY,
|
||||
type OverlayAgentOverlayResetMessage,
|
||||
type OverlayAgentStateMessage,
|
||||
type OverlayAutomationBypassMessage,
|
||||
type OverlayWhoAmIResponse,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
@@ -42,12 +40,12 @@ import {
|
||||
type RecordStartAck,
|
||||
type RecordStopAck,
|
||||
} from "@/lib/record-bridge";
|
||||
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
BorrowRequestMessage,
|
||||
BorrowResponseMessage,
|
||||
} from "@/tools/borrow-confirmation";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
// Run at document_end so the overlay does not block first paint. Only attach
|
||||
// in the top-level frame so iframes do not double-render overlays.
|
||||
@@ -65,34 +63,264 @@ export default defineContentScript({
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
let overlayContainer: HTMLElement | null = null;
|
||||
let controlRoot: HTMLElement | null = null;
|
||||
let activeAgentState: OverlayAgentStateMessage | null = null;
|
||||
let hostLossReported = false;
|
||||
let remountInProgress = false;
|
||||
|
||||
const ui = await createShadowRootUi(ctx, {
|
||||
name: OVERLAY_HOST_NAME,
|
||||
name: "browser-skill-overlay",
|
||||
position: "inline",
|
||||
anchor: "html",
|
||||
css: overlayCss,
|
||||
onMount(container, _shadow, shadowHost) {
|
||||
shadowHost.setAttribute("aria-hidden", "true");
|
||||
shadowHost.setAttribute(OVERLAY_HOST_MARKER_ATTR, "");
|
||||
shadowHost.setAttribute("data-bsk-overlay", "");
|
||||
overlayHost = shadowHost;
|
||||
overlayContainer = container;
|
||||
hostLossReported = false;
|
||||
const app = document.createElement("div");
|
||||
app.className = "bsk-overlay-root";
|
||||
container.append(app);
|
||||
reactRoot = ReactDOM.createRoot(app);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
void requestOverlayState();
|
||||
return reactRoot;
|
||||
},
|
||||
onRemove(root) {
|
||||
overlayHost = null;
|
||||
overlayContainer = null;
|
||||
controlRoot = null;
|
||||
root?.unmount();
|
||||
reactRoot = null;
|
||||
},
|
||||
});
|
||||
|
||||
function renderOverlay() {
|
||||
function removeControlOverlay(): void {
|
||||
controlRoot?.remove();
|
||||
controlRoot = null;
|
||||
}
|
||||
|
||||
function setControlSurfaceActive(active: boolean): void {
|
||||
const host = overlayHost;
|
||||
const container = overlayContainer;
|
||||
if (!host || !container) return;
|
||||
|
||||
if (active) {
|
||||
Object.assign(host.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
zIndex: "2147483647",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
Object.assign(container.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
pointerEvents: "none",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
host.style.removeProperty("position");
|
||||
host.style.removeProperty("inset");
|
||||
host.style.removeProperty("z-index");
|
||||
host.style.removeProperty("pointer-events");
|
||||
container.style.removeProperty("position");
|
||||
container.style.removeProperty("inset");
|
||||
container.style.removeProperty("pointer-events");
|
||||
}
|
||||
|
||||
function ensureControlOverlayRoot(): HTMLElement | null {
|
||||
if (!overlayContainer) return null;
|
||||
if (!controlRoot) {
|
||||
controlRoot = document.createElement("div");
|
||||
controlRoot.className = "bsk-control-overlay";
|
||||
controlRoot.style.opacity = "0";
|
||||
controlRoot.style.transition = "opacity 300ms ease-out";
|
||||
overlayContainer.append(controlRoot);
|
||||
window.requestAnimationFrame(() => {
|
||||
if (controlRoot) controlRoot.style.opacity = "1";
|
||||
});
|
||||
}
|
||||
return controlRoot;
|
||||
}
|
||||
|
||||
function createStopCircleIcon(): SVGSVGElement {
|
||||
const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
|
||||
svg.setAttribute("width", "18");
|
||||
svg.setAttribute("height", "18");
|
||||
svg.setAttribute("viewBox", "0 0 24 24");
|
||||
svg.setAttribute("fill", "currentColor");
|
||||
svg.setAttribute("aria-hidden", "true");
|
||||
svg.style.color = "#fff";
|
||||
svg.style.flexShrink = "0";
|
||||
const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
|
||||
path.setAttribute(
|
||||
"d",
|
||||
"M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10Zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16ZM9 9h6v6H9V9Z",
|
||||
);
|
||||
svg.append(path);
|
||||
return svg;
|
||||
}
|
||||
|
||||
function renderControlOverlay(): void {
|
||||
const overlayState = overlays.snapshot();
|
||||
if (!shouldShowAgentControlOverlay(overlayState)) {
|
||||
removeControlOverlay();
|
||||
setControlSurfaceActive(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const root = ensureControlOverlayRoot();
|
||||
if (!root) return;
|
||||
|
||||
const pointerEvents = overlayState.automationBypassCount > 0 ? "none" : "auto";
|
||||
setControlSurfaceActive(true);
|
||||
root.innerHTML = "";
|
||||
Object.assign(root.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
pointerEvents: "none",
|
||||
});
|
||||
|
||||
const style = document.createElement("style");
|
||||
style.textContent = `
|
||||
@keyframes bsk-breathe {
|
||||
0%, 100% { box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25); }
|
||||
50% { box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5); }
|
||||
}
|
||||
`;
|
||||
|
||||
const border = document.createElement("div");
|
||||
border.setAttribute("data-slot", "control-overlay");
|
||||
Object.assign(border.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
zIndex: "2147483646",
|
||||
pointerEvents: "none",
|
||||
animation: "bsk-breathe 3s ease-in-out infinite",
|
||||
});
|
||||
|
||||
const blocker = document.createElement("div");
|
||||
blocker.setAttribute("data-slot", "control-overlay-blocker");
|
||||
Object.assign(blocker.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
zIndex: "2147483646",
|
||||
pointerEvents,
|
||||
background: "transparent",
|
||||
});
|
||||
blocker.addEventListener("pointerdown", (event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
});
|
||||
blocker.addEventListener("click", (event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
});
|
||||
blocker.addEventListener(
|
||||
"wheel",
|
||||
(event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
},
|
||||
{ passive: false },
|
||||
);
|
||||
blocker.addEventListener(
|
||||
"touchmove",
|
||||
(event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
},
|
||||
{ passive: false },
|
||||
);
|
||||
|
||||
const pill = document.createElement("div");
|
||||
pill.setAttribute("data-slot", "control-overlay-pill");
|
||||
Object.assign(pill.style, {
|
||||
position: "fixed",
|
||||
bottom: "32px",
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
zIndex: "2147483647",
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "12px",
|
||||
borderRadius: "999px",
|
||||
backgroundColor: "#fff",
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
});
|
||||
|
||||
const logo = document.createElement("img");
|
||||
logo.src = logoUrl;
|
||||
logo.alt = "browser-skill";
|
||||
Object.assign(logo.style, {
|
||||
width: "24px",
|
||||
height: "24px",
|
||||
borderRadius: "4px",
|
||||
flexShrink: "0",
|
||||
});
|
||||
|
||||
const label = document.createElement("span");
|
||||
label.textContent = i18n.t("controlOverlay.status", { ns: "extension" });
|
||||
Object.assign(label.style, {
|
||||
fontSize: "16px",
|
||||
fontWeight: "500",
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
});
|
||||
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.setAttribute("data-slot", "control-overlay-stop-all");
|
||||
button.disabled = overlayState.interrupting;
|
||||
Object.assign(button.style, {
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "6px",
|
||||
border: "none",
|
||||
borderRadius: "999px",
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: "15px",
|
||||
fontWeight: "600",
|
||||
color: "#fff",
|
||||
backgroundColor: overlayState.interrupting ? "#9ca3af" : "#f97316",
|
||||
cursor: overlayState.interrupting ? "default" : "pointer",
|
||||
opacity: overlayState.interrupting ? "0.7" : "1",
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: "1",
|
||||
});
|
||||
button.append(
|
||||
createStopCircleIcon(),
|
||||
document.createTextNode(
|
||||
overlayState.interrupting
|
||||
? i18n.t("controlOverlay.interrupting", { ns: "extension" })
|
||||
: i18n.t("controlOverlay.interrupt", { ns: "extension" }),
|
||||
),
|
||||
);
|
||||
button.addEventListener("click", (event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
handleInterrupt();
|
||||
});
|
||||
|
||||
pill.append(logo, label, button);
|
||||
root.append(style, border, blocker, pill);
|
||||
}
|
||||
|
||||
function renderReactOverlays(): void {
|
||||
const overlayState = overlays.snapshot();
|
||||
reactRoot?.render(
|
||||
React.createElement(
|
||||
@@ -104,12 +332,6 @@ export default defineContentScript({
|
||||
React.createElement(BorrowConfirmationOverlay, {
|
||||
requests: overlayState.borrowRequests,
|
||||
}),
|
||||
React.createElement(ControlOverlay, {
|
||||
visible: shouldShowAgentControlOverlay(overlayState),
|
||||
interrupting: overlayState.interrupting,
|
||||
automationBypass: overlayState.automationBypassCount > 0,
|
||||
onInterrupt: handleInterrupt,
|
||||
}),
|
||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
||||
),
|
||||
@@ -117,6 +339,23 @@ export default defineContentScript({
|
||||
);
|
||||
}
|
||||
|
||||
function renderAll(): void {
|
||||
renderReactOverlays();
|
||||
renderControlOverlay();
|
||||
}
|
||||
|
||||
function clearCurrentAgentSession(): void {
|
||||
const sessionId = overlays.snapshot().activeSessionId;
|
||||
if (!sessionId) return;
|
||||
resetAgentOverlayState(sessionId);
|
||||
}
|
||||
|
||||
function applyOverlayState(state: OverlayAgentStateMessage): void {
|
||||
activeAgentState = state;
|
||||
overlays.applyAgentControlMode(state.sessionId, state.mode);
|
||||
renderAll();
|
||||
}
|
||||
|
||||
function resetAgentOverlayState(sessionId: string) {
|
||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||
if (previousHelp) {
|
||||
@@ -125,7 +364,7 @@ export default defineContentScript({
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
}
|
||||
|
||||
function handleInterrupt() {
|
||||
@@ -137,12 +376,8 @@ export default defineContentScript({
|
||||
return;
|
||||
}
|
||||
overlays.setInterrupting(true);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => {
|
||||
// Always retract the mask after the round trip resolves
|
||||
// (success, failure, or timeout). Cancellation is fire-and-
|
||||
// forget on the daemon side; the user must not be stuck
|
||||
// behind a transient issue. The Agent Window stays open.
|
||||
resetAgentOverlayState(sessionId);
|
||||
if (!reply.ok) {
|
||||
console.warn("[bsk overlay] interrupt did not get a clean ack from daemon");
|
||||
@@ -157,6 +392,7 @@ export default defineContentScript({
|
||||
| HelpRequestMessage
|
||||
| HelpCancelMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||
@@ -183,11 +419,11 @@ export default defineContentScript({
|
||||
});
|
||||
},
|
||||
});
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
},
|
||||
},
|
||||
sendResponse as unknown as
|
||||
@@ -205,7 +441,12 @@ export default defineContentScript({
|
||||
) {
|
||||
const bypassMsg = message as OverlayAutomationBypassMessage;
|
||||
overlays.setAutomationBypass(bypassMsg.enabled);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isOverlayAgentStateMessage(message)) {
|
||||
applyOverlayState(message);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -216,7 +457,7 @@ export default defineContentScript({
|
||||
|
||||
if (message.type === "borrow-cancel") {
|
||||
overlays.removeBorrowRequest(message.requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -224,7 +465,7 @@ export default defineContentScript({
|
||||
const state = overlays.snapshot();
|
||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||
overlays.clearAgentHelpRequest(message.requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -244,7 +485,7 @@ export default defineContentScript({
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
sendResponse({ type: HELP_ACK, ok: true });
|
||||
return false;
|
||||
}
|
||||
@@ -256,7 +497,7 @@ export default defineContentScript({
|
||||
responded = true;
|
||||
sendResponse({ type: "borrow-response", allowed });
|
||||
overlays.removeBorrowRequest(message.requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
};
|
||||
|
||||
overlays.addBorrowRequest({
|
||||
@@ -267,7 +508,7 @@ export default defineContentScript({
|
||||
onAllow: () => respond(true),
|
||||
onDeny: () => respond(false),
|
||||
});
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -286,7 +527,7 @@ export default defineContentScript({
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
overlays.clearAgentHelpRequest(requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||
console.debug("[bsk overlay] help finish failed", err);
|
||||
});
|
||||
@@ -305,7 +546,7 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<boolean> {
|
||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
try {
|
||||
const helpQuery = (await chrome.runtime.sendMessage({
|
||||
@@ -313,38 +554,21 @@ export default defineContentScript({
|
||||
})) as HelpQueryResponse | undefined;
|
||||
if (helpQuery?.active && helpQuery.request) {
|
||||
mountHelpRequest(helpQuery.request);
|
||||
renderOverlay();
|
||||
return true;
|
||||
renderAll();
|
||||
return;
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] help query failed", err);
|
||||
}
|
||||
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function syncAgentOverlay(): Promise<void> {
|
||||
if (!(await anySessionLive())) return;
|
||||
async function queryActiveRecord(): Promise<void> {
|
||||
try {
|
||||
const helpActive = await queryActiveHelpWithRetry();
|
||||
const reply = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_WHO_AM_I,
|
||||
})) as OverlayWhoAmIResponse | undefined;
|
||||
if (!reply?.sessionId) return;
|
||||
|
||||
// Query / re-arm recording *before* activating the control overlay so
|
||||
// record start (navigate → content load) does not flash
|
||||
// 「Agent 正在控制」before RecordOverlay mounts.
|
||||
let recordQuery: { active?: boolean; requestId?: string } | undefined;
|
||||
try {
|
||||
recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as { active?: boolean; requestId?: string } | undefined;
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] record query failed", err);
|
||||
}
|
||||
|
||||
const recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as { active?: boolean; requestId?: string } | undefined;
|
||||
if (
|
||||
recordQuery?.active &&
|
||||
typeof recordQuery.requestId === "string" &&
|
||||
@@ -360,26 +584,38 @@ export default defineContentScript({
|
||||
});
|
||||
},
|
||||
});
|
||||
renderAll();
|
||||
}
|
||||
|
||||
if (!helpActive && overlays.snapshot().activeHelp === null) {
|
||||
void queryActiveHelpWithRetry();
|
||||
}
|
||||
|
||||
overlays.activateAgentSession(reply.sessionId);
|
||||
renderOverlay();
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] syncAgentOverlay failed", err);
|
||||
console.debug("[bsk overlay] record query failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshAuxiliaryOverlayState(): Promise<void> {
|
||||
await Promise.all([queryActiveHelpWithRetry(), queryActiveRecord()]);
|
||||
}
|
||||
|
||||
async function requestOverlayState(): Promise<void> {
|
||||
try {
|
||||
const state = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_READY,
|
||||
})) as OverlayAgentStateMessage | undefined;
|
||||
if (state && isOverlayAgentStateMessage(state)) {
|
||||
applyOverlayState(state);
|
||||
}
|
||||
void refreshAuxiliaryOverlayState();
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] overlay.ready failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
const onPageShow = (event: PageTransitionEvent) => {
|
||||
if (event.persisted) void syncAgentOverlay();
|
||||
if (event.persisted) void requestOverlayState();
|
||||
};
|
||||
|
||||
ui.mount();
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void syncAgentOverlay();
|
||||
void requestOverlayState();
|
||||
|
||||
window.addEventListener("pageshow", onPageShow);
|
||||
|
||||
@@ -391,7 +627,8 @@ export default defineContentScript({
|
||||
remountInProgress = true;
|
||||
try {
|
||||
ui.mount();
|
||||
void syncAgentOverlay();
|
||||
if (activeAgentState) applyOverlayState(activeAgentState);
|
||||
void requestOverlayState();
|
||||
} finally {
|
||||
remountInProgress = false;
|
||||
}
|
||||
@@ -414,16 +651,3 @@ export default defineContentScript({
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
async function anySessionLive(): Promise<boolean> {
|
||||
if (!chrome.storage?.session?.get) return true;
|
||||
try {
|
||||
const result = (await chrome.storage.session.get({
|
||||
[SESSIONS_LIVE_FLAG_KEY]: false,
|
||||
})) as Record<string, unknown> | undefined;
|
||||
return Boolean(result?.[SESSIONS_LIVE_FLAG_KEY]);
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] sessions-live flag read failed", err);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*
|
||||
* Mirrors the borrow-confirmation message style (see
|
||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||
* enter "help mode" (render HelpRequestOverlay + hide the control mask);
|
||||
* the content script acks display, then later reports Done / Cancel.
|
||||
*/
|
||||
|
||||
|
||||
@@ -3,9 +3,8 @@
|
||||
* between the content-script control overlay and the background SW.
|
||||
*
|
||||
* Content script → background:
|
||||
* - `{ kind: "overlay.who_am_i", tabId, windowId }` → background
|
||||
* replies `{ sessionId: string | null }` so the overlay knows
|
||||
* whether to render itself.
|
||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||
* overlay state for the sender's window.
|
||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||
* every inflight + queued tool call for that session with
|
||||
@@ -14,6 +13,7 @@
|
||||
*/
|
||||
|
||||
export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i";
|
||||
export const OVERLAY_MSG_READY = "overlay.ready";
|
||||
export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt";
|
||||
|
||||
/**
|
||||
@@ -57,6 +57,22 @@ export interface OverlayWhoAmIResponse {
|
||||
sessionId: string | null;
|
||||
}
|
||||
|
||||
export interface OverlayReadyRequest {
|
||||
kind: typeof OVERLAY_MSG_READY;
|
||||
}
|
||||
|
||||
export type OverlayMode = "control" | "interrupting" | "paused" | "hidden";
|
||||
|
||||
/** Background → content: complete, authoritative control-overlay state. */
|
||||
export const OVERLAY_AGENT_STATE = "bh-agent-overlay-state";
|
||||
|
||||
export interface OverlayAgentStateMessage {
|
||||
type: typeof OVERLAY_AGENT_STATE;
|
||||
sessionId: string | null;
|
||||
mode: OverlayMode;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export interface OverlayInterruptRequest {
|
||||
kind: typeof OVERLAY_MSG_INTERRUPT;
|
||||
sessionId: string;
|
||||
@@ -90,4 +106,23 @@ export function isOverlayAgentOverlayResetMessage(
|
||||
return candidate.type === OVERLAY_AGENT_OVERLAY_RESET && typeof candidate.sessionId === "string";
|
||||
}
|
||||
|
||||
export type OverlayMessage = OverlayWhoAmIRequest | OverlayInterruptRequest;
|
||||
export function isOverlayAgentStateMessage(message: unknown): message is OverlayAgentStateMessage {
|
||||
if (!message || typeof message !== "object") return false;
|
||||
const candidate = message as {
|
||||
type?: unknown;
|
||||
sessionId?: unknown;
|
||||
mode?: unknown;
|
||||
generation?: unknown;
|
||||
};
|
||||
return (
|
||||
candidate.type === OVERLAY_AGENT_STATE &&
|
||||
(typeof candidate.sessionId === "string" || candidate.sessionId === null) &&
|
||||
(candidate.mode === "control" ||
|
||||
candidate.mode === "interrupting" ||
|
||||
candidate.mode === "paused" ||
|
||||
candidate.mode === "hidden") &&
|
||||
typeof candidate.generation === "number"
|
||||
);
|
||||
}
|
||||
|
||||
export type OverlayMessage = OverlayWhoAmIRequest | OverlayReadyRequest | OverlayInterruptRequest;
|
||||
|
||||
@@ -274,6 +274,35 @@ describe("ToolDispatcher", () => {
|
||||
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
sent.length = 0;
|
||||
|
||||
deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).not.toHaveBeenCalled();
|
||||
|
||||
deliver({
|
||||
...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
|
||||
id: "r-control",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
});
|
||||
|
||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
parsePngDimensions,
|
||||
@@ -1412,7 +1413,7 @@ describe("buildVomScene", () => {
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
@@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => {
|
||||
expect(ctx.refStore.resolve("e2")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps snapshots static without conditional surface probing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleSnapshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "User tab" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Read" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const deps = makeDeps([root, button]);
|
||||
deps.tabsApi.get = vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
|
||||
);
|
||||
|
||||
const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.tab_id).toBe(9);
|
||||
expect(res.text).toContain('@e1 button "Read"');
|
||||
expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {});
|
||||
});
|
||||
|
||||
it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const deps = makeDeps([]);
|
||||
deps.tabsApi.get = vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
|
||||
);
|
||||
|
||||
const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "agent_window_scope" },
|
||||
});
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resets the RefStore on every fresh snapshot", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { lookupSession, parseBufferedReadBounds } from "../shared";
|
||||
import {
|
||||
cdpBlockedUrlReason,
|
||||
enforceCdpAccessibleTarget,
|
||||
lookupSession,
|
||||
parseBufferedReadBounds,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
} from "../shared";
|
||||
|
||||
function fakeAgentWindow() {
|
||||
return {
|
||||
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("CDP target URL guard", () => {
|
||||
it("allows ordinary pages and about:blank", () => {
|
||||
expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
|
||||
expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
|
||||
expect(cdpBlockedUrlReason("about:blank")).toBeNull();
|
||||
});
|
||||
|
||||
it("blocks browser and extension internal pages", () => {
|
||||
expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
|
||||
expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
|
||||
expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
|
||||
});
|
||||
|
||||
it("returns a structured permission error before page CDP access", () => {
|
||||
expect(
|
||||
enforceCdpAccessibleTarget(
|
||||
{
|
||||
tabId: 7,
|
||||
windowId: 100,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
},
|
||||
"snapshot",
|
||||
),
|
||||
).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "restricted_tab_url" },
|
||||
message: expect.stringContaining("snapshot cannot access tab 7"),
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back from a restricted default active tab to an accessible agent tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const tabsApi = {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
|
||||
if (query.active) {
|
||||
return [
|
||||
{
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
} as chrome.tabs.Tab,
|
||||
];
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
} as chrome.tabs.Tab,
|
||||
{
|
||||
id: 8,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: false,
|
||||
url: "https://example.test/",
|
||||
} as chrome.tabs.Tab,
|
||||
];
|
||||
}),
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
|
||||
).resolves.toMatchObject({
|
||||
tabId: 8,
|
||||
url: "https://example.test/",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not replace an explicitly requested restricted tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
async () =>
|
||||
({
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
}) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
|
||||
).resolves.toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "restricted_tab_url" },
|
||||
});
|
||||
expect(tabsApi.query).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
NavigateForwardParams,
|
||||
NavigateParams,
|
||||
NetworkParams,
|
||||
ObserveParams,
|
||||
PressParams,
|
||||
ProtocolFrame,
|
||||
RecordAwaitParams,
|
||||
@@ -42,6 +43,7 @@ import {
|
||||
type CdpRunner,
|
||||
chromeTabsCaptureApi,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
@@ -86,6 +88,8 @@ export interface DispatcherDeps {
|
||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||
*/
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
@@ -114,6 +118,7 @@ export class ToolDispatcher {
|
||||
private readonly sessions: SessionManager;
|
||||
private readonly cdp?: DispatcherCdpRunner;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
@@ -130,6 +135,7 @@ export class ToolDispatcher {
|
||||
this.sessions = deps.sessions;
|
||||
this.cdp = deps.cdp;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
}
|
||||
@@ -193,6 +199,8 @@ export class ToolDispatcher {
|
||||
let body: ResponseFrame;
|
||||
let startedSession: string | null = null;
|
||||
try {
|
||||
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
@@ -304,6 +312,12 @@ export class ToolDispatcher {
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.observe":
|
||||
return handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.get_html":
|
||||
return handleGetHtml(
|
||||
this.sessions,
|
||||
@@ -457,6 +471,33 @@ function isRpcError(v: unknown): v is RpcError {
|
||||
);
|
||||
}
|
||||
|
||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
switch (req.method) {
|
||||
case "tool.tab_create":
|
||||
case "tool.tab_close":
|
||||
case "tool.tab_select":
|
||||
case "tool.tab_borrow":
|
||||
case "tool.tab_return":
|
||||
case "tool.navigate":
|
||||
case "tool.navigate_back":
|
||||
case "tool.navigate_forward":
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.evaluate":
|
||||
case "tool.observe":
|
||||
case "tool.request_help":
|
||||
case "tool.record_start": {
|
||||
const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||
return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null;
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Read-only observation handlers — `tool.screenshot`, `tool.snapshot`,
|
||||
// and `tool.get_html` (design §7). Each handler resolves the target
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
@@ -15,6 +15,8 @@ import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
GetHtmlParams,
|
||||
GetHtmlResult,
|
||||
ObserveParams,
|
||||
ObserveResult,
|
||||
RpcError,
|
||||
ScreenshotParams,
|
||||
ScreenshotResult,
|
||||
@@ -26,11 +28,13 @@ import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
@@ -167,7 +171,13 @@ export async function handleScreenshot(
|
||||
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
"screenshot",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
||||
const withShotDialogs = <T extends object>(result: T) =>
|
||||
@@ -181,6 +191,7 @@ export async function handleScreenshot(
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(captured)) return captured;
|
||||
return withShotDialogs({
|
||||
@@ -664,6 +675,7 @@ function inputStateFor(
|
||||
value: string | undefined,
|
||||
): VomNode["inputState"] {
|
||||
if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined;
|
||||
if (capturedNode.formState) return capturedNode.formState;
|
||||
const formValue = capturedNode.formValue;
|
||||
if (formValue !== undefined) {
|
||||
if (formValue === "") return "empty";
|
||||
@@ -1099,7 +1111,13 @@ export async function handleGetHtml(
|
||||
const ctxOrErr = lookupSession(manager, params, "get_html");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
"get_html",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
@@ -1108,6 +1126,7 @@ export async function handleGetHtml(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
let html: string;
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
@@ -1178,28 +1197,44 @@ async function fallbackCapturedViewModel(
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId);
|
||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
toolName: "snapshot" | "observe",
|
||||
effect: ToolEffect,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "snapshot");
|
||||
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, toolName);
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
toolName,
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
||||
target.tabId,
|
||||
@@ -1207,7 +1242,7 @@ export async function handleSnapshot(
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
@@ -1232,3 +1267,19 @@ export async function handleSnapshot(
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
|
||||
}
|
||||
|
||||
export async function handleObserve(
|
||||
manager: SessionManager,
|
||||
params: ObserveParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
|
||||
}
|
||||
|
||||
@@ -369,7 +369,7 @@ async function rearmRecording(
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// content-script counter, and a single stop decrement left the control mask
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
@@ -561,9 +561,9 @@ export async function handleRecordStart(
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
// Register the recording *before* navigate so content-script syncAgentOverlay
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
// Register the recording *before* navigate so the destination content script's
|
||||
// overlay.ready flow can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing the control mask ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
|
||||
@@ -32,6 +32,13 @@ export const chromeTabsApi: ChromeTabsApi = {
|
||||
query: (q) => chrome.tabs.query(q),
|
||||
};
|
||||
|
||||
export interface ResolvedTargetTab {
|
||||
tabId: number;
|
||||
windowId: number;
|
||||
active: boolean;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export type { DialogCursor };
|
||||
|
||||
/**
|
||||
@@ -44,6 +51,7 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -146,7 +154,7 @@ export async function resolveTargetTab(
|
||||
ctx: SessionContext,
|
||||
tabId: number | undefined,
|
||||
api: ChromeTabsApi,
|
||||
): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> {
|
||||
): Promise<ResolvedTargetTab | RpcError> {
|
||||
if (tabId !== undefined) {
|
||||
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
|
||||
return {
|
||||
@@ -176,7 +184,7 @@ export async function resolveTargetTab(
|
||||
message: `tab ${tabId} not found in session scope`,
|
||||
};
|
||||
}
|
||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true };
|
||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url };
|
||||
}
|
||||
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
@@ -186,7 +194,12 @@ export async function resolveTargetTab(
|
||||
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
|
||||
};
|
||||
}
|
||||
return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true };
|
||||
return {
|
||||
tabId: first.id,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: first.active === true,
|
||||
url: first.url,
|
||||
};
|
||||
}
|
||||
|
||||
export function isRpcError(v: unknown): v is RpcError {
|
||||
@@ -202,6 +215,91 @@ export function isRpcError(v: unknown): v is RpcError {
|
||||
/** Re-export so M6/M7 tools keep a stable import path. */
|
||||
export { normaliseRef };
|
||||
|
||||
export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation";
|
||||
|
||||
const CDP_BLOCKED_PROTOCOLS = new Set([
|
||||
"chrome:",
|
||||
"chrome-extension:",
|
||||
"devtools:",
|
||||
"edge:",
|
||||
"brave:",
|
||||
"vivaldi:",
|
||||
"opera:",
|
||||
]);
|
||||
|
||||
export function cdpBlockedUrlReason(url: string | undefined): string | null {
|
||||
if (!url) return null;
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol;
|
||||
if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Page CDP tools cannot inspect browser/extension internal pages. Check this
|
||||
* at the tool boundary so Chrome's low-level "Cannot access..." errors do not
|
||||
* leak as ambiguous page-read failures.
|
||||
*/
|
||||
export function enforceCdpAccessibleTarget(
|
||||
target: ResolvedTargetTab,
|
||||
toolName: string,
|
||||
): RpcError | null {
|
||||
const reason = cdpBlockedUrlReason(target.url);
|
||||
if (!reason) return null;
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"restricted_tab_url",
|
||||
`${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`,
|
||||
);
|
||||
}
|
||||
|
||||
function resolvedTargetFromChromeTab(
|
||||
tab: chrome.tabs.Tab,
|
||||
fallbackWindowId: number,
|
||||
): ResolvedTargetTab | null {
|
||||
if (typeof tab.id !== "number") return null;
|
||||
return {
|
||||
tabId: tab.id,
|
||||
windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId,
|
||||
active: tab.active === true,
|
||||
url: tab.url,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a target for page CDP reads. Explicit tab_ids stay exact and are
|
||||
* rejected when browser security blocks the URL. For default targeting, skip a
|
||||
* restricted Agent Window active tab if another CDP-accessible tab exists in
|
||||
* the same Agent Window.
|
||||
*/
|
||||
export async function resolveCdpAccessibleTargetTab(
|
||||
manager: SessionManager,
|
||||
ctx: SessionContext,
|
||||
tabId: number | undefined,
|
||||
api: ChromeTabsApi,
|
||||
toolName: string,
|
||||
): Promise<ResolvedTargetTab | RpcError> {
|
||||
const target = await resolveTargetTab(manager, ctx, tabId, api);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
const restricted = enforceCdpAccessibleTarget(target, toolName);
|
||||
if (!restricted) return target;
|
||||
if (tabId !== undefined) return restricted;
|
||||
|
||||
const tabs = await api.query({ windowId: ctx.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId);
|
||||
if (!candidate) continue;
|
||||
if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate;
|
||||
}
|
||||
return restricted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
|
||||
* MUST refuse to touch a tab outside the session's Agent Window
|
||||
@@ -224,3 +322,17 @@ export function enforceAgentWindow(
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified target-scope policy by tool effect. Passive reads may inspect user
|
||||
* tabs; any tool that dispatches page input must stay inside the Agent Window.
|
||||
*/
|
||||
export function enforceToolTargetScope(
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number; windowId: number },
|
||||
effect: ToolEffect,
|
||||
toolName: string,
|
||||
): RpcError | null {
|
||||
if (effect === "passive_read") return null;
|
||||
return enforceAgentWindow(ctx, target, toolName);
|
||||
}
|
||||
|
||||
@@ -59,6 +59,23 @@ function makeCdp(snapshot: unknown) {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
controls: [
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
childFrames: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
@@ -80,6 +97,89 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
it("batch-enriches form controls without per-node object resolution", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
const input = result.nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let runtimeCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
runtimeCalls += 1;
|
||||
return runtimeCalls === 1
|
||||
? {
|
||||
result: {
|
||||
value: [
|
||||
{
|
||||
triggerSel: ".menu",
|
||||
affectedSub: " .items",
|
||||
label: "Products",
|
||||
x: 50,
|
||||
y: 20,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
: { result: { value: ["Shoes", "Bags"] } };
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: -10, y: -10 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
@@ -349,8 +449,11 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
@@ -557,8 +660,187 @@ describe("captureViewModel", () => {
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
|
||||
const input = subNodes.find((n) => n.backendNodeId === 101);
|
||||
expect(input?.tag).toBe("input");
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
});
|
||||
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [100, 101],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[20, 20, 120, 60]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 20, 300, 200],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [20, 21, 23],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [2] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[5, 6, 100, 80],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [30, 31],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[1, 2, 40, 20]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
|
||||
@@ -16,9 +16,14 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
@@ -31,6 +36,7 @@ export interface CapturedNode {
|
||||
*/
|
||||
cursor?: string;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
@@ -53,6 +59,10 @@ export interface CapturedViewModel {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
@@ -102,48 +112,112 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) {
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface CapturedFrameFormState {
|
||||
controls: CapturedFormState[];
|
||||
childFrames: CapturedFrameFormState[];
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
const controls = [];
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
controls.push(controlState(el));
|
||||
remaining -= 1;
|
||||
}
|
||||
const childFrames = [];
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
||||
}
|
||||
return { controls, childFrames };
|
||||
};
|
||||
return collect(document);
|
||||
})()`;
|
||||
}
|
||||
|
||||
function flattenFrameFormStates(
|
||||
frame: CapturedFrameFormState | undefined,
|
||||
out: CapturedFormState[][] = [],
|
||||
): CapturedFormState[][] {
|
||||
if (!frame) return out;
|
||||
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
||||
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
||||
flattenFrameFormStates(child, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
||||
let index = 0;
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(
|
||||
tabId,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: node.backendNodeId,
|
||||
},
|
||||
);
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) continue;
|
||||
const result = await cdp.send<{
|
||||
result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
returnByValue: true,
|
||||
functionDeclaration: `function(){
|
||||
return {
|
||||
value: typeof this.value === "string" ? this.value : "",
|
||||
defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "",
|
||||
placeholder: typeof this.placeholder === "string" ? this.placeholder : ""
|
||||
};
|
||||
}`,
|
||||
});
|
||||
const value = result.result?.value;
|
||||
if (!value) continue;
|
||||
node.formValue = value.value ?? "";
|
||||
node.formDefaultValue = value.defaultValue ?? "";
|
||||
node.formPlaceholder = value.placeholder ?? "";
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the node.
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
const state = states[index];
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
@@ -169,6 +243,14 @@ interface ParseDocumentResult {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
@@ -183,6 +265,49 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
@@ -235,14 +360,24 @@ function hoverCssScanExpression(): string {
|
||||
|
||||
const candidates = [];
|
||||
const seenLabels = new Set();
|
||||
const isUnsafeTrigger = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return true;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||
if (el.isContentEditable) return true;
|
||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
||||
return false;
|
||||
};
|
||||
for (const pair of pairs) {
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof Element)) continue;
|
||||
if (isUnsafeTrigger(el)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
if (!label || seenLabels.has(label)) continue;
|
||||
seenLabels.add(label);
|
||||
@@ -304,6 +439,20 @@ async function wait(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
await cdp
|
||||
.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: -10,
|
||||
y: -10,
|
||||
})
|
||||
.catch(() =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
||||
// best effort
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
@@ -328,7 +477,6 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
||||
expression: hoverCollectExpression(candidate),
|
||||
returnByValue: true,
|
||||
});
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
|
||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
@@ -341,6 +489,8 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
@@ -391,15 +541,13 @@ export async function collectOverlayExcludedBackendIds(
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param scrollX - horizontal scroll offset to subtract (CSS px)
|
||||
* @param scrollY - vertical scroll offset to subtract (CSS px)
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
scrollX: number,
|
||||
scrollY: number,
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
@@ -477,6 +625,7 @@ function parseDocumentNodes(
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
@@ -485,12 +634,13 @@ function parseDocumentNodes(
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
rect = {
|
||||
x: b[0] / dpr - scrollX,
|
||||
y: b[1] / dpr - scrollY,
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -508,9 +658,11 @@ function parseDocumentNodes(
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
@@ -521,7 +673,81 @@ function parseDocumentNodes(
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function parseChildFrameDocuments(
|
||||
documents: SnapshotDocument[],
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
const childDoc = documents[childDocIndex];
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
|
||||
const childContext: FrameContext = {
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
documents,
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
@@ -552,42 +778,26 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
|
||||
};
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
const topContext: FrameContext = {
|
||||
ownerFrameBackendNodeId: null,
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
// Build a map from node-array-index → sub-document-index using the sparse
|
||||
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
if (documents.length > 1) {
|
||||
const cdi = doc0.nodes?.contentDocumentIndex;
|
||||
if (cdi?.index && cdi.value) {
|
||||
for (let k = 0; k < cdi.index.length; k++) {
|
||||
const nodeArrayIdx = cdi.index[k];
|
||||
const docIdx = cdi.value[k];
|
||||
const subDoc = documents[docIdx];
|
||||
if (!subDoc) continue;
|
||||
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
|
||||
if (iframeBid === undefined) continue;
|
||||
// Sub-documents have their own scroll offsets; bounds are relative
|
||||
// to the iframe's own coordinate space (not the main viewport).
|
||||
const subScrollX = subDoc.scrollOffsetX ?? 0;
|
||||
const subScrollY = subDoc.scrollOffsetY ?? 0;
|
||||
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
|
||||
iframeNodes.set(iframeBid, subParsed.nodes);
|
||||
for (const id of subParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlState(cdp, tabId, nodes);
|
||||
for (const subNodes of iframeNodes.values()) {
|
||||
await enrichFormControlState(cdp, tabId, subNodes);
|
||||
}
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
|
||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ export type RpcErrorReason =
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict";
|
||||
|
||||
export interface RpcErrorData {
|
||||
@@ -313,6 +314,9 @@ export interface SnapshotResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export type ObserveParams = SnapshotParams;
|
||||
export type ObserveResult = SnapshotResult;
|
||||
|
||||
export interface GetHtmlParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
|
||||
Reference in New Issue
Block a user