figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): update vom algorithm, fix overlay problem with background state machine

Browse Source
This commit is contained in:
Ljy-0827
2026-07-27 13:46:33 +08:00
parent e14ae9cea0
commit 4fb2e331a6
36 changed files with 2276 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/entrypoints/background.ts
+102 -6
View File
@@ -9,12 +9,16 @@ import {
} from "@/lib/instance-id";
import { startKeepalive } from "@/lib/keepalive";
import {
OVERLAY_AGENT_STATE,
OVERLAY_AUTOMATION_BYPASS,
OVERLAY_MSG_INTERRUPT,
OVERLAY_MSG_READY,
OVERLAY_MSG_WHO_AM_I,
type OverlayAgentStateMessage,
type OverlayInterruptRequest,
type OverlayInterruptResponse,
type OverlayMessage,
type OverlayMode,
} from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
@@ -44,6 +48,91 @@ export default defineBackground(() => {
const sessions = new SessionManager();
const cdp = new ChromiumCdp();
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
let overlayGeneration = 0;
const controlModes = new Map<string, OverlayMode>();
function setControlMode(sessionId: string, mode: OverlayMode): void {
if (controlModes.get(sessionId) === mode) return;
controlModes.set(sessionId, mode);
overlayGeneration += 1;
const ctx = sessions.get(sessionId);
if (ctx) void pushOverlayStateForWindow(ctx.agentWindowId);
}
function overlayStateForWindow(windowId?: number): OverlayAgentStateMessage {
const ctx = typeof windowId === "number" ? sessions.findByWindowId(windowId) : null;
if (!ctx) {
return {
type: OVERLAY_AGENT_STATE,
sessionId: null,
mode: "hidden",
generation: overlayGeneration,
};
}
return {
type: OVERLAY_AGENT_STATE,
sessionId: ctx.sessionId,
mode: controlModes.get(ctx.sessionId) ?? "control",
generation: overlayGeneration,
};
}
async function pushOverlayStateToTab(
tabId: number,
state: OverlayAgentStateMessage,
): Promise<void> {
try {
await chrome.tabs.sendMessage(tabId, state);
} catch {
// Restricted pages and not-yet-loaded content scripts cannot receive messages.
}
}
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
const state = overlayStateForWindow(windowId);
const tabs = await chrome.tabs.query({ windowId });
await Promise.all(
tabs.map((tab) =>
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
),
);
}
function pushAllAgentOverlayStates(): void {
const windowIds = new Set(sessions.list().map((ctx) => ctx.agentWindowId));
for (const windowId of windowIds) {
void pushOverlayStateForWindow(windowId);
}
}
function onOverlaySessionStateChanged(): void {
void sessionsLive.syncFromManager();
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
for (const sessionId of controlModes.keys()) {
if (!liveSessionIds.has(sessionId)) controlModes.delete(sessionId);
}
overlayGeneration += 1;
pushAllAgentOverlayStates();
}
function onBrowserControlResumed(sessionId: string): void {
const ctx = sessions.get(sessionId);
if (!ctx) return;
setControlMode(sessionId, "control");
}
function pushOverlayStateForAgentWindow(windowId: number): void {
if (!sessions.findByWindowId(windowId)) return;
void pushOverlayStateForWindow(windowId);
}
chrome.tabs.onActivated.addListener((activeInfo) => {
pushOverlayStateForAgentWindow(activeInfo.windowId);
});
chrome.tabs.onUpdated.addListener((_tabId, changeInfo, tab) => {
if (changeInfo.status !== "complete") return;
if (typeof tab.windowId !== "number") return;
pushOverlayStateForAgentWindow(tab.windowId);
});
// Re-sync the storage.session flag on SW startup so a previous SW's
// stale `true` does not keep waking us on every page load until the
// first mutation (review M4/M5 round 3 m-R3-1).
@@ -52,9 +141,8 @@ export default defineBackground(() => {
transport,
sessions,
cdp,
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
approveBorrow: (ctx) =>
requestBorrowConfirmation(ctx.tabId, {
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
@@ -124,9 +212,7 @@ export default defineBackground(() => {
manager: sessions,
transport,
cdp,
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
onSessionsChanged: onOverlaySessionStateChanged,
});
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
@@ -197,9 +283,19 @@ export default defineBackground(() => {
return false;
}
if (msg.kind === OVERLAY_MSG_READY) {
sendResponse(overlayStateForWindow(sender.tab?.windowId));
return false;
}
if (msg.kind === OVERLAY_MSG_INTERRUPT) {
const req = msg as OverlayInterruptRequest;
const ctx = sessions.get(req.sessionId);
if (ctx) setControlMode(req.sessionId, "interrupting");
void handleOverlayInterrupt(transport, req.sessionId).then((reply) => {
if (reply.ok && sessions.get(req.sessionId)) {
setControlMode(req.sessionId, "paused");
}
sendResponse(reply);
});
return true; // keep channel open
apps/extension/src/entrypoints/content.ts
+303 -79
View File
@@ -3,7 +3,6 @@ import { I18nextProvider } from "@browser-skill/i18n/react";
import React from "react";
import ReactDOM from "react-dom/client";
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
@@ -27,13 +26,12 @@ import {
} from "@/lib/help-bridge";
import {
isOverlayAgentOverlayResetMessage,
isOverlayAgentStateMessage,
OVERLAY_AUTOMATION_BYPASS,
OVERLAY_HOST_MARKER_ATTR,
OVERLAY_HOST_NAME,
OVERLAY_MSG_WHO_AM_I,
OVERLAY_MSG_READY,
type OverlayAgentOverlayResetMessage,
type OverlayAgentStateMessage,
type OverlayAutomationBypassMessage,
type OverlayWhoAmIResponse,
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
@@ -42,12 +40,12 @@ import {
type RecordStartAck,
type RecordStopAck,
} from "@/lib/record-bridge";
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
import type {
BorrowCancelMessage,
BorrowRequestMessage,
BorrowResponseMessage,
} from "@/tools/borrow-confirmation";
import logoUrl from "../../assets/logo.png";
// Run at document_end so the overlay does not block first paint. Only attach
// in the top-level frame so iframes do not double-render overlays.
@@ -65,34 +63,264 @@ export default defineContentScript({
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
let overlayContainer: HTMLElement | null = null;
let controlRoot: HTMLElement | null = null;
let activeAgentState: OverlayAgentStateMessage | null = null;
let hostLossReported = false;
let remountInProgress = false;
const ui = await createShadowRootUi(ctx, {
name: OVERLAY_HOST_NAME,
name: "browser-skill-overlay",
position: "inline",
anchor: "html",
css: overlayCss,
onMount(container, _shadow, shadowHost) {
shadowHost.setAttribute("aria-hidden", "true");
shadowHost.setAttribute(OVERLAY_HOST_MARKER_ATTR, "");
shadowHost.setAttribute("data-bsk-overlay", "");
overlayHost = shadowHost;
overlayContainer = container;
hostLossReported = false;
const app = document.createElement("div");
app.className = "bsk-overlay-root";
container.append(app);
reactRoot = ReactDOM.createRoot(app);
renderOverlay();
renderAll();
void requestOverlayState();
return reactRoot;
},
onRemove(root) {
overlayHost = null;
overlayContainer = null;
controlRoot = null;
root?.unmount();
reactRoot = null;
},
});
function renderOverlay() {
function removeControlOverlay(): void {
controlRoot?.remove();
controlRoot = null;
}
function setControlSurfaceActive(active: boolean): void {
const host = overlayHost;
const container = overlayContainer;
if (!host || !container) return;
if (active) {
Object.assign(host.style, {
position: "fixed",
inset: "0",
zIndex: "2147483647",
pointerEvents: "auto",
});
Object.assign(container.style, {
position: "fixed",
inset: "0",
pointerEvents: "none",
});
return;
}
host.style.removeProperty("position");
host.style.removeProperty("inset");
host.style.removeProperty("z-index");
host.style.removeProperty("pointer-events");
container.style.removeProperty("position");
container.style.removeProperty("inset");
container.style.removeProperty("pointer-events");
}
function ensureControlOverlayRoot(): HTMLElement | null {
if (!overlayContainer) return null;
if (!controlRoot) {
controlRoot = document.createElement("div");
controlRoot.className = "bsk-control-overlay";
controlRoot.style.opacity = "0";
controlRoot.style.transition = "opacity 300ms ease-out";
overlayContainer.append(controlRoot);
window.requestAnimationFrame(() => {
if (controlRoot) controlRoot.style.opacity = "1";
});
}
return controlRoot;
}
function createStopCircleIcon(): SVGSVGElement {
const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
svg.setAttribute("width", "18");
svg.setAttribute("height", "18");
svg.setAttribute("viewBox", "0 0 24 24");
svg.setAttribute("fill", "currentColor");
svg.setAttribute("aria-hidden", "true");
svg.style.color = "#fff";
svg.style.flexShrink = "0";
const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
path.setAttribute(
"d",
"M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10Zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16ZM9 9h6v6H9V9Z",
);
svg.append(path);
return svg;
}
function renderControlOverlay(): void {
const overlayState = overlays.snapshot();
if (!shouldShowAgentControlOverlay(overlayState)) {
removeControlOverlay();
setControlSurfaceActive(false);
return;
}
const root = ensureControlOverlayRoot();
if (!root) return;
const pointerEvents = overlayState.automationBypassCount > 0 ? "none" : "auto";
setControlSurfaceActive(true);
root.innerHTML = "";
Object.assign(root.style, {
position: "fixed",
inset: "0",
pointerEvents: "none",
});
const style = document.createElement("style");
style.textContent = `
@keyframes bsk-breathe {
0%, 100% { box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25); }
50% { box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5); }
}
`;
const border = document.createElement("div");
border.setAttribute("data-slot", "control-overlay");
Object.assign(border.style, {
position: "fixed",
inset: "0",
zIndex: "2147483646",
pointerEvents: "none",
animation: "bsk-breathe 3s ease-in-out infinite",
});
const blocker = document.createElement("div");
blocker.setAttribute("data-slot", "control-overlay-blocker");
Object.assign(blocker.style, {
position: "fixed",
inset: "0",
zIndex: "2147483646",
pointerEvents,
background: "transparent",
});
blocker.addEventListener("pointerdown", (event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
});
blocker.addEventListener("click", (event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
});
blocker.addEventListener(
"wheel",
(event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
},
{ passive: false },
);
blocker.addEventListener(
"touchmove",
(event) => {
if (overlayState.automationBypassCount > 0) return;
event.preventDefault();
event.stopPropagation();
},
{ passive: false },
);
const pill = document.createElement("div");
pill.setAttribute("data-slot", "control-overlay-pill");
Object.assign(pill.style, {
position: "fixed",
bottom: "32px",
left: "50%",
transform: "translateX(-50%)",
zIndex: "2147483647",
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: "12px",
borderRadius: "999px",
backgroundColor: "#fff",
padding: "10px 10px 10px 20px",
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
fontFamily:
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
});
const logo = document.createElement("img");
logo.src = logoUrl;
logo.alt = "browser-skill";
Object.assign(logo.style, {
width: "24px",
height: "24px",
borderRadius: "4px",
flexShrink: "0",
});
const label = document.createElement("span");
label.textContent = i18n.t("controlOverlay.status", { ns: "extension" });
Object.assign(label.style, {
fontSize: "16px",
fontWeight: "500",
color: "#333",
whiteSpace: "nowrap",
userSelect: "none",
});
const button = document.createElement("button");
button.type = "button";
button.setAttribute("data-slot", "control-overlay-stop-all");
button.disabled = overlayState.interrupting;
Object.assign(button.style, {
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: "6px",
border: "none",
borderRadius: "999px",
padding: "8px 20px 8px 16px",
fontSize: "15px",
fontWeight: "600",
color: "#fff",
backgroundColor: overlayState.interrupting ? "#9ca3af" : "#f97316",
cursor: overlayState.interrupting ? "default" : "pointer",
opacity: overlayState.interrupting ? "0.7" : "1",
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
whiteSpace: "nowrap",
lineHeight: "1",
});
button.append(
createStopCircleIcon(),
document.createTextNode(
overlayState.interrupting
? i18n.t("controlOverlay.interrupting", { ns: "extension" })
: i18n.t("controlOverlay.interrupt", { ns: "extension" }),
),
);
button.addEventListener("click", (event) => {
event.preventDefault();
event.stopPropagation();
handleInterrupt();
});
pill.append(logo, label, button);
root.append(style, border, blocker, pill);
}
function renderReactOverlays(): void {
const overlayState = overlays.snapshot();
reactRoot?.render(
React.createElement(
@@ -104,12 +332,6 @@ export default defineContentScript({
React.createElement(BorrowConfirmationOverlay, {
requests: overlayState.borrowRequests,
}),
React.createElement(ControlOverlay, {
visible: shouldShowAgentControlOverlay(overlayState),
interrupting: overlayState.interrupting,
automationBypass: overlayState.automationBypassCount > 0,
onInterrupt: handleInterrupt,
}),
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
),
@@ -117,6 +339,23 @@ export default defineContentScript({
);
}
function renderAll(): void {
renderReactOverlays();
renderControlOverlay();
}
function clearCurrentAgentSession(): void {
const sessionId = overlays.snapshot().activeSessionId;
if (!sessionId) return;
resetAgentOverlayState(sessionId);
}
function applyOverlayState(state: OverlayAgentStateMessage): void {
activeAgentState = state;
overlays.applyAgentControlMode(state.sessionId, state.mode);
renderAll();
}
function resetAgentOverlayState(sessionId: string) {
const previousHelp = overlays.resetAgentOverlays(sessionId);
if (previousHelp) {
@@ -125,7 +364,7 @@ export default defineContentScript({
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderOverlay();
renderAll();
}
function handleInterrupt() {
@@ -137,12 +376,8 @@ export default defineContentScript({
return;
}
overlays.setInterrupting(true);
renderOverlay();
renderAll();
void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => {
// Always retract the mask after the round trip resolves
// (success, failure, or timeout). Cancellation is fire-and-
// forget on the daemon side; the user must not be stuck
// behind a transient issue. The Agent Window stays open.
resetAgentOverlayState(sessionId);
if (!reply.ok) {
console.warn("[bsk overlay] interrupt did not get a clean ack from daemon");
@@ -157,6 +392,7 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
@@ -183,11 +419,11 @@ export default defineContentScript({
});
},
});
renderOverlay();
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderOverlay();
renderAll();
},
},
sendResponse as unknown as
@@ -205,7 +441,12 @@ export default defineContentScript({
) {
const bypassMsg = message as OverlayAutomationBypassMessage;
overlays.setAutomationBypass(bypassMsg.enabled);
renderOverlay();
renderAll();
return false;
}
if (isOverlayAgentStateMessage(message)) {
applyOverlayState(message);
return false;
}
@@ -216,7 +457,7 @@ export default defineContentScript({
if (message.type === "borrow-cancel") {
overlays.removeBorrowRequest(message.requestId);
renderOverlay();
renderAll();
return false;
}
@@ -224,7 +465,7 @@ export default defineContentScript({
const state = overlays.snapshot();
if (state.activeHelp && state.activeHelp.id === message.requestId) {
overlays.clearAgentHelpRequest(message.requestId);
renderOverlay();
renderAll();
}
return false;
}
@@ -244,7 +485,7 @@ export default defineContentScript({
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
renderOverlay();
renderAll();
sendResponse({ type: HELP_ACK, ok: true });
return false;
}
@@ -256,7 +497,7 @@ export default defineContentScript({
responded = true;
sendResponse({ type: "borrow-response", allowed });
overlays.removeBorrowRequest(message.requestId);
renderOverlay();
renderAll();
};
overlays.addBorrowRequest({
@@ -267,7 +508,7 @@ export default defineContentScript({
onAllow: () => respond(true),
onDeny: () => respond(false),
});
renderOverlay();
renderAll();
return true;
}
@@ -286,7 +527,7 @@ export default defineContentScript({
...(note ? { note } : {}),
};
overlays.clearAgentHelpRequest(requestId);
renderOverlay();
renderAll();
await chrome.runtime.sendMessage(msg).catch((err) => {
console.debug("[bsk overlay] help finish failed", err);
});
@@ -305,7 +546,7 @@ export default defineContentScript({
});
}
async function queryActiveHelpWithRetry(): Promise<boolean> {
async function queryActiveHelpWithRetry(): Promise<void> {
for (let attempt = 0; attempt < 6; attempt += 1) {
try {
const helpQuery = (await chrome.runtime.sendMessage({
@@ -313,38 +554,21 @@ export default defineContentScript({
})) as HelpQueryResponse | undefined;
if (helpQuery?.active && helpQuery.request) {
mountHelpRequest(helpQuery.request);
renderOverlay();
return true;
renderAll();
return;
}
} catch (err) {
console.debug("[bsk overlay] help query failed", err);
}
await new Promise((resolve) => window.setTimeout(resolve, 150));
}
return false;
}
async function syncAgentOverlay(): Promise<void> {
if (!(await anySessionLive())) return;
async function queryActiveRecord(): Promise<void> {
try {
const helpActive = await queryActiveHelpWithRetry();
const reply = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_WHO_AM_I,
})) as OverlayWhoAmIResponse | undefined;
if (!reply?.sessionId) return;
// Query / re-arm recording *before* activating the control overlay so
// record start (navigate → content load) does not flash
// 「Agent 正在控制」before RecordOverlay mounts.
let recordQuery: { active?: boolean; requestId?: string } | undefined;
try {
recordQuery = (await chrome.runtime.sendMessage({
type: RECORD_QUERY,
})) as { active?: boolean; requestId?: string } | undefined;
} catch (err) {
console.debug("[bsk overlay] record query failed", err);
}
const recordQuery = (await chrome.runtime.sendMessage({
type: RECORD_QUERY,
})) as { active?: boolean; requestId?: string } | undefined;
if (
recordQuery?.active &&
typeof recordQuery.requestId === "string" &&
@@ -360,26 +584,38 @@ export default defineContentScript({
});
},
});
renderAll();
}
if (!helpActive && overlays.snapshot().activeHelp === null) {
void queryActiveHelpWithRetry();
}
overlays.activateAgentSession(reply.sessionId);
renderOverlay();
} catch (err) {
console.debug("[bsk overlay] syncAgentOverlay failed", err);
console.debug("[bsk overlay] record query failed", err);
}
}
async function refreshAuxiliaryOverlayState(): Promise<void> {
await Promise.all([queryActiveHelpWithRetry(), queryActiveRecord()]);
}
async function requestOverlayState(): Promise<void> {
try {
const state = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_READY,
})) as OverlayAgentStateMessage | undefined;
if (state && isOverlayAgentStateMessage(state)) {
applyOverlayState(state);
}
void refreshAuxiliaryOverlayState();
} catch (err) {
console.debug("[bsk overlay] overlay.ready failed", err);
}
}
const onPageShow = (event: PageTransitionEvent) => {
if (event.persisted) void syncAgentOverlay();
if (event.persisted) void requestOverlayState();
};
ui.mount();
chrome.runtime.onMessage.addListener(onMessage);
void syncAgentOverlay();
void requestOverlayState();
window.addEventListener("pageshow", onPageShow);
@@ -391,7 +627,8 @@ export default defineContentScript({
remountInProgress = true;
try {
ui.mount();
void syncAgentOverlay();
if (activeAgentState) applyOverlayState(activeAgentState);
void requestOverlayState();
} finally {
remountInProgress = false;
}
@@ -414,16 +651,3 @@ export default defineContentScript({
});
},
});
async function anySessionLive(): Promise<boolean> {
if (!chrome.storage?.session?.get) return true;
try {
const result = (await chrome.storage.session.get({
[SESSIONS_LIVE_FLAG_KEY]: false,
})) as Record<string, unknown> | undefined;
return Boolean(result?.[SESSIONS_LIVE_FLAG_KEY]);
} catch (err) {
console.debug("[bsk overlay] sessions-live flag read failed", err);
return true;
}
}