fix(runtime): fix early control return and update lifecycle

This commit is contained in:
Ljy-0827
2026-07-24 14:07:33 +08:00
parent c8963e3a68
commit 4e44fb2823
17 changed files with 1002 additions and 262 deletions
@@ -342,6 +342,8 @@ export function HelpRequestOverlay({ request }: Props) {
z-index: 2147483647;
display: flex;
flex-direction: column;
justify-content: flex-start;
min-height: 0;
gap: 10px;
width: var(--bsk-help-banner-width);
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
@@ -359,16 +361,18 @@ export function HelpRequestOverlay({ request }: Props) {
.bsk-help-banner[data-collapsed="true"] {
gap: 0;
width: var(--bsk-help-banner-width);
padding: 10px 12px;
padding: 12px;
}
.bsk-help-drag-strip {
flex-shrink: 0;
position: absolute;
top: 6px;
left: 0;
right: 0;
display: flex;
align-items: center;
justify-content: center;
height: 12px;
margin: -8px 0 -10px;
cursor: grab;
user-select: none;
touch-action: none;
@@ -395,10 +399,11 @@ export function HelpRequestOverlay({ request }: Props) {
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
margin: -6px 0 -8px;
top: 4px;
}
.bsk-help-header {
flex-shrink: 0;
display: flex;
align-items: center;
gap: 10px;
@@ -420,25 +425,17 @@ export function HelpRequestOverlay({ request }: Props) {
.bsk-help-header-actions {
flex-shrink: 0;
display: flex;
display: none;
align-items: center;
gap: 10px;
margin-left: auto;
overflow: hidden;
max-width: 0;
opacity: 0;
transform: translateX(8px);
pointer-events: none;
transition:
max-width var(--bsk-help-duration) var(--bsk-help-ease),
opacity 220ms var(--bsk-help-ease),
transform var(--bsk-help-duration) var(--bsk-help-ease);
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
display: flex;
max-width: 280px;
opacity: 1;
transform: translateX(0);
pointer-events: auto;
}
@@ -471,15 +468,13 @@ export function HelpRequestOverlay({ request }: Props) {
}
.bsk-help-body {
display: grid;
grid-template-rows: 1fr;
display: block;
flex-shrink: 0;
min-width: 0;
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
position: absolute;
grid-template-rows: 0fr;
width: 0;
height: 0;
overflow: hidden;
@@ -528,6 +523,7 @@ export function HelpRequestOverlay({ request }: Props) {
.bsk-help-footer-actions {
display: flex;
justify-content: flex-end;
flex-wrap: wrap;
gap: 10px;
}
@@ -539,7 +535,8 @@ export function HelpRequestOverlay({ request }: Props) {
font-size: 13px;
font-weight: 500;
color: #4b5563;
flex-shrink: 0;
min-width: 0;
white-space: normal;
}
.bsk-help-btn-continue {
@@ -550,16 +547,19 @@ export function HelpRequestOverlay({ request }: Props) {
font-size: 13px;
font-weight: 600;
color: #fff;
flex-shrink: 0;
min-width: 0;
white-space: normal;
}
.bsk-help-header-actions .bsk-help-btn-cancel,
.bsk-help-header-actions .bsk-help-btn-continue {
padding: 6px 12px;
white-space: nowrap;
}
.bsk-help-header-actions .bsk-help-btn-continue {
padding: 6px 14px;
max-width: 180px;
}
.bsk-help-footer-actions .bsk-help-btn-cancel {
@@ -204,6 +204,15 @@ describe("HelpRequestOverlay", () => {
expect(styles).toContain("width: 0");
});
it("uses natural body height in the expanded layout", () => {
const { container } = renderOverlay(baseRequest());
const styles = overlayStyles(container);
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
});
it("keeps the same banner width when collapsed", () => {
const { container } = renderOverlay(baseRequest());
const styles = overlayStyles(container);
+71 -28
View File
@@ -14,10 +14,14 @@ import {
type RecordCaptureController,
} from "@/content/record-capture";
import {
HELP_RESPONSE,
HELP_ACK,
HELP_FINISH,
HELP_QUERY,
type HelpAckMessage,
type HelpCancelMessage,
type HelpFinishMessage,
type HelpQueryResponse,
type HelpRequestMessage,
type HelpResponseMessage,
isHelpCancelMessage,
isHelpRequestMessage,
} from "@/lib/help-bridge";
@@ -55,8 +59,6 @@ export default defineContentScript({
if (window.top !== window) return;
const overlays = new OverlayController();
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
null;
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
@@ -116,8 +118,7 @@ export default defineContentScript({
function resetAgentOverlayState(sessionId: string) {
const previousHelp = overlays.resetAgentOverlays(sessionId);
if (previousHelp) {
activeHelpRespond?.("cancelled");
activeHelpRespond = null;
void sendHelpFinish(previousHelp.id, "cancelled");
}
recordCapture?.dispose();
recordCapture = null;
@@ -156,7 +157,7 @@ export default defineContentScript({
| OverlayAgentOverlayResetMessage
| OverlayAutomationBypassMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
) => {
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
@@ -220,41 +221,29 @@ export default defineContentScript({
if (isHelpCancelMessage(message)) {
const state = overlays.snapshot();
if (state.activeHelp && state.activeHelp.id === message.requestId) {
activeHelpRespond?.("cancelled");
overlays.clearAgentHelpRequest(message.requestId);
renderOverlay();
}
return false;
}
if (isHelpRequestMessage(message)) {
const helpMsg = message as HelpRequestMessage;
let responded = false;
const respond = (outcome: "continued" | "cancelled", note?: string) => {
if (responded) return;
responded = true;
const reply: HelpResponseMessage = {
type: HELP_RESPONSE,
outcome,
...(note ? { note } : {}),
};
sendResponse(reply);
activeHelpRespond = null;
overlays.clearAgentHelpRequest(helpMsg.requestId);
renderOverlay();
};
const previousHelp = overlays.setAgentHelpRequest({
id: helpMsg.requestId,
prompt: helpMsg.prompt,
...(helpMsg.title ? { title: helpMsg.title } : {}),
selectors: helpMsg.selectors,
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
onCancel: () => respond("cancelled"),
onContinue: (note: string) =>
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
});
if (previousHelp) {
activeHelpRespond?.("cancelled");
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
activeHelpRespond = respond;
renderOverlay();
return true; // async sendResponse
sendResponse({ type: HELP_ACK, ok: true });
return false;
}
if (message.type === "borrow-request") {
@@ -282,9 +271,59 @@ export default defineContentScript({
return false;
};
async function sendHelpFinish(
requestId: string,
outcome: "continued" | "cancelled",
note?: string,
): Promise<void> {
const msg: HelpFinishMessage = {
type: HELP_FINISH,
requestId,
outcome,
...(note ? { note } : {}),
};
overlays.clearAgentHelpRequest(requestId);
renderOverlay();
await chrome.runtime.sendMessage(msg).catch((err) => {
console.debug("[bsk overlay] help finish failed", err);
});
}
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
overlays.setAgentHelpRequest({
id: helpMsg.requestId,
prompt: helpMsg.prompt,
...(helpMsg.title ? { title: helpMsg.title } : {}),
selectors: helpMsg.selectors,
onContinue: (note: string) =>
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
});
}
async function queryActiveHelpWithRetry(): Promise<boolean> {
for (let attempt = 0; attempt < 6; attempt += 1) {
try {
const helpQuery = (await chrome.runtime.sendMessage({
type: HELP_QUERY,
})) as HelpQueryResponse | undefined;
if (helpQuery?.active && helpQuery.request) {
mountHelpRequest(helpQuery.request);
renderOverlay();
return true;
}
} catch (err) {
console.debug("[bsk overlay] help query failed", err);
}
await new Promise((resolve) => window.setTimeout(resolve, 150));
}
return false;
}
async function syncAgentOverlay(): Promise<void> {
if (!(await anySessionLive())) return;
try {
const helpActive = await queryActiveHelpWithRetry();
const reply = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_WHO_AM_I,
})) as OverlayWhoAmIResponse | undefined;
@@ -319,6 +358,10 @@ export default defineContentScript({
});
}
if (!helpActive && overlays.snapshot().activeHelp === null) {
void queryActiveHelpWithRetry();
}
overlays.activateAgentSession(reply.sessionId);
renderOverlay();
} catch (err) {
@@ -1,10 +1,17 @@
import { describe, expect, it } from "vitest";
import {
HELP_ACK,
HELP_CANCEL,
HELP_FINISH,
HELP_QUERY,
HELP_REQUEST,
HELP_RESPONSE,
isHelpAckMessage,
isHelpCancelMessage,
isHelpFinishMessage,
isHelpQueryMessage,
isHelpRequestMessage,
isHelpResponseMessage,
} from "../help-bridge";
describe("help-bridge", () => {
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
expect(HELP_REQUEST).toBe("bsk-help-request");
expect(HELP_RESPONSE).toBe("bsk-help-response");
expect(HELP_CANCEL).toBe("bsk-help-cancel");
expect(HELP_ACK).toBe("bsk-help-ack");
expect(HELP_QUERY).toBe("bsk-help-query");
expect(HELP_FINISH).toBe("bsk-help-finish");
});
it("type-guards a help-request message", () => {
@@ -55,4 +65,18 @@ describe("help-bridge", () => {
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
expect(isHelpCancelMessage(null)).toBe(false);
});
it("type-guards lifecycle messages", () => {
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
true,
);
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
true,
);
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
false,
);
});
});
+58 -1
View File
@@ -6,12 +6,15 @@
* Mirrors the borrow-confirmation message style (see
* `@/tools/borrow-confirmation`). The background asks a specific tab to
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
* the content script replies once the user clicks Continue / Cancel.
* the content script acks display, then later reports Done / Cancel.
*/
export const HELP_REQUEST = "bsk-help-request";
export const HELP_RESPONSE = "bsk-help-response";
export const HELP_CANCEL = "bsk-help-cancel";
export const HELP_ACK = "bsk-help-ack";
export const HELP_QUERY = "bsk-help-query";
export const HELP_FINISH = "bsk-help-finish";
export interface HelpRequestMessage {
type: typeof HELP_REQUEST;
@@ -24,6 +27,11 @@ export interface HelpRequestMessage {
timeoutMs: number;
}
export interface HelpAckMessage {
type: typeof HELP_ACK;
ok: true;
}
export interface HelpResponseMessage {
type: typeof HELP_RESPONSE;
outcome: "continued" | "cancelled";
@@ -35,6 +43,22 @@ export interface HelpCancelMessage {
requestId: string;
}
export interface HelpQueryMessage {
type: typeof HELP_QUERY;
}
export interface HelpQueryResponse {
active: boolean;
request?: Omit<HelpRequestMessage, "type">;
}
export interface HelpFinishMessage {
type: typeof HELP_FINISH;
requestId: string;
outcome: "continued" | "cancelled";
note?: string;
}
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
if (typeof msg !== "object" || msg === null) {
return false;
@@ -58,3 +82,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
const m = msg as Record<string, unknown>;
return m.type === HELP_CANCEL && typeof m.requestId === "string";
}
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return (
m.type === HELP_RESPONSE &&
(m.outcome === "continued" || m.outcome === "cancelled") &&
(m.note === undefined || typeof m.note === "string")
);
}
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === HELP_ACK && m.ok === true;
}
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === HELP_QUERY;
}
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return (
m.type === HELP_FINISH &&
typeof m.requestId === "string" &&
(m.outcome === "continued" || m.outcome === "cancelled") &&
(m.note === undefined || typeof m.note === "string")
);
}
@@ -27,9 +27,9 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
windows: { update: vi.fn(async () => ({}) as never) },
activateTab: vi.fn(async () => {}),
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
watchTabNavigation: () => () => {},
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
notifications: null,
autoAttachLifecycle: false,
...over,
};
}
@@ -78,22 +78,29 @@ describe("handleRequestHelp", () => {
expect(sentMsg.title).toBeUndefined();
});
it("returns navigated when the tab navigates during the wait", async () => {
const unwatch = vi.fn();
it("does not complete merely because the tab navigates during the wait", async () => {
vi.useFakeTimers();
const deps = baseDeps({
sendToTab: vi.fn(() => new Promise(() => {})),
watchTabNavigation: vi.fn((_tabId, cb) => {
queueMicrotask(() => cb());
return unwatch;
}),
});
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
deps,
);
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
expect(unwatch).toHaveBeenCalled();
try {
const pending = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 10 }),
deps,
);
await vi.advanceTimersByTimeAsync(5);
let settled = false;
void pending.then(() => {
settled = true;
});
await Promise.resolve();
expect(settled).toBe(false);
await vi.advanceTimersByTimeAsync(10);
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
} finally {
vi.useRealTimers();
}
});
it("returns timed_out when the wait expires", async () => {
@@ -115,6 +122,30 @@ describe("handleRequestHelp", () => {
}
});
it("returns completed when explicit completion criteria match", async () => {
const deps = baseDeps({
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
cdp: {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: true } };
return {};
}),
} as unknown as RequestHelpDeps["cdp"],
});
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({
tab_id: 5,
completion_criteria: {
any: [{ selector_exists: "#account-menu" }],
stable_for_ms: 0,
},
}),
deps,
);
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
});
it("tags ref targets via CDP and reports them matched", async () => {
const mgr = {
get: (id: string) =>
@@ -190,38 +221,40 @@ describe("handleRequestHelp", () => {
expect("code" in res).toBe(false);
});
it("returns protocol_error when sendToTab rejects", async () => {
it("keeps waiting when the initial overlay delivery rejects", async () => {
vi.useFakeTimers();
const deps = baseDeps({
sendToTab: vi.fn(async () => {
throw new Error("no receiver");
}),
});
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
deps,
);
expect("code" in res && res.code).toBe("protocol_error");
try {
const res = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 10 }),
deps,
);
await vi.advanceTimersByTimeAsync(20);
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
} finally {
vi.useRealTimers();
}
});
it("returns protocol_error for a malformed help response", async () => {
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
const res1 = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
undefinedReply,
);
expect("code" in res1 && res1.code).toBe("protocol_error");
const badOutcome = baseDeps({
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
});
const res2 = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
badOutcome,
);
expect("code" in res2 && res2.code).toBe("protocol_error");
it("keeps waiting for explicit completion after a malformed help response", async () => {
vi.useFakeTimers();
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
try {
const res = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 10 }),
deps,
);
await vi.advanceTimersByTimeAsync(20);
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
} finally {
vi.useRealTimers();
}
});
it("reports selector match status from CDP", async () => {
+1 -2
View File
@@ -29,7 +29,7 @@ import type {
import { isRequestFrame } from "@/transport/types";
import { handleConsole } from "./console";
import { handleEvaluate } from "./evaluate";
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
import {
handleNavigate,
@@ -394,7 +394,6 @@ export class ToolDispatcher {
await chrome.tabs.update(tabId, { active: true });
},
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
watchTabNavigation: defaultWatchTabNavigation,
...(this.cdp ? { cdp: this.cdp } : {}),
notifications: makeHelpNotifications(),
notificationCopy: this.helpNotificationCopy?.(),
+530 -154
View File
@@ -1,21 +1,24 @@
// `tool.request_help` — pause and ask the human to act on a tab.
//
// Brings the target tab to the foreground, resolves `ref` targets to a
// temporary attribute selector via CDP, asks the content script to enter
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
// until the user clicks Continue / Cancel, the wait times out, or the
// daemon cancels the RPC.
// `tool.request_help` — pause automation and let the human control a tab.
import {
HELP_ACK,
HELP_CANCEL,
HELP_FINISH,
HELP_QUERY,
HELP_REQUEST,
HELP_RESPONSE,
type HelpCancelMessage,
type HelpFinishMessage,
type HelpQueryResponse,
type HelpRequestMessage,
type HelpResponseMessage,
isHelpAckMessage,
isHelpFinishMessage,
isHelpQueryMessage,
isHelpResponseMessage,
} from "@/lib/help-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type {
HelpCompletionCondition,
HelpCompletionCriteria,
HelpTarget,
RequestHelpParams,
RequestHelpResult,
@@ -34,53 +37,60 @@ import {
import { lookupSnapshotRef } from "./snapshot-ref";
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
/** Attribute the overlay highlights for resolved `ref` targets. */
const HELP_ATTR = "data-bsk-help";
const HELP_SEND_RETRIES = 3;
const HELP_SEND_RETRY_DELAY_MS = 350;
const HELP_REARM_DEBOUNCE_MS = 150;
const HELP_REARM_MAX_ATTEMPTS = 12;
const HELP_REARM_RETRY_DELAY_MS = 400;
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
const COMPLETION_POLL_MS = 500;
export interface RequestHelpNotifications {
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
clear(id: string): Promise<boolean>;
}
export type TabNavigationUnsubscribe = () => void;
export interface RequestHelpDeps {
tabsApi: ChromeTabsApi;
windows: {
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
};
/** Activate a tab inside its window. */
activateTab(tabId: number): Promise<void>;
/** Send the help-request to the tab's content script and await reply. */
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
/**
* Watch for navigation on `tabId` (full page load or SPA URL change).
* Invokes `onNavigated` once; returns unsubscribe.
*/
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
cdp?: CdpRunner;
/** Pass `null` to skip OS notifications (tests). */
notifications: RequestHelpNotifications | null;
notificationCopy?: { title: string; body: string };
signal?: AbortSignal;
autoAttachLifecycle?: boolean;
}
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
export function defaultWatchTabNavigation(
tabId: number,
onNavigated: () => void,
): TabNavigationUnsubscribe {
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
if (updatedTabId !== tabId) return;
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
chrome.tabs.onUpdated.removeListener(listener);
onNavigated();
}
};
chrome.tabs.onUpdated.addListener(listener);
return () => chrome.tabs.onUpdated.removeListener(listener);
interface ActiveHelpRequest {
sessionId: string;
ctx: SessionContext;
requestId: string;
tabId: number;
agentWindowId: number;
prompt: string;
title?: string;
targets: HelpTarget[];
selectors: string[];
timeoutMs: number;
notificationId: string;
resolvedTargets?: ResolvedTarget[];
completionCriteria?: HelpCompletionCriteria;
deps: RequestHelpDeps;
settled: boolean;
resolve: (value: RequestHelpResult | RpcError) => void;
timeoutTimer: ReturnType<typeof setTimeout>;
completionTimer: ReturnType<typeof setInterval> | null;
completionMatchedSince: number | null;
abortHandler: (() => void) | null;
}
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
let defaultDeps: RequestHelpDeps | null = null;
function getDefaultDeps(): RequestHelpDeps {
if (!defaultDeps) {
@@ -91,7 +101,6 @@ function getDefaultDeps(): RequestHelpDeps {
await chrome.tabs.update(tabId, { active: true });
},
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
watchTabNavigation: defaultWatchTabNavigation,
notifications: {
create: (id, opts) =>
new Promise((resolve, reject) =>
@@ -113,11 +122,48 @@ function makeRequestId(tabId: number): string {
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
}
/**
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
* content overlay can highlight it with a plain attribute selector.
* Returns the selector on success, or null when the ref can't resolve.
*/
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function helpRequestMessage(help: ActiveHelpRequest): HelpRequestMessage {
return {
type: HELP_REQUEST,
requestId: help.requestId,
prompt: help.prompt,
...(help.title ? { title: help.title } : {}),
selectors: help.selectors,
timeoutMs: help.timeoutMs,
};
}
async function sendHelpRequestWithAck(
tabId: number,
msg: HelpRequestMessage,
deps: RequestHelpDeps,
): Promise<HelpFinishMessage | null> {
let lastError: unknown;
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
try {
const response = await deps.sendToTab(tabId, msg);
if (isHelpAckMessage(response)) return null;
if (isHelpResponseMessage(response)) {
return {
type: HELP_FINISH,
requestId: msg.requestId,
outcome: response.outcome,
...(response.note ? { note: response.note } : {}),
};
}
lastError = new Error("content script did not ack HELP_REQUEST");
} catch (err) {
lastError = err;
}
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
}
throw lastError ?? new Error("failed to show help overlay");
}
async function tagRefTarget(
cdp: CdpRunner,
tabId: number,
@@ -145,7 +191,6 @@ async function tagRefTarget(
}
}
/** Returns true when `selector` matches a live element in the page. */
async function selectorExists(
cdp: CdpRunner,
tabId: number,
@@ -163,7 +208,6 @@ async function selectorExists(
}
}
/** Remove every help attribute we added (best-effort cleanup). */
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
if (!cdp) return;
try {
@@ -180,34 +224,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
}
} catch {
// best-effort
// Best-effort cleanup.
}
}
export async function handleRequestHelp(
manager: SessionManager,
params: RequestHelpParams,
deps: RequestHelpDeps = getDefaultDeps(),
): Promise<RequestHelpResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "request_help");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
if (!params.prompt || typeof params.prompt !== "string") {
return { code: "invalid_params", message: "request_help requires a prompt" };
}
if (deps.signal?.aborted) {
return { code: "cancelled", message: "request_help aborted" };
}
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "request_help");
if (denied) return denied;
const tabId = target.tabId;
// Resolve targets → selectors. `selector` passes through; `ref` is
// tagged with an attribute via CDP and reported as its attr selector.
const targets = params.targets ?? [];
async function resolveHelpTargets(
ctx: ReturnType<SessionManager["get"]>,
tabId: number,
targets: HelpTarget[],
deps: RequestHelpDeps,
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
const selectors: string[] = [];
const resolved: ResolvedTarget[] = [];
let rootNodeId: number | undefined;
@@ -221,22 +247,20 @@ export async function handleRequestHelp(
rootNodeId = undefined;
}
}
for (let i = 0; i < targets.length; i++) {
const tgt = targets[i] as HelpTarget;
for (let i = 0; i < targets.length; i += 1) {
const tgt = targets[i];
if (tgt.selector) {
selectors.push(tgt.selector);
let matched: boolean;
if (deps.cdp) {
matched =
rootNodeId !== undefined
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
: false;
} else {
matched = true;
}
const matched =
deps.cdp && rootNodeId !== undefined
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
: deps.cdp
? false
: true;
resolved.push({ matched, selector: tgt.selector });
} else if (tgt.ref) {
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
const backendNodeId = looked?.backendNodeId ?? null;
let sel: string | null = null;
if (backendNodeId !== null && deps.cdp) {
@@ -247,7 +271,385 @@ export async function handleRequestHelp(
}
}
// Bring the tab to the foreground.
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
}
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
await clearRefTags(help.deps.cdp, help.tabId);
const { selectors, resolvedTargets } = await resolveHelpTargets(
help.ctx,
help.tabId,
help.targets,
help.deps,
);
help.selectors = selectors;
help.resolvedTargets = resolvedTargets;
}
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
await clearRefTags(help.deps.cdp, help.tabId);
if (help.deps.notifications) {
await help.deps.notifications.clear(help.notificationId).catch(() => {});
}
await help.deps
.sendToTab(help.tabId, { type: HELP_CANCEL, requestId: help.requestId })
.catch(() => {});
}
function finishHelp(
help: ActiveHelpRequest,
value: RequestHelpResult | RpcError,
notifyContent = true,
): void {
if (help.settled) return;
help.settled = true;
activeHelpRequests.delete(help.requestId);
clearTimeout(help.timeoutTimer);
if (help.completionTimer) clearInterval(help.completionTimer);
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
clearRearmTimer(help.tabId);
if (notifyContent) void cleanupHelp(help);
help.resolve(value);
}
function findHelpByTabId(tabId: number): ActiveHelpRequest | null {
for (const help of activeHelpRequests.values()) {
if (!help.settled && help.tabId === tabId) return help;
}
return null;
}
async function findHelpForTab(
tabId: number,
deps: RequestHelpDeps,
): Promise<ActiveHelpRequest | null> {
const direct = findHelpByTabId(tabId);
if (direct) return direct;
try {
const tab = await deps.tabsApi.get(tabId);
const windowId = tab.windowId;
if (typeof windowId !== "number") return null;
for (const help of activeHelpRequests.values()) {
if (!help.settled && help.agentWindowId === windowId) return help;
}
} catch {
return null;
}
return null;
}
function clearRearmTimer(tabId: number): void {
const timer = helpRearmTimers.get(tabId);
if (!timer) return;
clearTimeout(timer);
helpRearmTimers.delete(tabId);
}
async function rearmHelp(help: ActiveHelpRequest, targetTabId: number): Promise<boolean> {
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
try {
help.tabId = targetTabId;
await refreshHelpTargets(help);
const msg = helpRequestMessage(help);
const legacyFinish = await sendHelpRequestWithAck(targetTabId, msg, help.deps);
if (legacyFinish && !help.settled) {
finishHelp(help, {
outcome: legacyFinish.outcome,
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
tab_id: help.tabId,
resolved_targets: help.resolvedTargets,
});
}
return true;
} catch {
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
}
}
return false;
}
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
const existing = helpRearmTimers.get(tabId);
if (existing) clearTimeout(existing);
helpRearmTimers.set(
tabId,
setTimeout(() => {
helpRearmTimers.delete(tabId);
void (async () => {
const help = await findHelpForTab(tabId, deps);
if (help) await rearmHelp(help, tabId);
})();
}, HELP_REARM_DEBOUNCE_MS),
);
}
let detachHelpObservation: (() => void) | null = null;
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
if (detachHelpObservation) return;
const detachRuntime = attachHelpRuntimeListener(deps);
const detachTabs = attachHelpTabListener(deps);
const detachNav = attachHelpNavigationListener(deps);
detachHelpObservation = () => {
detachRuntime();
detachTabs();
detachNav();
};
}
export function resetHelpLifecycleForTests(): void {
detachHelpObservation?.();
detachHelpObservation = null;
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
helpRearmTimers.clear();
for (const help of activeHelpRequests.values()) {
clearTimeout(help.timeoutTimer);
if (help.completionTimer) clearInterval(help.completionTimer);
}
activeHelpRequests.clear();
}
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
const listener = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: HelpQueryResponse) => void,
) => {
if (isHelpFinishMessage(message)) {
void finishHelpFromContent(message, sender, deps);
return false;
}
if (isHelpQueryMessage(message)) {
const tabId = sender.tab?.id;
if (tabId === undefined) {
sendResponse({ active: false });
return false;
}
void (async () => {
const help = await findHelpForTab(tabId, deps);
if (!help) {
sendResponse({ active: false });
return;
}
help.tabId = tabId;
await refreshHelpTargets(help);
sendResponse({
active: true,
request: {
requestId: help.requestId,
prompt: help.prompt,
...(help.title ? { title: help.title } : {}),
selectors: help.selectors,
timeoutMs: help.timeoutMs,
},
});
})();
return true;
}
return false;
};
chrome.runtime.onMessage.addListener(listener);
return () => chrome.runtime.onMessage.removeListener(listener);
}
async function finishHelpFromContent(
message: HelpFinishMessage,
sender: chrome.runtime.MessageSender,
deps: RequestHelpDeps,
): Promise<void> {
const help = activeHelpRequests.get(message.requestId);
if (!help || help.settled) return;
const tabId = sender.tab?.id;
if (tabId !== undefined) {
const match = await findHelpForTab(tabId, deps);
if (match !== help) return;
}
finishHelp(
help,
{
outcome: message.outcome,
...(message.note ? { note: message.note } : {}),
tab_id: help.tabId,
resolved_targets: help.resolvedTargets,
},
false,
);
}
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
const onCreated = (tab: chrome.tabs.Tab) => {
const tabId = tab.id;
const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return;
for (const help of activeHelpRequests.values()) {
if (!help.settled && help.agentWindowId === windowId) {
scheduleRearmForTab(tabId, deps);
return;
}
}
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
scheduleRearmForTab(activeInfo.tabId, deps);
};
chrome.tabs.onCreated?.addListener(onCreated);
chrome.tabs.onActivated?.addListener(onActivated);
return () => {
chrome.tabs.onCreated?.removeListener(onCreated);
chrome.tabs.onActivated?.removeListener(onActivated);
};
}
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
if (chrome.webNavigation?.onCompleted) {
const completedListener = (
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
) => {
if (details.frameId !== 0) return;
onMainFrameComplete(details.tabId);
};
chrome.webNavigation.onCompleted.addListener(completedListener);
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
}
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
if (info.status !== "complete") return;
onMainFrameComplete(tabId);
};
chrome.tabs.onUpdated?.addListener(listener);
return () => chrome.tabs.onUpdated?.removeListener(listener);
}
async function evaluateCompletionCondition(
condition: HelpCompletionCondition,
help: ActiveHelpRequest,
): Promise<boolean> {
if (condition.url_contains || condition.url_matches) {
let url = "";
try {
const tab = await help.deps.tabsApi.get(help.tabId);
url = tab.url ?? "";
} catch {
return false;
}
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
if (condition.url_matches) {
try {
if (!new RegExp(condition.url_matches).test(url)) return false;
} catch {
return false;
}
}
}
if (
condition.selector_exists ||
condition.selector_missing ||
condition.text_exists ||
condition.text_missing
) {
if (!help.deps.cdp) return false;
const expression = `(() => {
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
if (selectorExists && !document.querySelector(selectorExists)) return false;
if (selectorMissing && document.querySelector(selectorMissing)) return false;
const text = document.body ? document.body.innerText || "" : "";
if (textExists && !text.includes(textExists)) return false;
if (textMissing && text.includes(textMissing)) return false;
return true;
})()`;
try {
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
help.tabId,
"Runtime.evaluate",
{ expression, returnByValue: true },
);
if (result.result?.value !== true) return false;
} catch {
return false;
}
}
return true;
}
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
const criteria = help.completionCriteria;
if (!criteria) return false;
const any = criteria.any ?? [];
const all = criteria.all ?? [];
const hasAny = any.length > 0;
const hasAll = all.length > 0;
if (!hasAny && !hasAll) return false;
if (hasAll) {
for (const condition of all) {
if (!(await evaluateCompletionCondition(condition, help))) return false;
}
}
if (hasAny) {
for (const condition of any) {
if (await evaluateCompletionCondition(condition, help)) return true;
}
return false;
}
return true;
}
function startCompletionPolling(help: ActiveHelpRequest): void {
if (!help.completionCriteria) return;
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
const check = async () => {
if (help.settled) return;
const matched = await completionCriteriaMatches(help);
const now = Date.now();
if (!matched) {
help.completionMatchedSince = null;
return;
}
help.completionMatchedSince ??= now;
if (now - help.completionMatchedSince < stableMs) return;
finishHelp(help, {
outcome: "completed",
completed_by: "system",
tab_id: help.tabId,
resolved_targets: help.resolvedTargets,
});
};
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
void check();
}
export async function handleRequestHelp(
manager: SessionManager,
params: RequestHelpParams,
deps: RequestHelpDeps = getDefaultDeps(),
): Promise<RequestHelpResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "request_help");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
if (!params.prompt || typeof params.prompt !== "string") {
return { code: "invalid_params", message: "request_help requires a prompt" };
}
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "request_help");
if (denied) return denied;
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
const tabId = target.tabId;
const initialTargets = params.targets ?? [];
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
await deps.activateTab(tabId).catch(() => {});
@@ -255,7 +657,6 @@ export async function handleRequestHelp(
const notificationId = `bsk-help:${requestId}`;
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
// OS notification (best-effort).
if (deps.notifications) {
const copy = deps.notificationCopy ?? {
title: "BrowserSkill: Agent needs your help",
@@ -272,85 +673,60 @@ export async function handleRequestHelp(
.catch(() => {});
}
const cleanup = async () => {
await clearRefTags(deps.cdp, tabId);
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
// Retract the overlay in case we resolved via timeout / abort.
await deps
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
.catch(() => {});
};
return new Promise<RequestHelpResult | RpcError>((resolve) => {
const timeoutTimer = setTimeout(() => {
finishHelp(help, {
outcome: "timed_out",
tab_id: help.tabId,
resolved_targets: help.resolvedTargets,
});
}, timeoutMs);
const request: HelpRequestMessage = {
type: HELP_REQUEST,
requestId,
prompt: params.prompt,
...(params.title ? { title: params.title } : {}),
selectors,
timeoutMs,
};
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
let settled = false;
const unwatchNav = deps.watchTabNavigation(tabId, () => {
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
});
const finish = (value: RequestHelpResult | RpcError) => {
if (settled) return;
settled = true;
if (timer !== null) clearTimeout(timer);
unwatchNav();
deps.signal?.removeEventListener("abort", onAbort);
void cleanup();
resolveOuter(value);
const help: ActiveHelpRequest = {
sessionId: params.session_id,
ctx,
requestId,
tabId,
agentWindowId: ctx.agentWindowId,
prompt: params.prompt,
...(params.title ? { title: params.title } : {}),
targets: initialTargets,
selectors,
timeoutMs,
notificationId,
resolvedTargets,
completionCriteria: params.completion_criteria,
deps,
settled: false,
resolve,
timeoutTimer,
completionTimer: null,
completionMatchedSince: null,
abortHandler: null,
};
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
const timer = setTimeout(() => {
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
}, timeoutMs) as unknown as number;
if (deps.signal) {
if (deps.signal.aborted) {
onAbort();
return;
}
deps.signal.addEventListener("abort", onAbort, { once: true });
const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
help.abortHandler = onAbort;
activeHelpRequests.set(requestId, help);
if (deps.signal?.aborted) {
onAbort();
return;
}
deps.signal?.addEventListener("abort", onAbort, { once: true });
startCompletionPolling(help);
deps
.sendToTab(tabId, request)
.then((reply) => {
const res = reply as HelpResponseMessage | undefined;
if (
!res ||
res.type !== HELP_RESPONSE ||
(res.outcome !== "continued" && res.outcome !== "cancelled")
) {
finish({
code: "protocol_error",
message: `invalid help response from tab ${tabId}`,
});
return;
}
finish({
outcome: res.outcome,
...(res.note ? { note: res.note } : {}),
tab_id: tabId,
resolved_targets: resolvedTargets,
void sendHelpRequestWithAck(tabId, helpRequestMessage(help), deps)
.then((legacyFinish) => {
if (!legacyFinish || help.settled) return;
finishHelp(help, {
outcome: legacyFinish.outcome,
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
tab_id: help.tabId,
resolved_targets: help.resolvedTargets,
});
})
.catch((err) => {
finish({
code: "protocol_error",
message: `failed to show help overlay on tab ${tabId}: ${
err instanceof Error ? err.message : String(err)
}`,
});
.catch(() => {
scheduleRearmForTab(tabId, deps);
});
});
}
+18 -1
View File
@@ -514,7 +514,22 @@ export interface HelpTarget {
selector?: string;
}
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated";
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated";
export interface HelpCompletionCondition {
url_contains?: string;
url_matches?: string;
selector_exists?: string;
selector_missing?: string;
text_exists?: string;
text_missing?: string;
}
export interface HelpCompletionCriteria {
any?: HelpCompletionCondition[];
all?: HelpCompletionCondition[];
stable_for_ms?: number;
}
export interface ResolvedTarget {
matched: boolean;
@@ -528,11 +543,13 @@ export interface RequestHelpParams {
prompt: string;
title?: string;
targets?: HelpTarget[];
completion_criteria?: HelpCompletionCriteria;
timeout_ms?: number;
}
export interface RequestHelpResult {
outcome: HelpOutcome;
completed_by?: "system";
note?: string;
tab_id: number;
resolved_targets?: ResolvedTarget[];