mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(runtime): fix early control return and update lifecycle
This commit is contained in:
@@ -342,6 +342,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
z-index: 2147483647;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: flex-start;
|
||||
min-height: 0;
|
||||
gap: 10px;
|
||||
width: var(--bsk-help-banner-width);
|
||||
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
||||
@@ -359,16 +361,18 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
.bsk-help-banner[data-collapsed="true"] {
|
||||
gap: 0;
|
||||
width: var(--bsk-help-banner-width);
|
||||
padding: 10px 12px;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.bsk-help-drag-strip {
|
||||
flex-shrink: 0;
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
left: 0;
|
||||
right: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 12px;
|
||||
margin: -8px 0 -10px;
|
||||
cursor: grab;
|
||||
user-select: none;
|
||||
touch-action: none;
|
||||
@@ -395,10 +399,11 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
||||
margin: -6px 0 -8px;
|
||||
top: 4px;
|
||||
}
|
||||
|
||||
.bsk-help-header {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
@@ -420,25 +425,17 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
|
||||
.bsk-help-header-actions {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-left: auto;
|
||||
overflow: hidden;
|
||||
max-width: 0;
|
||||
opacity: 0;
|
||||
transform: translateX(8px);
|
||||
pointer-events: none;
|
||||
transition:
|
||||
max-width var(--bsk-help-duration) var(--bsk-help-ease),
|
||||
opacity 220ms var(--bsk-help-ease),
|
||||
transform var(--bsk-help-duration) var(--bsk-help-ease);
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
||||
display: flex;
|
||||
max-width: 280px;
|
||||
opacity: 1;
|
||||
transform: translateX(0);
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
@@ -471,15 +468,13 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
|
||||
.bsk-help-body {
|
||||
display: grid;
|
||||
grid-template-rows: 1fr;
|
||||
display: block;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
||||
position: absolute;
|
||||
grid-template-rows: 0fr;
|
||||
width: 0;
|
||||
height: 0;
|
||||
overflow: hidden;
|
||||
@@ -528,6 +523,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
.bsk-help-footer-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
@@ -539,7 +535,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: #4b5563;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.bsk-help-btn-continue {
|
||||
@@ -550,16 +547,19 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #fff;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.bsk-help-header-actions .bsk-help-btn-cancel,
|
||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||
padding: 6px 12px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||
padding: 6px 14px;
|
||||
max-width: 180px;
|
||||
}
|
||||
|
||||
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
||||
|
||||
@@ -204,6 +204,15 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(styles).toContain("width: 0");
|
||||
});
|
||||
|
||||
it("uses natural body height in the expanded layout", () => {
|
||||
const { container } = renderOverlay(baseRequest());
|
||||
const styles = overlayStyles(container);
|
||||
|
||||
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
|
||||
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
|
||||
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
|
||||
});
|
||||
|
||||
it("keeps the same banner width when collapsed", () => {
|
||||
const { container } = renderOverlay(baseRequest());
|
||||
const styles = overlayStyles(container);
|
||||
|
||||
@@ -14,10 +14,14 @@ import {
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
HELP_RESPONSE,
|
||||
HELP_ACK,
|
||||
HELP_FINISH,
|
||||
HELP_QUERY,
|
||||
type HelpAckMessage,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
type HelpResponseMessage,
|
||||
isHelpCancelMessage,
|
||||
isHelpRequestMessage,
|
||||
} from "@/lib/help-bridge";
|
||||
@@ -55,8 +59,6 @@ export default defineContentScript({
|
||||
if (window.top !== window) return;
|
||||
|
||||
const overlays = new OverlayController();
|
||||
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
|
||||
null;
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
@@ -116,8 +118,7 @@ export default defineContentScript({
|
||||
function resetAgentOverlayState(sessionId: string) {
|
||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||
if (previousHelp) {
|
||||
activeHelpRespond?.("cancelled");
|
||||
activeHelpRespond = null;
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
@@ -156,7 +157,7 @@ export default defineContentScript({
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||
) => {
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
@@ -220,41 +221,29 @@ export default defineContentScript({
|
||||
if (isHelpCancelMessage(message)) {
|
||||
const state = overlays.snapshot();
|
||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||
activeHelpRespond?.("cancelled");
|
||||
overlays.clearAgentHelpRequest(message.requestId);
|
||||
renderOverlay();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isHelpRequestMessage(message)) {
|
||||
const helpMsg = message as HelpRequestMessage;
|
||||
let responded = false;
|
||||
const respond = (outcome: "continued" | "cancelled", note?: string) => {
|
||||
if (responded) return;
|
||||
responded = true;
|
||||
const reply: HelpResponseMessage = {
|
||||
type: HELP_RESPONSE,
|
||||
outcome,
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
sendResponse(reply);
|
||||
activeHelpRespond = null;
|
||||
overlays.clearAgentHelpRequest(helpMsg.requestId);
|
||||
renderOverlay();
|
||||
};
|
||||
const previousHelp = overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
|
||||
onCancel: () => respond("cancelled"),
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
if (previousHelp) {
|
||||
activeHelpRespond?.("cancelled");
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
activeHelpRespond = respond;
|
||||
renderOverlay();
|
||||
return true; // async sendResponse
|
||||
sendResponse({ type: HELP_ACK, ok: true });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (message.type === "borrow-request") {
|
||||
@@ -282,9 +271,59 @@ export default defineContentScript({
|
||||
return false;
|
||||
};
|
||||
|
||||
async function sendHelpFinish(
|
||||
requestId: string,
|
||||
outcome: "continued" | "cancelled",
|
||||
note?: string,
|
||||
): Promise<void> {
|
||||
const msg: HelpFinishMessage = {
|
||||
type: HELP_FINISH,
|
||||
requestId,
|
||||
outcome,
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
overlays.clearAgentHelpRequest(requestId);
|
||||
renderOverlay();
|
||||
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||
console.debug("[bsk overlay] help finish failed", err);
|
||||
});
|
||||
}
|
||||
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||
overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<boolean> {
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
try {
|
||||
const helpQuery = (await chrome.runtime.sendMessage({
|
||||
type: HELP_QUERY,
|
||||
})) as HelpQueryResponse | undefined;
|
||||
if (helpQuery?.active && helpQuery.request) {
|
||||
mountHelpRequest(helpQuery.request);
|
||||
renderOverlay();
|
||||
return true;
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] help query failed", err);
|
||||
}
|
||||
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function syncAgentOverlay(): Promise<void> {
|
||||
if (!(await anySessionLive())) return;
|
||||
try {
|
||||
const helpActive = await queryActiveHelpWithRetry();
|
||||
const reply = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_WHO_AM_I,
|
||||
})) as OverlayWhoAmIResponse | undefined;
|
||||
@@ -319,6 +358,10 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
if (!helpActive && overlays.snapshot().activeHelp === null) {
|
||||
void queryActiveHelpWithRetry();
|
||||
}
|
||||
|
||||
overlays.activateAgentSession(reply.sessionId);
|
||||
renderOverlay();
|
||||
} catch (err) {
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
HELP_ACK,
|
||||
HELP_CANCEL,
|
||||
HELP_FINISH,
|
||||
HELP_QUERY,
|
||||
HELP_REQUEST,
|
||||
HELP_RESPONSE,
|
||||
isHelpAckMessage,
|
||||
isHelpCancelMessage,
|
||||
isHelpFinishMessage,
|
||||
isHelpQueryMessage,
|
||||
isHelpRequestMessage,
|
||||
isHelpResponseMessage,
|
||||
} from "../help-bridge";
|
||||
|
||||
describe("help-bridge", () => {
|
||||
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
|
||||
expect(HELP_REQUEST).toBe("bsk-help-request");
|
||||
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
||||
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
||||
expect(HELP_ACK).toBe("bsk-help-ack");
|
||||
expect(HELP_QUERY).toBe("bsk-help-query");
|
||||
expect(HELP_FINISH).toBe("bsk-help-finish");
|
||||
});
|
||||
|
||||
it("type-guards a help-request message", () => {
|
||||
@@ -55,4 +65,18 @@ describe("help-bridge", () => {
|
||||
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
||||
expect(isHelpCancelMessage(null)).toBe(false);
|
||||
});
|
||||
|
||||
it("type-guards lifecycle messages", () => {
|
||||
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
|
||||
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
|
||||
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,12 +6,15 @@
|
||||
* Mirrors the borrow-confirmation message style (see
|
||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||
* the content script replies once the user clicks Continue / Cancel.
|
||||
* the content script acks display, then later reports Done / Cancel.
|
||||
*/
|
||||
|
||||
export const HELP_REQUEST = "bsk-help-request";
|
||||
export const HELP_RESPONSE = "bsk-help-response";
|
||||
export const HELP_CANCEL = "bsk-help-cancel";
|
||||
export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
@@ -24,6 +27,11 @@ export interface HelpRequestMessage {
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
export interface HelpAckMessage {
|
||||
type: typeof HELP_ACK;
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export interface HelpResponseMessage {
|
||||
type: typeof HELP_RESPONSE;
|
||||
outcome: "continued" | "cancelled";
|
||||
@@ -35,6 +43,22 @@ export interface HelpCancelMessage {
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface HelpQueryMessage {
|
||||
type: typeof HELP_QUERY;
|
||||
}
|
||||
|
||||
export interface HelpQueryResponse {
|
||||
active: boolean;
|
||||
request?: Omit<HelpRequestMessage, "type">;
|
||||
}
|
||||
|
||||
export interface HelpFinishMessage {
|
||||
type: typeof HELP_FINISH;
|
||||
requestId: string;
|
||||
outcome: "continued" | "cancelled";
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
if (typeof msg !== "object" || msg === null) {
|
||||
return false;
|
||||
@@ -58,3 +82,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
||||
}
|
||||
|
||||
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return (
|
||||
m.type === HELP_RESPONSE &&
|
||||
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||
(m.note === undefined || typeof m.note === "string")
|
||||
);
|
||||
}
|
||||
|
||||
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === HELP_ACK && m.ok === true;
|
||||
}
|
||||
|
||||
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === HELP_QUERY;
|
||||
}
|
||||
|
||||
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return (
|
||||
m.type === HELP_FINISH &&
|
||||
typeof m.requestId === "string" &&
|
||||
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||
(m.note === undefined || typeof m.note === "string")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -27,9 +27,9 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
|
||||
windows: { update: vi.fn(async () => ({}) as never) },
|
||||
activateTab: vi.fn(async () => {}),
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
||||
watchTabNavigation: () => () => {},
|
||||
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
||||
notifications: null,
|
||||
autoAttachLifecycle: false,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
@@ -78,22 +78,29 @@ describe("handleRequestHelp", () => {
|
||||
expect(sentMsg.title).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns navigated when the tab navigates during the wait", async () => {
|
||||
const unwatch = vi.fn();
|
||||
it("does not complete merely because the tab navigates during the wait", async () => {
|
||||
vi.useFakeTimers();
|
||||
const deps = baseDeps({
|
||||
sendToTab: vi.fn(() => new Promise(() => {})),
|
||||
watchTabNavigation: vi.fn((_tabId, cb) => {
|
||||
queueMicrotask(() => cb());
|
||||
return unwatch;
|
||||
}),
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
deps,
|
||||
);
|
||||
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
|
||||
expect(unwatch).toHaveBeenCalled();
|
||||
try {
|
||||
const pending = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
let settled = false;
|
||||
void pending.then(() => {
|
||||
settled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns timed_out when the wait expires", async () => {
|
||||
@@ -115,6 +122,30 @@ describe("handleRequestHelp", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("returns completed when explicit completion criteria match", async () => {
|
||||
const deps = baseDeps({
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||
cdp: {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: true } };
|
||||
return {};
|
||||
}),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({
|
||||
tab_id: 5,
|
||||
completion_criteria: {
|
||||
any: [{ selector_exists: "#account-menu" }],
|
||||
stable_for_ms: 0,
|
||||
},
|
||||
}),
|
||||
deps,
|
||||
);
|
||||
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
|
||||
});
|
||||
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
@@ -190,38 +221,40 @@ describe("handleRequestHelp", () => {
|
||||
expect("code" in res).toBe(false);
|
||||
});
|
||||
|
||||
it("returns protocol_error when sendToTab rejects", async () => {
|
||||
it("keeps waiting when the initial overlay delivery rejects", async () => {
|
||||
vi.useFakeTimers();
|
||||
const deps = baseDeps({
|
||||
sendToTab: vi.fn(async () => {
|
||||
throw new Error("no receiver");
|
||||
}),
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
deps,
|
||||
);
|
||||
expect("code" in res && res.code).toBe("protocol_error");
|
||||
try {
|
||||
const res = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns protocol_error for a malformed help response", async () => {
|
||||
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||
const res1 = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
undefinedReply,
|
||||
);
|
||||
expect("code" in res1 && res1.code).toBe("protocol_error");
|
||||
|
||||
const badOutcome = baseDeps({
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
|
||||
});
|
||||
const res2 = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
badOutcome,
|
||||
);
|
||||
expect("code" in res2 && res2.code).toBe("protocol_error");
|
||||
it("keeps waiting for explicit completion after a malformed help response", async () => {
|
||||
vi.useFakeTimers();
|
||||
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||
try {
|
||||
const res = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("reports selector match status from CDP", async () => {
|
||||
|
||||
@@ -29,7 +29,7 @@ import type {
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
@@ -394,7 +394,6 @@ export class ToolDispatcher {
|
||||
await chrome.tabs.update(tabId, { active: true });
|
||||
},
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
watchTabNavigation: defaultWatchTabNavigation,
|
||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||
notifications: makeHelpNotifications(),
|
||||
notificationCopy: this.helpNotificationCopy?.(),
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
// `tool.request_help` — pause and ask the human to act on a tab.
|
||||
//
|
||||
// Brings the target tab to the foreground, resolves `ref` targets to a
|
||||
// temporary attribute selector via CDP, asks the content script to enter
|
||||
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
|
||||
// until the user clicks Continue / Cancel, the wait times out, or the
|
||||
// daemon cancels the RPC.
|
||||
// `tool.request_help` — pause automation and let the human control a tab.
|
||||
|
||||
import {
|
||||
HELP_ACK,
|
||||
HELP_CANCEL,
|
||||
HELP_FINISH,
|
||||
HELP_QUERY,
|
||||
HELP_REQUEST,
|
||||
HELP_RESPONSE,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
type HelpResponseMessage,
|
||||
isHelpAckMessage,
|
||||
isHelpFinishMessage,
|
||||
isHelpQueryMessage,
|
||||
isHelpResponseMessage,
|
||||
} from "@/lib/help-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
HelpCompletionCondition,
|
||||
HelpCompletionCriteria,
|
||||
HelpTarget,
|
||||
RequestHelpParams,
|
||||
RequestHelpResult,
|
||||
@@ -34,53 +37,60 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
/** Attribute the overlay highlights for resolved `ref` targets. */
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
const HELP_REARM_MAX_ATTEMPTS = 12;
|
||||
const HELP_REARM_RETRY_DELAY_MS = 400;
|
||||
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
|
||||
const COMPLETION_POLL_MS = 500;
|
||||
|
||||
export interface RequestHelpNotifications {
|
||||
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
||||
clear(id: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export type TabNavigationUnsubscribe = () => void;
|
||||
|
||||
export interface RequestHelpDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
windows: {
|
||||
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
||||
};
|
||||
/** Activate a tab inside its window. */
|
||||
activateTab(tabId: number): Promise<void>;
|
||||
/** Send the help-request to the tab's content script and await reply. */
|
||||
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
||||
/**
|
||||
* Watch for navigation on `tabId` (full page load or SPA URL change).
|
||||
* Invokes `onNavigated` once; returns unsubscribe.
|
||||
*/
|
||||
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
|
||||
cdp?: CdpRunner;
|
||||
/** Pass `null` to skip OS notifications (tests). */
|
||||
notifications: RequestHelpNotifications | null;
|
||||
notificationCopy?: { title: string; body: string };
|
||||
signal?: AbortSignal;
|
||||
autoAttachLifecycle?: boolean;
|
||||
}
|
||||
|
||||
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
|
||||
export function defaultWatchTabNavigation(
|
||||
tabId: number,
|
||||
onNavigated: () => void,
|
||||
): TabNavigationUnsubscribe {
|
||||
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
|
||||
if (updatedTabId !== tabId) return;
|
||||
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
|
||||
chrome.tabs.onUpdated.removeListener(listener);
|
||||
onNavigated();
|
||||
}
|
||||
};
|
||||
chrome.tabs.onUpdated.addListener(listener);
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
interface ActiveHelpRequest {
|
||||
sessionId: string;
|
||||
ctx: SessionContext;
|
||||
requestId: string;
|
||||
tabId: number;
|
||||
agentWindowId: number;
|
||||
prompt: string;
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
completionCriteria?: HelpCompletionCriteria;
|
||||
deps: RequestHelpDeps;
|
||||
settled: boolean;
|
||||
resolve: (value: RequestHelpResult | RpcError) => void;
|
||||
timeoutTimer: ReturnType<typeof setTimeout>;
|
||||
completionTimer: ReturnType<typeof setInterval> | null;
|
||||
completionMatchedSince: number | null;
|
||||
abortHandler: (() => void) | null;
|
||||
}
|
||||
|
||||
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
|
||||
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
|
||||
let defaultDeps: RequestHelpDeps | null = null;
|
||||
function getDefaultDeps(): RequestHelpDeps {
|
||||
if (!defaultDeps) {
|
||||
@@ -91,7 +101,6 @@ function getDefaultDeps(): RequestHelpDeps {
|
||||
await chrome.tabs.update(tabId, { active: true });
|
||||
},
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
watchTabNavigation: defaultWatchTabNavigation,
|
||||
notifications: {
|
||||
create: (id, opts) =>
|
||||
new Promise((resolve, reject) =>
|
||||
@@ -113,11 +122,48 @@ function makeRequestId(tabId: number): string {
|
||||
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
|
||||
* content overlay can highlight it with a plain attribute selector.
|
||||
* Returns the selector on success, or null when the ref can't resolve.
|
||||
*/
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function helpRequestMessage(help: ActiveHelpRequest): HelpRequestMessage {
|
||||
return {
|
||||
type: HELP_REQUEST,
|
||||
requestId: help.requestId,
|
||||
prompt: help.prompt,
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
selectors: help.selectors,
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
async function sendHelpRequestWithAck(
|
||||
tabId: number,
|
||||
msg: HelpRequestMessage,
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<HelpFinishMessage | null> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, msg);
|
||||
if (isHelpAckMessage(response)) return null;
|
||||
if (isHelpResponseMessage(response)) {
|
||||
return {
|
||||
type: HELP_FINISH,
|
||||
requestId: msg.requestId,
|
||||
outcome: response.outcome,
|
||||
...(response.note ? { note: response.note } : {}),
|
||||
};
|
||||
}
|
||||
lastError = new Error("content script did not ack HELP_REQUEST");
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
|
||||
}
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -145,7 +191,6 @@ async function tagRefTarget(
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns true when `selector` matches a live element in the page. */
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -163,7 +208,6 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove every help attribute we added (best-effort cleanup). */
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
@@ -180,34 +224,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// best-effort
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRequestHelp(
|
||||
manager: SessionManager,
|
||||
params: RequestHelpParams,
|
||||
deps: RequestHelpDeps = getDefaultDeps(),
|
||||
): Promise<RequestHelpResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "request_help");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (!params.prompt || typeof params.prompt !== "string") {
|
||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||
}
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "request_help aborted" };
|
||||
}
|
||||
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||
if (denied) return denied;
|
||||
const tabId = target.tabId;
|
||||
|
||||
// Resolve targets → selectors. `selector` passes through; `ref` is
|
||||
// tagged with an attribute via CDP and reported as its attr selector.
|
||||
const targets = params.targets ?? [];
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
const selectors: string[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
@@ -221,22 +247,20 @@ export async function handleRequestHelp(
|
||||
rootNodeId = undefined;
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < targets.length; i++) {
|
||||
const tgt = targets[i] as HelpTarget;
|
||||
|
||||
for (let i = 0; i < targets.length; i += 1) {
|
||||
const tgt = targets[i];
|
||||
if (tgt.selector) {
|
||||
selectors.push(tgt.selector);
|
||||
let matched: boolean;
|
||||
if (deps.cdp) {
|
||||
matched =
|
||||
rootNodeId !== undefined
|
||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||
: false;
|
||||
} else {
|
||||
matched = true;
|
||||
}
|
||||
const matched =
|
||||
deps.cdp && rootNodeId !== undefined
|
||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||
: deps.cdp
|
||||
? false
|
||||
: true;
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
@@ -247,7 +271,385 @@ export async function handleRequestHelp(
|
||||
}
|
||||
}
|
||||
|
||||
// Bring the tab to the foreground.
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.tabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.tabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.tabId);
|
||||
if (help.deps.notifications) {
|
||||
await help.deps.notifications.clear(help.notificationId).catch(() => {});
|
||||
}
|
||||
await help.deps
|
||||
.sendToTab(help.tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
function finishHelp(
|
||||
help: ActiveHelpRequest,
|
||||
value: RequestHelpResult | RpcError,
|
||||
notifyContent = true,
|
||||
): void {
|
||||
if (help.settled) return;
|
||||
help.settled = true;
|
||||
activeHelpRequests.delete(help.requestId);
|
||||
clearTimeout(help.timeoutTimer);
|
||||
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
|
||||
clearRearmTimer(help.tabId);
|
||||
if (notifyContent) void cleanupHelp(help);
|
||||
help.resolve(value);
|
||||
}
|
||||
|
||||
function findHelpByTabId(tabId: number): ActiveHelpRequest | null {
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
if (!help.settled && help.tabId === tabId) return help;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function findHelpForTab(
|
||||
tabId: number,
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<ActiveHelpRequest | null> {
|
||||
const direct = findHelpByTabId(tabId);
|
||||
if (direct) return direct;
|
||||
|
||||
try {
|
||||
const tab = await deps.tabsApi.get(tabId);
|
||||
const windowId = tab.windowId;
|
||||
if (typeof windowId !== "number") return null;
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
if (!help.settled && help.agentWindowId === windowId) return help;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function clearRearmTimer(tabId: number): void {
|
||||
const timer = helpRearmTimers.get(tabId);
|
||||
if (!timer) return;
|
||||
clearTimeout(timer);
|
||||
helpRearmTimers.delete(tabId);
|
||||
}
|
||||
|
||||
async function rearmHelp(help: ActiveHelpRequest, targetTabId: number): Promise<boolean> {
|
||||
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
help.tabId = targetTabId;
|
||||
await refreshHelpTargets(help);
|
||||
const msg = helpRequestMessage(help);
|
||||
const legacyFinish = await sendHelpRequestWithAck(targetTabId, msg, help.deps);
|
||||
if (legacyFinish && !help.settled) {
|
||||
finishHelp(help, {
|
||||
outcome: legacyFinish.outcome,
|
||||
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||
tab_id: help.tabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
|
||||
const existing = helpRearmTimers.get(tabId);
|
||||
if (existing) clearTimeout(existing);
|
||||
helpRearmTimers.set(
|
||||
tabId,
|
||||
setTimeout(() => {
|
||||
helpRearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const help = await findHelpForTab(tabId, deps);
|
||||
if (help) await rearmHelp(help, tabId);
|
||||
})();
|
||||
}, HELP_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
}
|
||||
|
||||
let detachHelpObservation: (() => void) | null = null;
|
||||
|
||||
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
|
||||
if (detachHelpObservation) return;
|
||||
const detachRuntime = attachHelpRuntimeListener(deps);
|
||||
const detachTabs = attachHelpTabListener(deps);
|
||||
const detachNav = attachHelpNavigationListener(deps);
|
||||
detachHelpObservation = () => {
|
||||
detachRuntime();
|
||||
detachTabs();
|
||||
detachNav();
|
||||
};
|
||||
}
|
||||
|
||||
export function resetHelpLifecycleForTests(): void {
|
||||
detachHelpObservation?.();
|
||||
detachHelpObservation = null;
|
||||
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
|
||||
helpRearmTimers.clear();
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
clearTimeout(help.timeoutTimer);
|
||||
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||
}
|
||||
activeHelpRequests.clear();
|
||||
}
|
||||
|
||||
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: HelpQueryResponse) => void,
|
||||
) => {
|
||||
if (isHelpFinishMessage(message)) {
|
||||
void finishHelpFromContent(message, sender, deps);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isHelpQueryMessage(message)) {
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId === undefined) {
|
||||
sendResponse({ active: false });
|
||||
return false;
|
||||
}
|
||||
void (async () => {
|
||||
const help = await findHelpForTab(tabId, deps);
|
||||
if (!help) {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
help.tabId = tabId;
|
||||
await refreshHelpTargets(help);
|
||||
sendResponse({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: help.requestId,
|
||||
prompt: help.prompt,
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
selectors: help.selectors,
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
})();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
async function finishHelpFromContent(
|
||||
message: HelpFinishMessage,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<void> {
|
||||
const help = activeHelpRequests.get(message.requestId);
|
||||
if (!help || help.settled) return;
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId !== undefined) {
|
||||
const match = await findHelpForTab(tabId, deps);
|
||||
if (match !== help) return;
|
||||
}
|
||||
finishHelp(
|
||||
help,
|
||||
{
|
||||
outcome: message.outcome,
|
||||
...(message.note ? { note: message.note } : {}),
|
||||
tab_id: help.tabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
},
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
|
||||
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||
const tabId = tab.id;
|
||||
const windowId = tab.windowId;
|
||||
if (tabId === undefined || windowId === undefined) return;
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
if (!help.settled && help.agentWindowId === windowId) {
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||
};
|
||||
chrome.tabs.onCreated?.addListener(onCreated);
|
||||
chrome.tabs.onActivated?.addListener(onActivated);
|
||||
return () => {
|
||||
chrome.tabs.onCreated?.removeListener(onCreated);
|
||||
chrome.tabs.onActivated?.removeListener(onActivated);
|
||||
};
|
||||
}
|
||||
|
||||
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
|
||||
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
|
||||
|
||||
if (chrome.webNavigation?.onCompleted) {
|
||||
const completedListener = (
|
||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
onMainFrameComplete(details.tabId);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||
}
|
||||
|
||||
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||
if (info.status !== "complete") return;
|
||||
onMainFrameComplete(tabId);
|
||||
};
|
||||
chrome.tabs.onUpdated?.addListener(listener);
|
||||
return () => chrome.tabs.onUpdated?.removeListener(listener);
|
||||
}
|
||||
|
||||
async function evaluateCompletionCondition(
|
||||
condition: HelpCompletionCondition,
|
||||
help: ActiveHelpRequest,
|
||||
): Promise<boolean> {
|
||||
if (condition.url_contains || condition.url_matches) {
|
||||
let url = "";
|
||||
try {
|
||||
const tab = await help.deps.tabsApi.get(help.tabId);
|
||||
url = tab.url ?? "";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
|
||||
if (condition.url_matches) {
|
||||
try {
|
||||
if (!new RegExp(condition.url_matches).test(url)) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
condition.selector_exists ||
|
||||
condition.selector_missing ||
|
||||
condition.text_exists ||
|
||||
condition.text_missing
|
||||
) {
|
||||
if (!help.deps.cdp) return false;
|
||||
const expression = `(() => {
|
||||
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
|
||||
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
|
||||
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
|
||||
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
|
||||
if (selectorExists && !document.querySelector(selectorExists)) return false;
|
||||
if (selectorMissing && document.querySelector(selectorMissing)) return false;
|
||||
const text = document.body ? document.body.innerText || "" : "";
|
||||
if (textExists && !text.includes(textExists)) return false;
|
||||
if (textMissing && text.includes(textMissing)) return false;
|
||||
return true;
|
||||
})()`;
|
||||
try {
|
||||
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
|
||||
help.tabId,
|
||||
"Runtime.evaluate",
|
||||
{ expression, returnByValue: true },
|
||||
);
|
||||
if (result.result?.value !== true) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
|
||||
const criteria = help.completionCriteria;
|
||||
if (!criteria) return false;
|
||||
const any = criteria.any ?? [];
|
||||
const all = criteria.all ?? [];
|
||||
const hasAny = any.length > 0;
|
||||
const hasAll = all.length > 0;
|
||||
if (!hasAny && !hasAll) return false;
|
||||
|
||||
if (hasAll) {
|
||||
for (const condition of all) {
|
||||
if (!(await evaluateCompletionCondition(condition, help))) return false;
|
||||
}
|
||||
}
|
||||
if (hasAny) {
|
||||
for (const condition of any) {
|
||||
if (await evaluateCompletionCondition(condition, help)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function startCompletionPolling(help: ActiveHelpRequest): void {
|
||||
if (!help.completionCriteria) return;
|
||||
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
|
||||
const check = async () => {
|
||||
if (help.settled) return;
|
||||
const matched = await completionCriteriaMatches(help);
|
||||
const now = Date.now();
|
||||
if (!matched) {
|
||||
help.completionMatchedSince = null;
|
||||
return;
|
||||
}
|
||||
help.completionMatchedSince ??= now;
|
||||
if (now - help.completionMatchedSince < stableMs) return;
|
||||
finishHelp(help, {
|
||||
outcome: "completed",
|
||||
completed_by: "system",
|
||||
tab_id: help.tabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
};
|
||||
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
|
||||
void check();
|
||||
}
|
||||
|
||||
export async function handleRequestHelp(
|
||||
manager: SessionManager,
|
||||
params: RequestHelpParams,
|
||||
deps: RequestHelpDeps = getDefaultDeps(),
|
||||
): Promise<RequestHelpResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "request_help");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (!params.prompt || typeof params.prompt !== "string") {
|
||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||
}
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
|
||||
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||
if (denied) return denied;
|
||||
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
|
||||
@@ -255,7 +657,6 @@ export async function handleRequestHelp(
|
||||
const notificationId = `bsk-help:${requestId}`;
|
||||
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
||||
|
||||
// OS notification (best-effort).
|
||||
if (deps.notifications) {
|
||||
const copy = deps.notificationCopy ?? {
|
||||
title: "BrowserSkill: Agent needs your help",
|
||||
@@ -272,85 +673,60 @@ export async function handleRequestHelp(
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
const cleanup = async () => {
|
||||
await clearRefTags(deps.cdp, tabId);
|
||||
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
|
||||
// Retract the overlay in case we resolved via timeout / abort.
|
||||
await deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
|
||||
.catch(() => {});
|
||||
};
|
||||
return new Promise<RequestHelpResult | RpcError>((resolve) => {
|
||||
const timeoutTimer = setTimeout(() => {
|
||||
finishHelp(help, {
|
||||
outcome: "timed_out",
|
||||
tab_id: help.tabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
const request: HelpRequestMessage = {
|
||||
type: HELP_REQUEST,
|
||||
requestId,
|
||||
prompt: params.prompt,
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
selectors,
|
||||
timeoutMs,
|
||||
};
|
||||
|
||||
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
|
||||
|
||||
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
|
||||
let settled = false;
|
||||
const unwatchNav = deps.watchTabNavigation(tabId, () => {
|
||||
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
|
||||
});
|
||||
|
||||
const finish = (value: RequestHelpResult | RpcError) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer !== null) clearTimeout(timer);
|
||||
unwatchNav();
|
||||
deps.signal?.removeEventListener("abort", onAbort);
|
||||
void cleanup();
|
||||
resolveOuter(value);
|
||||
const help: ActiveHelpRequest = {
|
||||
sessionId: params.session_id,
|
||||
ctx,
|
||||
requestId,
|
||||
tabId,
|
||||
agentWindowId: ctx.agentWindowId,
|
||||
prompt: params.prompt,
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
completionCriteria: params.completion_criteria,
|
||||
deps,
|
||||
settled: false,
|
||||
resolve,
|
||||
timeoutTimer,
|
||||
completionTimer: null,
|
||||
completionMatchedSince: null,
|
||||
abortHandler: null,
|
||||
};
|
||||
|
||||
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
|
||||
}, timeoutMs) as unknown as number;
|
||||
|
||||
if (deps.signal) {
|
||||
if (deps.signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
deps.signal.addEventListener("abort", onAbort, { once: true });
|
||||
const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
|
||||
help.abortHandler = onAbort;
|
||||
activeHelpRequests.set(requestId, help);
|
||||
if (deps.signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
startCompletionPolling(help);
|
||||
|
||||
deps
|
||||
.sendToTab(tabId, request)
|
||||
.then((reply) => {
|
||||
const res = reply as HelpResponseMessage | undefined;
|
||||
if (
|
||||
!res ||
|
||||
res.type !== HELP_RESPONSE ||
|
||||
(res.outcome !== "continued" && res.outcome !== "cancelled")
|
||||
) {
|
||||
finish({
|
||||
code: "protocol_error",
|
||||
message: `invalid help response from tab ${tabId}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
finish({
|
||||
outcome: res.outcome,
|
||||
...(res.note ? { note: res.note } : {}),
|
||||
tab_id: tabId,
|
||||
resolved_targets: resolvedTargets,
|
||||
void sendHelpRequestWithAck(tabId, helpRequestMessage(help), deps)
|
||||
.then((legacyFinish) => {
|
||||
if (!legacyFinish || help.settled) return;
|
||||
finishHelp(help, {
|
||||
outcome: legacyFinish.outcome,
|
||||
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||
tab_id: help.tabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
finish({
|
||||
code: "protocol_error",
|
||||
message: `failed to show help overlay on tab ${tabId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
});
|
||||
.catch(() => {
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -514,7 +514,22 @@ export interface HelpTarget {
|
||||
selector?: string;
|
||||
}
|
||||
|
||||
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated";
|
||||
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated";
|
||||
|
||||
export interface HelpCompletionCondition {
|
||||
url_contains?: string;
|
||||
url_matches?: string;
|
||||
selector_exists?: string;
|
||||
selector_missing?: string;
|
||||
text_exists?: string;
|
||||
text_missing?: string;
|
||||
}
|
||||
|
||||
export interface HelpCompletionCriteria {
|
||||
any?: HelpCompletionCondition[];
|
||||
all?: HelpCompletionCondition[];
|
||||
stable_for_ms?: number;
|
||||
}
|
||||
|
||||
export interface ResolvedTarget {
|
||||
matched: boolean;
|
||||
@@ -528,11 +543,13 @@ export interface RequestHelpParams {
|
||||
prompt: string;
|
||||
title?: string;
|
||||
targets?: HelpTarget[];
|
||||
completion_criteria?: HelpCompletionCriteria;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface RequestHelpResult {
|
||||
outcome: HelpOutcome;
|
||||
completed_by?: "system";
|
||||
note?: string;
|
||||
tab_id: number;
|
||||
resolved_targets?: ResolvedTarget[];
|
||||
|
||||
Reference in New Issue
Block a user