Merge pull request #43 from Tencent/fix/early-control-return

fix(extension): fix early control return and preserve request-help handoff across tabs
LGTM
This commit is contained in:
Zhang GH
2026-07-24 15:27:41 +08:00
committed by GitHub
17 changed files with 1390 additions and 358 deletions
+170 -114
View File
@@ -15,6 +15,8 @@ export interface HelpRequestData {
prompt: string;
/** Custom overlay title; falls back to i18n when omitted. */
title?: string;
/** Full task UI on the subject tab; compact status UI on related tabs. */
displayMode?: "full" | "compact";
/** CSS selectors to scroll to + flash-highlight. */
selectors: string[];
onContinue: (note: string) => void;
@@ -174,6 +176,8 @@ function measure(selectors: string[]): Box[] {
export function HelpRequestOverlay({ request }: Props) {
const { t } = useTranslation("extension");
const isCompact = request?.displayMode === "compact";
const effectiveSelectors = isCompact ? [] : (request?.selectors ?? []);
const [note, setNote] = useState("");
const [collapsed, setCollapsed] = useState(false);
const [boxes, setBoxes] = useState<Box[]>([]);
@@ -207,8 +211,9 @@ export function HelpRequestOverlay({ request }: Props) {
// Scroll the first matched target into view once per distinct request id.
useLayoutEffect(() => {
if (!request) return;
if (isCompact) return;
if (lastScrolledIdRef.current === request.id) return;
for (const sel of request.selectors) {
for (const sel of effectiveSelectors) {
const el = querySelectorSafe(sel);
if (el) {
lastScrolledIdRef.current = request.id;
@@ -216,7 +221,7 @@ export function HelpRequestOverlay({ request }: Props) {
break;
}
}
}, [request]);
}, [request, isCompact, effectiveSelectors]);
// Keep highlight boxes aligned with the page as it scrolls / resizes.
useEffect(() => {
@@ -225,7 +230,7 @@ export function HelpRequestOverlay({ request }: Props) {
return;
}
const update = () => {
const nextBoxes = measure(request.selectors);
const nextBoxes = measure(effectiveSelectors);
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
};
update();
@@ -237,7 +242,7 @@ export function HelpRequestOverlay({ request }: Props) {
window.removeEventListener("resize", update);
window.clearInterval(interval);
};
}, [request]);
}, [request, effectiveSelectors]);
const onHeaderPointerDown = useCallback(
(e: ReactPointerEvent<HTMLDivElement>) => {
@@ -294,8 +299,12 @@ export function HelpRequestOverlay({ request }: Props) {
setPanelPos(null);
return;
}
if (isCompact) {
setPanelPos(null);
return;
}
if (userMovedRef.current) return;
const measured = boxes.length > 0 ? boxes : measure(request.selectors);
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
const anchor = unionBox(measured);
const banner = bannerRef.current;
if (!anchor || !banner) {
@@ -319,7 +328,7 @@ export function HelpRequestOverlay({ request }: Props) {
}
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
}, [request, boxes, collapsed]);
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
if (!request) return null;
@@ -342,6 +351,8 @@ export function HelpRequestOverlay({ request }: Props) {
z-index: 2147483647;
display: flex;
flex-direction: column;
justify-content: flex-start;
min-height: 0;
gap: 10px;
width: var(--bsk-help-banner-width);
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
@@ -359,16 +370,25 @@ export function HelpRequestOverlay({ request }: Props) {
.bsk-help-banner[data-collapsed="true"] {
gap: 0;
width: var(--bsk-help-banner-width);
padding: 10px 12px;
padding: 12px;
}
.bsk-help-banner[data-display-mode="compact"] {
width: min(360px, calc(100vw - ${VIEWPORT_MARGIN * 2}px));
padding: 12px 14px;
gap: 0;
border-radius: 12px;
}
.bsk-help-drag-strip {
flex-shrink: 0;
position: absolute;
top: 6px;
left: 0;
right: 0;
display: flex;
align-items: center;
justify-content: center;
height: 12px;
margin: -8px 0 -10px;
cursor: grab;
user-select: none;
touch-action: none;
@@ -395,15 +415,20 @@ export function HelpRequestOverlay({ request }: Props) {
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
margin: -6px 0 -8px;
top: 4px;
}
.bsk-help-header {
flex-shrink: 0;
display: flex;
align-items: center;
gap: 10px;
}
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header {
gap: 8px;
}
.bsk-help-title {
flex: 1 1 auto;
min-width: 0;
@@ -414,31 +439,34 @@ export function HelpRequestOverlay({ request }: Props) {
color: #111;
}
.bsk-help-compact-title {
flex: 1 1 auto;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-size: 13px;
font-weight: 500;
color: #374151;
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-title {
white-space: nowrap;
}
.bsk-help-header-actions {
flex-shrink: 0;
display: flex;
display: none;
align-items: center;
gap: 10px;
margin-left: auto;
overflow: hidden;
max-width: 0;
opacity: 0;
transform: translateX(8px);
pointer-events: none;
transition:
max-width var(--bsk-help-duration) var(--bsk-help-ease),
opacity 220ms var(--bsk-help-ease),
transform var(--bsk-help-duration) var(--bsk-help-ease);
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
display: flex;
max-width: 280px;
opacity: 1;
transform: translateX(0);
pointer-events: auto;
}
@@ -471,15 +499,13 @@ export function HelpRequestOverlay({ request }: Props) {
}
.bsk-help-body {
display: grid;
grid-template-rows: 1fr;
display: block;
flex-shrink: 0;
min-width: 0;
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
}
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
position: absolute;
grid-template-rows: 0fr;
width: 0;
height: 0;
overflow: hidden;
@@ -528,6 +554,7 @@ export function HelpRequestOverlay({ request }: Props) {
.bsk-help-footer-actions {
display: flex;
justify-content: flex-end;
flex-wrap: wrap;
gap: 10px;
}
@@ -539,7 +566,8 @@ export function HelpRequestOverlay({ request }: Props) {
font-size: 13px;
font-weight: 500;
color: #4b5563;
flex-shrink: 0;
min-width: 0;
white-space: normal;
}
.bsk-help-btn-continue {
@@ -550,16 +578,30 @@ export function HelpRequestOverlay({ request }: Props) {
font-size: 13px;
font-weight: 600;
color: #fff;
flex-shrink: 0;
min-width: 0;
white-space: normal;
}
.bsk-help-header-actions .bsk-help-btn-cancel,
.bsk-help-header-actions .bsk-help-btn-continue {
padding: 6px 12px;
white-space: nowrap;
}
.bsk-help-header-actions .bsk-help-btn-continue {
padding: 6px 14px;
max-width: 180px;
}
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions {
display: flex;
max-width: none;
}
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-cancel,
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-continue {
padding: 6px 10px;
font-size: 12px;
}
.bsk-help-footer-actions .bsk-help-btn-cancel {
@@ -584,29 +626,31 @@ export function HelpRequestOverlay({ request }: Props) {
}
`}</style>
{boxes.map((b, i) => (
<div
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
key={i}
data-slot="help-highlight"
style={{
position: "fixed",
top: b.top,
left: b.left,
width: b.width,
height: b.height,
borderRadius: 6,
zIndex: 2147483646,
pointerEvents: "none",
animation: "bsk-help-flash 1.2s ease-in-out infinite",
}}
/>
))}
{!isCompact &&
boxes.map((b, i) => (
<div
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
key={i}
data-slot="help-highlight"
style={{
position: "fixed",
top: b.top,
left: b.left,
width: b.width,
height: b.height,
borderRadius: 6,
zIndex: 2147483646,
pointerEvents: "none",
animation: "bsk-help-flash 1.2s ease-in-out infinite",
}}
/>
))}
<div
ref={bannerRef}
data-slot="help-request-banner"
data-collapsed={collapsed ? "true" : "false"}
data-display-mode={isCompact ? "compact" : "full"}
data-anchored={panelPos ? "true" : "false"}
data-dragging={dragging ? "true" : "false"}
data-placement={panelPos?.placement}
@@ -629,15 +673,17 @@ export function HelpRequestOverlay({ request }: Props) {
: undefined
}
>
<div
data-slot="help-drag-handle"
className="bsk-help-drag-strip"
role="img"
aria-label={t("helpRequest.dragHandle")}
onPointerDown={onHeaderPointerDown}
>
<span className="bsk-help-drag-pill" aria-hidden />
</div>
{!isCompact && (
<div
data-slot="help-drag-handle"
className="bsk-help-drag-strip"
role="img"
aria-label={t("helpRequest.dragHandle")}
onPointerDown={onHeaderPointerDown}
>
<span className="bsk-help-drag-pill" aria-hidden />
</div>
)}
<div className="bsk-help-header">
<img
@@ -645,77 +691,87 @@ export function HelpRequestOverlay({ request }: Props) {
alt="browser-skill"
style={{ width: 22, height: 22, borderRadius: 4 }}
/>
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
{isCompact ? (
<span className="bsk-help-compact-title">{t("helpRequest.compactStatus")}</span>
) : (
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
)}
{!isCompact && (
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
<button
type="button"
data-slot={collapsed ? "help-cancel-button" : undefined}
className="bsk-help-btn-cancel"
tabIndex={collapsed ? 0 : -1}
onClick={() => request.onCancel()}
>
{t("helpRequest.cancel")}
</button>
<button
type="button"
data-slot={collapsed ? "help-continue-button" : undefined}
className="bsk-help-btn-continue"
tabIndex={collapsed ? 0 : -1}
onClick={() => request.onContinue(note)}
>
{t("helpRequest.continue")}
</button>
</div>
)}
{!isCompact && (
<button
type="button"
data-slot={collapsed ? "help-cancel-button" : undefined}
className="bsk-help-btn-cancel"
tabIndex={collapsed ? 0 : -1}
onClick={() => request.onCancel()}
data-slot="help-collapse-toggle"
className="bsk-help-collapse-toggle"
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
aria-expanded={!collapsed}
onClick={() => setCollapsed((c) => !c)}
>
{t("helpRequest.cancel")}
<span className="bsk-help-collapse-icon">
<RiArrowDownSLine size={20} aria-hidden />
</span>
</button>
<button
type="button"
data-slot={collapsed ? "help-continue-button" : undefined}
className="bsk-help-btn-continue"
tabIndex={collapsed ? 0 : -1}
onClick={() => request.onContinue(note)}
>
{t("helpRequest.continue")}
</button>
</div>
<button
type="button"
data-slot="help-collapse-toggle"
className="bsk-help-collapse-toggle"
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
aria-expanded={!collapsed}
onClick={() => setCollapsed((c) => !c)}
>
<span className="bsk-help-collapse-icon">
<RiArrowDownSLine size={20} aria-hidden />
</span>
</button>
)}
</div>
<div className="bsk-help-body" aria-hidden={collapsed}>
<div className="bsk-help-body-inner">
<div className="bsk-help-body-content">
<p className="bsk-help-prompt">{request.prompt}</p>
<input
type="text"
className="bsk-help-note-input"
aria-label={t("helpRequest.noteLabel")}
placeholder={t("helpRequest.notePlaceholder")}
value={note}
tabIndex={collapsed ? -1 : 0}
onChange={(e) => setNote(e.target.value)}
/>
<div className="bsk-help-footer-actions">
<button
type="button"
data-slot={collapsed ? undefined : "help-cancel-button"}
className="bsk-help-btn-cancel"
{!isCompact && (
<div className="bsk-help-body" aria-hidden={collapsed}>
<div className="bsk-help-body-inner">
<div className="bsk-help-body-content">
<p className="bsk-help-prompt">{request.prompt}</p>
<input
type="text"
className="bsk-help-note-input"
aria-label={t("helpRequest.noteLabel")}
placeholder={t("helpRequest.notePlaceholder")}
value={note}
tabIndex={collapsed ? -1 : 0}
onClick={() => request.onCancel()}
>
{t("helpRequest.cancel")}
</button>
<button
type="button"
data-slot={collapsed ? undefined : "help-continue-button"}
className="bsk-help-btn-continue"
tabIndex={collapsed ? -1 : 0}
onClick={() => request.onContinue(note)}
>
{t("helpRequest.continue")}
</button>
onChange={(e) => setNote(e.target.value)}
/>
<div className="bsk-help-footer-actions">
<button
type="button"
data-slot={collapsed ? undefined : "help-cancel-button"}
className="bsk-help-btn-cancel"
tabIndex={collapsed ? -1 : 0}
onClick={() => request.onCancel()}
>
{t("helpRequest.cancel")}
</button>
<button
type="button"
data-slot={collapsed ? undefined : "help-continue-button"}
className="bsk-help-btn-continue"
tabIndex={collapsed ? -1 : 0}
onClick={() => request.onContinue(note)}
>
{t("helpRequest.continue")}
</button>
</div>
</div>
</div>
</div>
</div>
)}
</div>
</>
);
@@ -38,6 +38,31 @@ describe("HelpRequestOverlay", () => {
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
});
it("renders a compact status without full request controls", () => {
const el = document.createElement("div");
el.id = "login";
el.getBoundingClientRect = () =>
({ top: 10, left: 10, width: 100, height: 40, right: 110, bottom: 50 }) as DOMRect;
document.body.append(el);
const { container } = renderOverlay(
baseRequest({
displayMode: "compact",
selectors: ["#login"],
}),
);
expect(screen.getByText(i18n.t("helpRequest.compactStatus", { ns: "extension" }))).toBeTruthy();
expect(screen.queryByText("Please complete the captcha")).toBeNull();
expect(screen.queryByRole("textbox")).toBeNull();
expect(screen.queryByLabelText(i18n.t("helpRequest.collapse", { ns: "extension" }))).toBeNull();
expect(container.querySelector("[data-slot='help-continue-button']")).toBeNull();
expect(container.querySelector("[data-slot='help-cancel-button']")).toBeNull();
expect(document.querySelectorAll("[data-slot='help-highlight']").length).toBe(0);
el.remove();
});
it("renders a custom title when provided", () => {
renderOverlay(baseRequest({ title: "Verify your identity" }));
expect(screen.getByText("Verify your identity")).toBeTruthy();
@@ -204,6 +229,15 @@ describe("HelpRequestOverlay", () => {
expect(styles).toContain("width: 0");
});
it("uses natural body height in the expanded layout", () => {
const { container } = renderOverlay(baseRequest());
const styles = overlayStyles(container);
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
});
it("keeps the same banner width when collapsed", () => {
const { container } = renderOverlay(baseRequest());
const styles = overlayStyles(container);
+73 -28
View File
@@ -14,10 +14,14 @@ import {
type RecordCaptureController,
} from "@/content/record-capture";
import {
HELP_RESPONSE,
HELP_ACK,
HELP_FINISH,
HELP_QUERY,
type HelpAckMessage,
type HelpCancelMessage,
type HelpFinishMessage,
type HelpQueryResponse,
type HelpRequestMessage,
type HelpResponseMessage,
isHelpCancelMessage,
isHelpRequestMessage,
} from "@/lib/help-bridge";
@@ -55,8 +59,6 @@ export default defineContentScript({
if (window.top !== window) return;
const overlays = new OverlayController();
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
null;
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
@@ -116,8 +118,7 @@ export default defineContentScript({
function resetAgentOverlayState(sessionId: string) {
const previousHelp = overlays.resetAgentOverlays(sessionId);
if (previousHelp) {
activeHelpRespond?.("cancelled");
activeHelpRespond = null;
void sendHelpFinish(previousHelp.id, "cancelled");
}
recordCapture?.dispose();
recordCapture = null;
@@ -156,7 +157,7 @@ export default defineContentScript({
| OverlayAgentOverlayResetMessage
| OverlayAutomationBypassMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
) => {
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
@@ -220,41 +221,30 @@ export default defineContentScript({
if (isHelpCancelMessage(message)) {
const state = overlays.snapshot();
if (state.activeHelp && state.activeHelp.id === message.requestId) {
activeHelpRespond?.("cancelled");
overlays.clearAgentHelpRequest(message.requestId);
renderOverlay();
}
return false;
}
if (isHelpRequestMessage(message)) {
const helpMsg = message as HelpRequestMessage;
let responded = false;
const respond = (outcome: "continued" | "cancelled", note?: string) => {
if (responded) return;
responded = true;
const reply: HelpResponseMessage = {
type: HELP_RESPONSE,
outcome,
...(note ? { note } : {}),
};
sendResponse(reply);
activeHelpRespond = null;
overlays.clearAgentHelpRequest(helpMsg.requestId);
renderOverlay();
};
const previousHelp = overlays.setAgentHelpRequest({
id: helpMsg.requestId,
prompt: helpMsg.prompt,
...(helpMsg.title ? { title: helpMsg.title } : {}),
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
selectors: helpMsg.selectors,
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
onCancel: () => respond("cancelled"),
onContinue: (note: string) =>
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
});
if (previousHelp) {
activeHelpRespond?.("cancelled");
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
activeHelpRespond = respond;
renderOverlay();
return true; // async sendResponse
sendResponse({ type: HELP_ACK, ok: true });
return false;
}
if (message.type === "borrow-request") {
@@ -282,9 +272,60 @@ export default defineContentScript({
return false;
};
async function sendHelpFinish(
requestId: string,
outcome: "continued" | "cancelled",
note?: string,
): Promise<void> {
const msg: HelpFinishMessage = {
type: HELP_FINISH,
requestId,
outcome,
...(note ? { note } : {}),
};
overlays.clearAgentHelpRequest(requestId);
renderOverlay();
await chrome.runtime.sendMessage(msg).catch((err) => {
console.debug("[bsk overlay] help finish failed", err);
});
}
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
overlays.setAgentHelpRequest({
id: helpMsg.requestId,
prompt: helpMsg.prompt,
...(helpMsg.title ? { title: helpMsg.title } : {}),
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
selectors: helpMsg.selectors,
onContinue: (note: string) =>
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
});
}
async function queryActiveHelpWithRetry(): Promise<boolean> {
for (let attempt = 0; attempt < 6; attempt += 1) {
try {
const helpQuery = (await chrome.runtime.sendMessage({
type: HELP_QUERY,
})) as HelpQueryResponse | undefined;
if (helpQuery?.active && helpQuery.request) {
mountHelpRequest(helpQuery.request);
renderOverlay();
return true;
}
} catch (err) {
console.debug("[bsk overlay] help query failed", err);
}
await new Promise((resolve) => window.setTimeout(resolve, 150));
}
return false;
}
async function syncAgentOverlay(): Promise<void> {
if (!(await anySessionLive())) return;
try {
const helpActive = await queryActiveHelpWithRetry();
const reply = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_WHO_AM_I,
})) as OverlayWhoAmIResponse | undefined;
@@ -319,6 +360,10 @@ export default defineContentScript({
});
}
if (!helpActive && overlays.snapshot().activeHelp === null) {
void queryActiveHelpWithRetry();
}
overlays.activateAgentSession(reply.sessionId);
renderOverlay();
} catch (err) {
@@ -1,10 +1,17 @@
import { describe, expect, it } from "vitest";
import {
HELP_ACK,
HELP_CANCEL,
HELP_FINISH,
HELP_QUERY,
HELP_REQUEST,
HELP_RESPONSE,
isHelpAckMessage,
isHelpCancelMessage,
isHelpFinishMessage,
isHelpQueryMessage,
isHelpRequestMessage,
isHelpResponseMessage,
} from "../help-bridge";
describe("help-bridge", () => {
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
expect(HELP_REQUEST).toBe("bsk-help-request");
expect(HELP_RESPONSE).toBe("bsk-help-response");
expect(HELP_CANCEL).toBe("bsk-help-cancel");
expect(HELP_ACK).toBe("bsk-help-ack");
expect(HELP_QUERY).toBe("bsk-help-query");
expect(HELP_FINISH).toBe("bsk-help-finish");
});
it("type-guards a help-request message", () => {
@@ -20,6 +30,7 @@ describe("help-bridge", () => {
type: HELP_REQUEST,
requestId: "r1",
prompt: "log in",
displayMode: "compact",
selectors: ["#login"],
timeoutMs: 1000,
}),
@@ -55,4 +66,18 @@ describe("help-bridge", () => {
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
expect(isHelpCancelMessage(null)).toBe(false);
});
it("type-guards lifecycle messages", () => {
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
true,
);
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
true,
);
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
false,
);
});
});
+61 -1
View File
@@ -6,12 +6,15 @@
* Mirrors the borrow-confirmation message style (see
* `@/tools/borrow-confirmation`). The background asks a specific tab to
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
* the content script replies once the user clicks Continue / Cancel.
* the content script acks display, then later reports Done / Cancel.
*/
export const HELP_REQUEST = "bsk-help-request";
export const HELP_RESPONSE = "bsk-help-response";
export const HELP_CANCEL = "bsk-help-cancel";
export const HELP_ACK = "bsk-help-ack";
export const HELP_QUERY = "bsk-help-query";
export const HELP_FINISH = "bsk-help-finish";
export interface HelpRequestMessage {
type: typeof HELP_REQUEST;
@@ -19,11 +22,18 @@ export interface HelpRequestMessage {
prompt: string;
/** Custom overlay title; omitted when the extension should use its default. */
title?: string;
/** Full task UI on the subject tab; compact status UI on related tabs. */
displayMode?: "full" | "compact";
/** CSS selectors to scroll to + flash-highlight (may be empty). */
selectors: string[];
timeoutMs: number;
}
export interface HelpAckMessage {
type: typeof HELP_ACK;
ok: true;
}
export interface HelpResponseMessage {
type: typeof HELP_RESPONSE;
outcome: "continued" | "cancelled";
@@ -35,6 +45,22 @@ export interface HelpCancelMessage {
requestId: string;
}
export interface HelpQueryMessage {
type: typeof HELP_QUERY;
}
export interface HelpQueryResponse {
active: boolean;
request?: Omit<HelpRequestMessage, "type">;
}
export interface HelpFinishMessage {
type: typeof HELP_FINISH;
requestId: string;
outcome: "continued" | "cancelled";
note?: string;
}
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
if (typeof msg !== "object" || msg === null) {
return false;
@@ -45,6 +71,7 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
typeof m.requestId === "string" &&
typeof m.prompt === "string" &&
(m.title === undefined || typeof m.title === "string") &&
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
Array.isArray(m.selectors) &&
m.selectors.every((selector) => typeof selector === "string") &&
typeof m.timeoutMs === "number"
@@ -58,3 +85,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
const m = msg as Record<string, unknown>;
return m.type === HELP_CANCEL && typeof m.requestId === "string";
}
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return (
m.type === HELP_RESPONSE &&
(m.outcome === "continued" || m.outcome === "cancelled") &&
(m.note === undefined || typeof m.note === "string")
);
}
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === HELP_ACK && m.ok === true;
}
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === HELP_QUERY;
}
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return (
m.type === HELP_FINISH &&
typeof m.requestId === "string" &&
(m.outcome === "continued" || m.outcome === "cancelled") &&
(m.note === undefined || typeof m.note === "string")
);
}
@@ -1,7 +1,53 @@
import { describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { RequestHelpParams } from "@/transport/types";
import { handleRequestHelp, type RequestHelpDeps } from "../human-loop";
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
function chromeEvent<T extends (...args: never[]) => unknown>() {
const listeners = new Set<T>();
return {
addListener: vi.fn((listener: T) => {
listeners.add(listener);
}),
removeListener: vi.fn((listener: T) => {
listeners.delete(listener);
}),
emit: (...args: Parameters<T>) => {
for (const listener of [...listeners]) listener(...args);
},
};
}
function installHelpLifecycleChrome() {
const runtimeOnMessage =
chromeEvent<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => unknown
>();
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
const tabsOnCreated = chromeEvent<(tab: chrome.tabs.Tab) => unknown>();
const tabsOnUpdated =
chromeEvent<(tabId: number, changeInfo: chrome.tabs.TabChangeInfo) => unknown>();
const webNavigationOnCompleted =
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
vi.stubGlobal("chrome", {
runtime: { onMessage: runtimeOnMessage },
tabs: { onActivated: tabsOnActivated, onCreated: tabsOnCreated, onUpdated: tabsOnUpdated },
webNavigation: { onCompleted: webNavigationOnCompleted },
});
return {
runtimeOnMessage,
tabsOnActivated,
tabsOnCreated,
tabsOnUpdated,
webNavigationOnCompleted,
};
}
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
const mgr = {
@@ -27,14 +73,19 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
windows: { update: vi.fn(async () => ({}) as never) },
activateTab: vi.fn(async () => {}),
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
watchTabNavigation: () => () => {},
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
notifications: null,
autoAttachLifecycle: false,
...over,
};
}
describe("handleRequestHelp", () => {
afterEach(() => {
resetHelpLifecycleForTests();
vi.unstubAllGlobals();
});
it("rejects unknown session", async () => {
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
@@ -78,22 +129,29 @@ describe("handleRequestHelp", () => {
expect(sentMsg.title).toBeUndefined();
});
it("returns navigated when the tab navigates during the wait", async () => {
const unwatch = vi.fn();
it("does not complete merely because the tab navigates during the wait", async () => {
vi.useFakeTimers();
const deps = baseDeps({
sendToTab: vi.fn(() => new Promise(() => {})),
watchTabNavigation: vi.fn((_tabId, cb) => {
queueMicrotask(() => cb());
return unwatch;
}),
});
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
deps,
);
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
expect(unwatch).toHaveBeenCalled();
try {
const pending = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 10 }),
deps,
);
await vi.advanceTimersByTimeAsync(5);
let settled = false;
void pending.then(() => {
settled = true;
});
await Promise.resolve();
expect(settled).toBe(false);
await vi.advanceTimersByTimeAsync(10);
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
} finally {
vi.useRealTimers();
}
});
it("returns timed_out when the wait expires", async () => {
@@ -115,6 +173,160 @@ describe("handleRequestHelp", () => {
}
});
it("returns completed when explicit completion criteria match", async () => {
const deps = baseDeps({
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
cdp: {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: true } };
return {};
}),
} as unknown as RequestHelpDeps["cdp"],
});
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({
tab_id: 5,
completion_criteria: {
any: [{ selector_exists: "#account-menu" }],
stable_for_ms: 0,
},
}),
deps,
);
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
});
it("keeps user control active on new tabs without moving completion off the primary tab", async () => {
vi.useFakeTimers();
const chromeEvents = installHelpLifecycleChrome();
const sendToTab = vi.fn(async () => ({ type: "bsk-help-ack", ok: true }));
const deps = baseDeps({
autoAttachLifecycle: undefined,
sendToTab,
tabsApi: {
get: vi.fn(async (tabId: number) => ({
id: tabId,
windowId: 99,
active: tabId === 6,
url: tabId === 6 ? "https://app.example/reset/success" : "https://app.example/login",
})) as never,
query: vi.fn(async () => [{ id: 5, windowId: 99, active: true }] as never),
},
});
try {
const pending = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({
tab_id: 5,
timeout_ms: 1_000,
completion_criteria: {
any: [{ url_contains: "/reset/success" }],
stable_for_ms: 0,
},
}),
deps,
);
await vi.waitFor(() => expect(sendToTab).toHaveBeenCalledWith(5, expect.anything()));
chromeEvents.tabsOnActivated.emit({ tabId: 6, windowId: 99 });
await vi.advanceTimersByTimeAsync(200);
await vi.waitFor(() =>
expect(sendToTab).toHaveBeenCalledWith(
6,
expect.objectContaining({
type: "bsk-help-request",
displayMode: "compact",
selectors: [],
}),
),
);
await vi.advanceTimersByTimeAsync(1_000);
await expect(pending).resolves.toMatchObject({
outcome: "timed_out",
tab_id: 5,
});
expect(sendToTab).toHaveBeenCalledWith(
6,
expect.objectContaining({ type: "bsk-help-cancel" }),
);
} finally {
vi.useRealTimers();
}
});
it("lets same-window tabs query active help after content-script load", async () => {
const chromeEvents = installHelpLifecycleChrome();
const ac = new AbortController();
const cdpSend = vi.fn(async () => ({ root: { nodeId: 1 }, nodeIds: [] }));
const deps = baseDeps({
autoAttachLifecycle: undefined,
signal: ac.signal,
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
cdp: {
send: cdpSend,
} as unknown as RequestHelpDeps["cdp"],
});
const pending = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 60_000 }),
deps,
);
await vi.waitFor(() => expect(deps.sendToTab).toHaveBeenCalledWith(5, expect.anything()));
const sameWindowResponse = vi.fn();
chromeEvents.runtimeOnMessage.emit(
{ type: "bsk-help-query" },
{ tab: { id: 6 } as chrome.tabs.Tab },
sameWindowResponse,
);
await vi.waitFor(() =>
expect(sameWindowResponse).toHaveBeenCalledWith({
active: true,
request: expect.objectContaining({
requestId: expect.any(String),
prompt: "log in",
displayMode: "compact",
selectors: [],
}),
}),
);
const subjectResponse = vi.fn();
chromeEvents.runtimeOnMessage.emit(
{ type: "bsk-help-query" },
{ tab: { id: 5 } as chrome.tabs.Tab },
subjectResponse,
);
await vi.waitFor(() =>
expect(subjectResponse).toHaveBeenCalledWith({
active: true,
request: expect.objectContaining({
requestId: expect.any(String),
prompt: "log in",
displayMode: "full",
}),
}),
);
ac.abort();
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
expect(cdpSend).toHaveBeenCalledWith(
5,
"DOM.querySelectorAll",
expect.objectContaining({ selector: "[data-bsk-help]" }),
);
expect(cdpSend).toHaveBeenCalledWith(
6,
"DOM.querySelectorAll",
expect.objectContaining({ selector: "[data-bsk-help]" }),
);
});
it("tags ref targets via CDP and reports them matched", async () => {
const mgr = {
get: (id: string) =>
@@ -190,38 +402,40 @@ describe("handleRequestHelp", () => {
expect("code" in res).toBe(false);
});
it("returns protocol_error when sendToTab rejects", async () => {
it("keeps waiting when the initial overlay delivery rejects", async () => {
vi.useFakeTimers();
const deps = baseDeps({
sendToTab: vi.fn(async () => {
throw new Error("no receiver");
}),
});
const res = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
deps,
);
expect("code" in res && res.code).toBe("protocol_error");
try {
const res = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 10 }),
deps,
);
await vi.advanceTimersByTimeAsync(20);
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
} finally {
vi.useRealTimers();
}
});
it("returns protocol_error for a malformed help response", async () => {
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
const res1 = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
undefinedReply,
);
expect("code" in res1 && res1.code).toBe("protocol_error");
const badOutcome = baseDeps({
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
});
const res2 = await handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5 }),
badOutcome,
);
expect("code" in res2 && res2.code).toBe("protocol_error");
it("keeps waiting for explicit completion after a malformed help response", async () => {
vi.useFakeTimers();
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
try {
const res = handleRequestHelp(
fakeManager("abcd", 99, 5),
baseParams({ tab_id: 5, timeout_ms: 10 }),
deps,
);
await vi.advanceTimersByTimeAsync(20);
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
} finally {
vi.useRealTimers();
}
});
it("reports selector match status from CDP", async () => {
+1 -2
View File
@@ -29,7 +29,7 @@ import type {
import { isRequestFrame } from "@/transport/types";
import { handleConsole } from "./console";
import { handleEvaluate } from "./evaluate";
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
import {
handleNavigate,
@@ -394,7 +394,6 @@ export class ToolDispatcher {
await chrome.tabs.update(tabId, { active: true });
},
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
watchTabNavigation: defaultWatchTabNavigation,
...(this.cdp ? { cdp: this.cdp } : {}),
notifications: makeHelpNotifications(),
notificationCopy: this.helpNotificationCopy?.(),
+552 -154
View File
@@ -1,21 +1,24 @@
// `tool.request_help` — pause and ask the human to act on a tab.
//
// Brings the target tab to the foreground, resolves `ref` targets to a
// temporary attribute selector via CDP, asks the content script to enter
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
// until the user clicks Continue / Cancel, the wait times out, or the
// daemon cancels the RPC.
// `tool.request_help` — pause automation and let the human control a tab.
import {
HELP_ACK,
HELP_CANCEL,
HELP_FINISH,
HELP_QUERY,
HELP_REQUEST,
HELP_RESPONSE,
type HelpCancelMessage,
type HelpFinishMessage,
type HelpQueryResponse,
type HelpRequestMessage,
type HelpResponseMessage,
isHelpAckMessage,
isHelpFinishMessage,
isHelpQueryMessage,
isHelpResponseMessage,
} from "@/lib/help-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type {
HelpCompletionCondition,
HelpCompletionCriteria,
HelpTarget,
RequestHelpParams,
RequestHelpResult,
@@ -34,53 +37,59 @@ import {
import { lookupSnapshotRef } from "./snapshot-ref";
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
/** Attribute the overlay highlights for resolved `ref` targets. */
const HELP_ATTR = "data-bsk-help";
const HELP_SEND_RETRIES = 3;
const HELP_SEND_RETRY_DELAY_MS = 350;
const HELP_REARM_DEBOUNCE_MS = 150;
const HELP_REARM_MAX_ATTEMPTS = 12;
const HELP_REARM_RETRY_DELAY_MS = 400;
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
const COMPLETION_POLL_MS = 500;
export interface RequestHelpNotifications {
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
clear(id: string): Promise<boolean>;
}
export type TabNavigationUnsubscribe = () => void;
export interface RequestHelpDeps {
tabsApi: ChromeTabsApi;
windows: {
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
};
/** Activate a tab inside its window. */
activateTab(tabId: number): Promise<void>;
/** Send the help-request to the tab's content script and await reply. */
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
/**
* Watch for navigation on `tabId` (full page load or SPA URL change).
* Invokes `onNavigated` once; returns unsubscribe.
*/
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
cdp?: CdpRunner;
/** Pass `null` to skip OS notifications (tests). */
notifications: RequestHelpNotifications | null;
notificationCopy?: { title: string; body: string };
signal?: AbortSignal;
autoAttachLifecycle?: boolean;
}
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
export function defaultWatchTabNavigation(
tabId: number,
onNavigated: () => void,
): TabNavigationUnsubscribe {
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
if (updatedTabId !== tabId) return;
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
chrome.tabs.onUpdated.removeListener(listener);
onNavigated();
}
};
chrome.tabs.onUpdated.addListener(listener);
return () => chrome.tabs.onUpdated.removeListener(listener);
interface ActiveHelpRequest {
ctx: SessionContext;
requestId: string;
primaryTabId: number;
overlayTabIds: Set<number>;
prompt: string;
title?: string;
targets: HelpTarget[];
selectors: string[];
timeoutMs: number;
notificationId: string;
resolvedTargets?: ResolvedTarget[];
completionCriteria?: HelpCompletionCriteria;
deps: RequestHelpDeps;
settled: boolean;
resolve: (value: RequestHelpResult | RpcError) => void;
timeoutTimer: ReturnType<typeof setTimeout>;
completionTimer: ReturnType<typeof setInterval> | null;
completionMatchedSince: number | null;
abortHandler: (() => void) | null;
}
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
let defaultDeps: RequestHelpDeps | null = null;
function getDefaultDeps(): RequestHelpDeps {
if (!defaultDeps) {
@@ -91,7 +100,6 @@ function getDefaultDeps(): RequestHelpDeps {
await chrome.tabs.update(tabId, { active: true });
},
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
watchTabNavigation: defaultWatchTabNavigation,
notifications: {
create: (id, opts) =>
new Promise((resolve, reject) =>
@@ -113,11 +121,50 @@ function makeRequestId(tabId: number): string {
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
}
/**
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
* content overlay can highlight it with a plain attribute selector.
* Returns the selector on success, or null when the ref can't resolve.
*/
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequestMessage {
const isPrimaryTab = tabId === help.primaryTabId;
return {
type: HELP_REQUEST,
requestId: help.requestId,
prompt: help.prompt,
...(help.title ? { title: help.title } : {}),
displayMode: isPrimaryTab ? "full" : "compact",
selectors: isPrimaryTab ? help.selectors : [],
timeoutMs: help.timeoutMs,
};
}
async function sendHelpRequestWithAck(
tabId: number,
msg: HelpRequestMessage,
deps: RequestHelpDeps,
): Promise<HelpFinishMessage | null> {
let lastError: unknown;
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
try {
const response = await deps.sendToTab(tabId, msg);
if (isHelpAckMessage(response)) return null;
if (isHelpResponseMessage(response)) {
return {
type: HELP_FINISH,
requestId: msg.requestId,
outcome: response.outcome,
...(response.note ? { note: response.note } : {}),
};
}
lastError = new Error("content script did not ack HELP_REQUEST");
} catch (err) {
lastError = err;
}
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
}
throw lastError ?? new Error("failed to show help overlay");
}
async function tagRefTarget(
cdp: CdpRunner,
tabId: number,
@@ -145,7 +192,6 @@ async function tagRefTarget(
}
}
/** Returns true when `selector` matches a live element in the page. */
async function selectorExists(
cdp: CdpRunner,
tabId: number,
@@ -163,7 +209,6 @@ async function selectorExists(
}
}
/** Remove every help attribute we added (best-effort cleanup). */
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
if (!cdp) return;
try {
@@ -180,34 +225,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
}
} catch {
// best-effort
// Best-effort cleanup.
}
}
export async function handleRequestHelp(
manager: SessionManager,
params: RequestHelpParams,
deps: RequestHelpDeps = getDefaultDeps(),
): Promise<RequestHelpResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "request_help");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
if (!params.prompt || typeof params.prompt !== "string") {
return { code: "invalid_params", message: "request_help requires a prompt" };
}
if (deps.signal?.aborted) {
return { code: "cancelled", message: "request_help aborted" };
}
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "request_help");
if (denied) return denied;
const tabId = target.tabId;
// Resolve targets → selectors. `selector` passes through; `ref` is
// tagged with an attribute via CDP and reported as its attr selector.
const targets = params.targets ?? [];
async function resolveHelpTargets(
ctx: ReturnType<SessionManager["get"]>,
tabId: number,
targets: HelpTarget[],
deps: RequestHelpDeps,
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
const selectors: string[] = [];
const resolved: ResolvedTarget[] = [];
let rootNodeId: number | undefined;
@@ -221,22 +248,20 @@ export async function handleRequestHelp(
rootNodeId = undefined;
}
}
for (let i = 0; i < targets.length; i++) {
const tgt = targets[i] as HelpTarget;
for (let i = 0; i < targets.length; i += 1) {
const tgt = targets[i];
if (tgt.selector) {
selectors.push(tgt.selector);
let matched: boolean;
if (deps.cdp) {
matched =
rootNodeId !== undefined
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
: false;
} else {
matched = true;
}
const matched =
deps.cdp && rootNodeId !== undefined
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
: deps.cdp
? false
: true;
resolved.push({ matched, selector: tgt.selector });
} else if (tgt.ref) {
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
const backendNodeId = looked?.backendNodeId ?? null;
let sel: string | null = null;
if (backendNodeId !== null && deps.cdp) {
@@ -247,7 +272,407 @@ export async function handleRequestHelp(
}
}
// Bring the tab to the foreground.
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
}
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
await clearRefTags(help.deps.cdp, help.primaryTabId);
const { selectors, resolvedTargets } = await resolveHelpTargets(
help.ctx,
help.primaryTabId,
help.targets,
help.deps,
);
help.selectors = selectors;
help.resolvedTargets = resolvedTargets;
}
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
if (help.deps.notifications) {
await help.deps.notifications.clear(help.notificationId).catch(() => {});
}
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
await Promise.all(
[...tabsToCancel].map((tabId) =>
Promise.all([
clearRefTags(help.deps.cdp, tabId),
help.deps
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
.catch(() => {}),
]),
),
);
}
function finishHelp(
help: ActiveHelpRequest,
value: RequestHelpResult | RpcError,
notifyContent = true,
): void {
if (help.settled) return;
help.settled = true;
activeHelpRequests.delete(help.requestId);
clearTimeout(help.timeoutTimer);
if (help.completionTimer) clearInterval(help.completionTimer);
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
clearRearmTimer(help.primaryTabId);
for (const tabId of help.overlayTabIds) clearRearmTimer(tabId);
if (notifyContent) void cleanupHelp(help);
help.resolve(value);
}
function findHelpByKnownTabId(tabId: number): ActiveHelpRequest | null {
for (const help of activeHelpRequests.values()) {
if (!help.settled && (help.primaryTabId === tabId || help.overlayTabIds.has(tabId))) {
return help;
}
}
return null;
}
async function findHelpForTab(
tabId: number,
deps: RequestHelpDeps,
): Promise<ActiveHelpRequest | null> {
const direct = findHelpByKnownTabId(tabId);
if (direct) return direct;
try {
const tab = await deps.tabsApi.get(tabId);
const windowId = tab.windowId;
if (typeof windowId !== "number") return null;
for (const help of activeHelpRequests.values()) {
if (!help.settled && help.ctx.agentWindowId === windowId) return help;
}
} catch {
return null;
}
return null;
}
function clearRearmTimer(tabId: number): void {
const timer = helpRearmTimers.get(tabId);
if (!timer) return;
clearTimeout(timer);
helpRearmTimers.delete(tabId);
}
async function sendCurrentHelpOverlay(help: ActiveHelpRequest): Promise<HelpFinishMessage | null> {
const legacyFinish = await sendHelpRequestWithAck(
help.primaryTabId,
helpRequestMessage(help, help.primaryTabId),
help.deps,
);
help.overlayTabIds.add(help.primaryTabId);
return legacyFinish;
}
async function refreshAndSendHelpOverlay(
help: ActiveHelpRequest,
tabId: number,
): Promise<HelpFinishMessage | null> {
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
const legacyFinish = await sendHelpRequestWithAck(
tabId,
helpRequestMessage(help, tabId),
help.deps,
);
help.overlayTabIds.add(tabId);
return legacyFinish;
}
async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise<boolean> {
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
try {
const legacyFinish = await refreshAndSendHelpOverlay(help, tabId);
if (legacyFinish && !help.settled) {
finishHelp(help, {
outcome: legacyFinish.outcome,
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
tab_id: help.primaryTabId,
resolved_targets: help.resolvedTargets,
});
}
return true;
} catch {
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
}
}
return false;
}
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
const existing = helpRearmTimers.get(tabId);
if (existing) clearTimeout(existing);
helpRearmTimers.set(
tabId,
setTimeout(() => {
helpRearmTimers.delete(tabId);
void (async () => {
const help = await findHelpForTab(tabId, deps);
if (!help) return;
await rearmHelp(help, tabId);
})();
}, HELP_REARM_DEBOUNCE_MS),
);
}
let detachHelpObservation: (() => void) | null = null;
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
if (detachHelpObservation) return;
const detachRuntime = attachHelpRuntimeListener(deps);
const detachTabs = attachHelpTabListener(deps);
const detachNav = attachHelpNavigationListener(deps);
detachHelpObservation = () => {
detachRuntime();
detachTabs();
detachNav();
};
}
export function resetHelpLifecycleForTests(): void {
detachHelpObservation?.();
detachHelpObservation = null;
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
helpRearmTimers.clear();
for (const help of activeHelpRequests.values()) {
clearTimeout(help.timeoutTimer);
if (help.completionTimer) clearInterval(help.completionTimer);
}
activeHelpRequests.clear();
}
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
const listener = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: HelpQueryResponse) => void,
) => {
if (isHelpFinishMessage(message)) {
void finishHelpFromContent(message, sender, deps);
return false;
}
if (isHelpQueryMessage(message)) {
const tabId = sender.tab?.id;
if (tabId === undefined) {
sendResponse({ active: false });
return false;
}
void (async () => {
const help = await findHelpForTab(tabId, deps);
if (!help) {
sendResponse({ active: false });
return;
}
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
help.overlayTabIds.add(tabId);
sendResponse({
active: true,
request: {
requestId: help.requestId,
prompt: help.prompt,
...(help.title ? { title: help.title } : {}),
displayMode: tabId === help.primaryTabId ? "full" : "compact",
selectors: tabId === help.primaryTabId ? help.selectors : [],
timeoutMs: help.timeoutMs,
},
});
})();
return true;
}
return false;
};
chrome.runtime.onMessage.addListener(listener);
return () => chrome.runtime.onMessage.removeListener(listener);
}
async function finishHelpFromContent(
message: HelpFinishMessage,
sender: chrome.runtime.MessageSender,
deps: RequestHelpDeps,
): Promise<void> {
const help = activeHelpRequests.get(message.requestId);
if (!help || help.settled) return;
const tabId = sender.tab?.id;
if (tabId !== undefined) {
const match = await findHelpForTab(tabId, deps);
if (match !== help) return;
}
finishHelp(help, {
outcome: message.outcome,
...(message.note ? { note: message.note } : {}),
tab_id: help.primaryTabId,
resolved_targets: help.resolvedTargets,
});
}
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
const onCreated = (tab: chrome.tabs.Tab) => {
if (typeof tab.id !== "number") return;
scheduleRearmForTab(tab.id, deps);
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
scheduleRearmForTab(activeInfo.tabId, deps);
};
chrome.tabs.onCreated?.addListener(onCreated);
chrome.tabs.onActivated?.addListener(onActivated);
return () => {
chrome.tabs.onCreated?.removeListener(onCreated);
chrome.tabs.onActivated?.removeListener(onActivated);
};
}
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
if (chrome.webNavigation?.onCompleted) {
const completedListener = (
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
) => {
if (details.frameId !== 0) return;
onMainFrameComplete(details.tabId);
};
chrome.webNavigation.onCompleted.addListener(completedListener);
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
}
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
if (info.status !== "complete") return;
onMainFrameComplete(tabId);
};
chrome.tabs.onUpdated?.addListener(listener);
return () => chrome.tabs.onUpdated?.removeListener(listener);
}
async function evaluateCompletionCondition(
condition: HelpCompletionCondition,
help: ActiveHelpRequest,
): Promise<boolean> {
if (condition.url_contains || condition.url_matches) {
let url = "";
try {
const tab = await help.deps.tabsApi.get(help.primaryTabId);
url = tab.url ?? "";
} catch {
return false;
}
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
if (condition.url_matches) {
try {
if (!new RegExp(condition.url_matches).test(url)) return false;
} catch {
return false;
}
}
}
if (
condition.selector_exists ||
condition.selector_missing ||
condition.text_exists ||
condition.text_missing
) {
if (!help.deps.cdp) return false;
const expression = `(() => {
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
if (selectorExists && !document.querySelector(selectorExists)) return false;
if (selectorMissing && document.querySelector(selectorMissing)) return false;
const text = document.body ? document.body.innerText || "" : "";
if (textExists && !text.includes(textExists)) return false;
if (textMissing && text.includes(textMissing)) return false;
return true;
})()`;
try {
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
help.primaryTabId,
"Runtime.evaluate",
{ expression, returnByValue: true },
);
if (result.result?.value !== true) return false;
} catch {
return false;
}
}
return true;
}
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
const criteria = help.completionCriteria;
if (!criteria) return false;
const any = criteria.any ?? [];
const all = criteria.all ?? [];
const hasAny = any.length > 0;
const hasAll = all.length > 0;
if (!hasAny && !hasAll) return false;
if (hasAll) {
for (const condition of all) {
if (!(await evaluateCompletionCondition(condition, help))) return false;
}
}
if (hasAny) {
for (const condition of any) {
if (await evaluateCompletionCondition(condition, help)) return true;
}
return false;
}
return true;
}
function startCompletionPolling(help: ActiveHelpRequest): void {
if (!help.completionCriteria) return;
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
const check = async () => {
if (help.settled) return;
const matched = await completionCriteriaMatches(help);
const now = Date.now();
if (!matched) {
help.completionMatchedSince = null;
return;
}
help.completionMatchedSince ??= now;
if (now - help.completionMatchedSince < stableMs) return;
finishHelp(help, {
outcome: "completed",
completed_by: "system",
tab_id: help.primaryTabId,
resolved_targets: help.resolvedTargets,
});
};
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
void check();
}
export async function handleRequestHelp(
manager: SessionManager,
params: RequestHelpParams,
deps: RequestHelpDeps = getDefaultDeps(),
): Promise<RequestHelpResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "request_help");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
if (!params.prompt || typeof params.prompt !== "string") {
return { code: "invalid_params", message: "request_help requires a prompt" };
}
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "request_help");
if (denied) return denied;
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
const tabId = target.tabId;
const initialTargets = params.targets ?? [];
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
await deps.activateTab(tabId).catch(() => {});
@@ -255,7 +680,6 @@ export async function handleRequestHelp(
const notificationId = `bsk-help:${requestId}`;
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
// OS notification (best-effort).
if (deps.notifications) {
const copy = deps.notificationCopy ?? {
title: "BrowserSkill: Agent needs your help",
@@ -272,85 +696,59 @@ export async function handleRequestHelp(
.catch(() => {});
}
const cleanup = async () => {
await clearRefTags(deps.cdp, tabId);
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
// Retract the overlay in case we resolved via timeout / abort.
await deps
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
.catch(() => {});
};
return new Promise<RequestHelpResult | RpcError>((resolve) => {
const timeoutTimer = setTimeout(() => {
finishHelp(help, {
outcome: "timed_out",
tab_id: help.primaryTabId,
resolved_targets: help.resolvedTargets,
});
}, timeoutMs);
const request: HelpRequestMessage = {
type: HELP_REQUEST,
requestId,
prompt: params.prompt,
...(params.title ? { title: params.title } : {}),
selectors,
timeoutMs,
};
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
let settled = false;
const unwatchNav = deps.watchTabNavigation(tabId, () => {
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
});
const finish = (value: RequestHelpResult | RpcError) => {
if (settled) return;
settled = true;
if (timer !== null) clearTimeout(timer);
unwatchNav();
deps.signal?.removeEventListener("abort", onAbort);
void cleanup();
resolveOuter(value);
const help: ActiveHelpRequest = {
ctx,
requestId,
primaryTabId: tabId,
overlayTabIds: new Set(),
prompt: params.prompt,
...(params.title ? { title: params.title } : {}),
targets: initialTargets,
selectors,
timeoutMs,
notificationId,
resolvedTargets,
completionCriteria: params.completion_criteria,
deps,
settled: false,
resolve,
timeoutTimer,
completionTimer: null,
completionMatchedSince: null,
abortHandler: null,
};
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
const timer = setTimeout(() => {
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
}, timeoutMs) as unknown as number;
if (deps.signal) {
if (deps.signal.aborted) {
onAbort();
return;
}
deps.signal.addEventListener("abort", onAbort, { once: true });
const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
help.abortHandler = onAbort;
activeHelpRequests.set(requestId, help);
if (deps.signal?.aborted) {
onAbort();
return;
}
deps.signal?.addEventListener("abort", onAbort, { once: true });
startCompletionPolling(help);
deps
.sendToTab(tabId, request)
.then((reply) => {
const res = reply as HelpResponseMessage | undefined;
if (
!res ||
res.type !== HELP_RESPONSE ||
(res.outcome !== "continued" && res.outcome !== "cancelled")
) {
finish({
code: "protocol_error",
message: `invalid help response from tab ${tabId}`,
});
return;
}
finish({
outcome: res.outcome,
...(res.note ? { note: res.note } : {}),
tab_id: tabId,
resolved_targets: resolvedTargets,
void sendCurrentHelpOverlay(help)
.then((legacyFinish) => {
if (!legacyFinish || help.settled) return;
finishHelp(help, {
outcome: legacyFinish.outcome,
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
tab_id: help.primaryTabId,
resolved_targets: help.resolvedTargets,
});
})
.catch((err) => {
finish({
code: "protocol_error",
message: `failed to show help overlay on tab ${tabId}: ${
err instanceof Error ? err.message : String(err)
}`,
});
.catch(() => {
scheduleRearmForTab(tabId, deps);
});
});
}
+18 -1
View File
@@ -514,7 +514,22 @@ export interface HelpTarget {
selector?: string;
}
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated";
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated";
export interface HelpCompletionCondition {
url_contains?: string;
url_matches?: string;
selector_exists?: string;
selector_missing?: string;
text_exists?: string;
text_missing?: string;
}
export interface HelpCompletionCriteria {
any?: HelpCompletionCondition[];
all?: HelpCompletionCondition[];
stable_for_ms?: number;
}
export interface ResolvedTarget {
matched: boolean;
@@ -528,11 +543,13 @@ export interface RequestHelpParams {
prompt: string;
title?: string;
targets?: HelpTarget[];
completion_criteria?: HelpCompletionCriteria;
timeout_ms?: number;
}
export interface RequestHelpResult {
outcome: HelpOutcome;
completed_by?: "system";
note?: string;
tab_id: number;
resolved_targets?: ResolvedTarget[];
+10 -4
View File
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
When a step needs a human (captcha, login, OTP) or you want the user to
confirm an important action, pause and ask:
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
- `--prompt` (required): what the user should do.
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
prompt with no `--target` for cases where there is genuinely no specific
element to point at (e.g. "wait for the page to finish loading").
- `--timeout` (default `5m`): how long to wait.
- `--completion-criteria` (optional): JSON success detector. Use it only
when there is a concrete post-help success signal, e.g.
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
The target tab is brought to the foreground; the page stays interactive
while the agent control mask is hidden. The call blocks until the user
acts. The result `outcome` is one of:
explicitly acts, the timeout expires, cancellation arrives, or explicit
completion criteria match. Page reloads, SPA route changes, and captcha
refreshes do not return control by themselves. The result `outcome` is one of:
- `continued` — the user finished and clicked Continue (treat as confirm).
- `continued` — the user finished and clicked Done / return control (treat as confirm).
- `cancelled` — the user clicked Cancel (treat as reject/abort).
- `timed_out` — nobody acted within the timeout.
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
- `completed` — the explicit `--completion-criteria` matched while the user had control.
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
`note` carries any text the user typed back. `resolved_targets` reports
which refs/selectors matched a live element.
+14 -1
View File
@@ -7,7 +7,9 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{HelpTarget, RequestHelpParams, RequestHelpResult};
use bsk_protocol::tools::{
HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult,
};
use clap::Args;
use crate::cli::ensure_daemon::ensure_daemon;
@@ -41,6 +43,10 @@ pub struct RequestHelpArgs {
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
pub timeout: u32,
/// JSON success detector, e.g. '{"any":[{"url_contains":"/dashboard"}],"stable_for_ms":1000}'.
#[arg(long = "completion-criteria")]
pub completion_criteria: Option<String>,
}
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
@@ -68,6 +74,12 @@ pub fn parse_target(raw: &str) -> HelpTarget {
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
let completion_criteria = args
.completion_criteria
.as_deref()
.map(serde_json::from_str::<HelpCompletionCriteria>)
.transpose()
.context("parse --completion-criteria JSON")?;
let params = RequestHelpParams {
session_id: args.session,
tab_id: args.tab_id,
@@ -78,6 +90,7 @@ pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
} else {
Some(targets)
},
completion_criteria,
timeout_ms: Some(args.timeout),
};
let reply = call(info.sock_path, params, args.timeout)?;
@@ -7,6 +7,17 @@
"session_id"
],
"properties": {
"completion_criteria": {
"description": "Optional explicit success detector. Navigation by itself is not a completion signal; these criteria must match before the tool can auto-complete without the user clicking Done.",
"anyOf": [
{
"$ref": "#/definitions/HelpCompletionCriteria"
},
{
"type": "null"
}
]
},
"prompt": {
"description": "Message shown to the user explaining what to do.",
"type": "string"
@@ -50,6 +61,78 @@
}
},
"definitions": {
"HelpCompletionCondition": {
"type": "object",
"properties": {
"selector_exists": {
"type": [
"string",
"null"
]
},
"selector_missing": {
"type": [
"string",
"null"
]
},
"text_exists": {
"type": [
"string",
"null"
]
},
"text_missing": {
"type": [
"string",
"null"
]
},
"url_contains": {
"type": [
"string",
"null"
]
},
"url_matches": {
"type": [
"string",
"null"
]
}
}
},
"HelpCompletionCriteria": {
"type": "object",
"properties": {
"all": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/HelpCompletionCondition"
}
},
"any": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/HelpCompletionCondition"
}
},
"stable_for_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
}
}
},
"HelpTarget": {
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
"type": "object",
@@ -7,6 +7,16 @@
"tab_id"
],
"properties": {
"completed_by": {
"anyOf": [
{
"$ref": "#/definitions/HelpCompletedBy"
},
{
"type": "null"
}
]
},
"note": {
"type": [
"string",
@@ -31,10 +41,16 @@
}
},
"definitions": {
"HelpCompletedBy": {
"type": "string",
"enum": [
"system"
]
},
"HelpOutcome": {
"oneOf": [
{
"description": "User completed the step and clicked Continue.",
"description": "User completed the step and clicked Done / return control.",
"type": "string",
"enum": [
"continued"
@@ -55,7 +71,14 @@
]
},
{
"description": "The page navigated while waiting (full reload or SPA URL change).",
"description": "Explicit success criteria matched while the user had control.",
"type": "string",
"enum": [
"completed"
]
},
{
"description": "The page navigated while waiting (full reload or SPA URL change). Deprecated: navigation alone should not end a help request.",
"type": "string",
"enum": [
"navigated"
+53 -2
View File
@@ -3,7 +3,8 @@
//! Lets an agent pause and ask the human to complete an in-page step
//! (captcha, login, confirmation). The extension brings the target tab
//! to the foreground, highlights the requested components, and blocks
//! until the user clicks Continue / Cancel (or the call times out).
//! until the user clicks Done / Cancel, explicit completion criteria match,
//! or the call times out.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -41,6 +42,11 @@ pub struct RequestHelpParams {
/// Optional components to scroll to + flash-highlight.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub targets: Option<Vec<HelpTarget>>,
/// Optional explicit success detector. Navigation by itself is not a
/// completion signal; these criteria must match before the tool can
/// auto-complete without the user clicking Done.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completion_criteria: Option<HelpCompletionCriteria>,
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
/// enforced daemon-side via the generic tool timeout.
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -48,16 +54,46 @@ pub struct RequestHelpParams {
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct HelpCompletionCriteria {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub any: Option<Vec<HelpCompletionCondition>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub all: Option<Vec<HelpCompletionCondition>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 0))]
pub stable_for_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct HelpCompletionCondition {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub url_contains: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub url_matches: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector_exists: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector_missing: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_exists: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_missing: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum HelpOutcome {
/// User completed the step and clicked Continue.
/// User completed the step and clicked Done / return control.
Continued,
/// User declined / aborted via the overlay's Cancel button.
Cancelled,
/// The wait expired before the user acted.
TimedOut,
/// Explicit success criteria matched while the user had control.
Completed,
/// The page navigated while waiting (full reload or SPA URL change).
/// Deprecated: navigation alone should not end a help request.
Navigated,
}
@@ -67,11 +103,18 @@ impl HelpOutcome {
Self::Continued => "continued",
Self::Cancelled => "cancelled",
Self::TimedOut => "timed_out",
Self::Completed => "completed",
Self::Navigated => "navigated",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum HelpCompletedBy {
System,
}
/// Per-target resolution status echoed back so the agent can tell which
/// `ref` / `selector` actually matched a live element.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -87,6 +130,8 @@ pub struct ResolvedTarget {
pub struct RequestHelpResult {
pub outcome: HelpOutcome,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completed_by: Option<HelpCompletedBy>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub note: Option<String>,
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -123,6 +168,7 @@ mod tests {
prompt: "log in".into(),
title: None,
targets: None,
completion_criteria: None,
timeout_ms: None,
};
let v = serde_json::to_value(&p).unwrap();
@@ -140,6 +186,10 @@ mod tests {
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
json!("timed_out")
);
assert_eq!(
serde_json::to_value(HelpOutcome::Completed).unwrap(),
json!("completed")
);
assert_eq!(
serde_json::to_value(HelpOutcome::Continued).unwrap(),
json!("continued")
@@ -158,6 +208,7 @@ mod tests {
fn result_round_trips() {
let r = RequestHelpResult {
outcome: HelpOutcome::Continued,
completed_by: None,
note: Some("done".into()),
tab_id: 7,
resolved_targets: Some(vec![ResolvedTarget {
@@ -74,10 +74,11 @@
},
"helpRequest": {
"title": "Agent needs your help",
"compactStatus": "Agent is still waiting for you to finish this step",
"noteLabel": "Optional note back to the agent",
"notePlaceholder": "Add a note for the agent (optional)",
"continue": "Continue",
"cancel": "Cancel",
"continue": "Done, return control",
"cancel": "Cancel request",
"collapse": "Collapse",
"expand": "Expand",
"dragHandle": "Drag to move",
@@ -74,10 +74,11 @@
},
"helpRequest": {
"title": "Agent 需要你的帮助",
"compactStatus": "当前 Agent 仍在等待你完成此步骤",
"noteLabel": "给 Agent 的备注(可选)",
"notePlaceholder": "给 Agent 留言(可选)",
"continue": "继续",
"cancel": "取消",
"continue": "完成并交还控制权",
"cancel": "取消请求",
"collapse": "折叠",
"expand": "展开",
"dragHandle": "拖动以移动",
+10 -4
View File
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
When a step needs a human (captcha, login, OTP) or you want the user to
confirm an important action, pause and ask:
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
- `--prompt` (required): what the user should do.
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
prompt with no `--target` for cases where there is genuinely no specific
element to point at (e.g. "wait for the page to finish loading").
- `--timeout` (default `5m`): how long to wait.
- `--completion-criteria` (optional): JSON success detector. Use it only
when there is a concrete post-help success signal, e.g.
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
The target tab is brought to the foreground; the page stays interactive
while the agent control mask is hidden. The call blocks until the user
acts. The result `outcome` is one of:
explicitly acts, the timeout expires, cancellation arrives, or explicit
completion criteria match. Page reloads, SPA route changes, and captcha
refreshes do not return control by themselves. The result `outcome` is one of:
- `continued` — the user finished and clicked Continue (treat as confirm).
- `continued` — the user finished and clicked Done / return control (treat as confirm).
- `cancelled` — the user clicked Cancel (treat as reject/abort).
- `timed_out` — nobody acted within the timeout.
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
- `completed` — the explicit `--completion-criteria` matched while the user had control.
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
`note` carries any text the user typed back. `resolved_targets` reports
which refs/selectors matched a live element.