mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Merge pull request #43 from Tencent/fix/early-control-return
fix(extension): fix early control return and preserve request-help handoff across tabs LGTM
This commit is contained in:
@@ -15,6 +15,8 @@ export interface HelpRequestData {
|
||||
prompt: string;
|
||||
/** Custom overlay title; falls back to i18n when omitted. */
|
||||
title?: string;
|
||||
/** Full task UI on the subject tab; compact status UI on related tabs. */
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight. */
|
||||
selectors: string[];
|
||||
onContinue: (note: string) => void;
|
||||
@@ -174,6 +176,8 @@ function measure(selectors: string[]): Box[] {
|
||||
|
||||
export function HelpRequestOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const isCompact = request?.displayMode === "compact";
|
||||
const effectiveSelectors = isCompact ? [] : (request?.selectors ?? []);
|
||||
const [note, setNote] = useState("");
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||
@@ -207,8 +211,9 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
// Scroll the first matched target into view once per distinct request id.
|
||||
useLayoutEffect(() => {
|
||||
if (!request) return;
|
||||
if (isCompact) return;
|
||||
if (lastScrolledIdRef.current === request.id) return;
|
||||
for (const sel of request.selectors) {
|
||||
for (const sel of effectiveSelectors) {
|
||||
const el = querySelectorSafe(sel);
|
||||
if (el) {
|
||||
lastScrolledIdRef.current = request.id;
|
||||
@@ -216,7 +221,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}, [request]);
|
||||
}, [request, isCompact, effectiveSelectors]);
|
||||
|
||||
// Keep highlight boxes aligned with the page as it scrolls / resizes.
|
||||
useEffect(() => {
|
||||
@@ -225,7 +230,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const nextBoxes = measure(request.selectors);
|
||||
const nextBoxes = measure(effectiveSelectors);
|
||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||
};
|
||||
update();
|
||||
@@ -237,7 +242,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
window.removeEventListener("resize", update);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request]);
|
||||
}, [request, effectiveSelectors]);
|
||||
|
||||
const onHeaderPointerDown = useCallback(
|
||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
@@ -294,8 +299,12 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
setPanelPos(null);
|
||||
return;
|
||||
}
|
||||
if (isCompact) {
|
||||
setPanelPos(null);
|
||||
return;
|
||||
}
|
||||
if (userMovedRef.current) return;
|
||||
const measured = boxes.length > 0 ? boxes : measure(request.selectors);
|
||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||
const anchor = unionBox(measured);
|
||||
const banner = bannerRef.current;
|
||||
if (!anchor || !banner) {
|
||||
@@ -319,7 +328,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||
}, [request, boxes, collapsed]);
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
@@ -342,6 +351,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
z-index: 2147483647;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: flex-start;
|
||||
min-height: 0;
|
||||
gap: 10px;
|
||||
width: var(--bsk-help-banner-width);
|
||||
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
||||
@@ -359,16 +370,25 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
.bsk-help-banner[data-collapsed="true"] {
|
||||
gap: 0;
|
||||
width: var(--bsk-help-banner-width);
|
||||
padding: 10px 12px;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-display-mode="compact"] {
|
||||
width: min(360px, calc(100vw - ${VIEWPORT_MARGIN * 2}px));
|
||||
padding: 12px 14px;
|
||||
gap: 0;
|
||||
border-radius: 12px;
|
||||
}
|
||||
|
||||
.bsk-help-drag-strip {
|
||||
flex-shrink: 0;
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
left: 0;
|
||||
right: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 12px;
|
||||
margin: -8px 0 -10px;
|
||||
cursor: grab;
|
||||
user-select: none;
|
||||
touch-action: none;
|
||||
@@ -395,15 +415,20 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
||||
margin: -6px 0 -8px;
|
||||
top: 4px;
|
||||
}
|
||||
|
||||
.bsk-help-header {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header {
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.bsk-help-title {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
@@ -414,31 +439,34 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
color: #111;
|
||||
}
|
||||
|
||||
.bsk-help-compact-title {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: #374151;
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-title {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.bsk-help-header-actions {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-left: auto;
|
||||
overflow: hidden;
|
||||
max-width: 0;
|
||||
opacity: 0;
|
||||
transform: translateX(8px);
|
||||
pointer-events: none;
|
||||
transition:
|
||||
max-width var(--bsk-help-duration) var(--bsk-help-ease),
|
||||
opacity 220ms var(--bsk-help-ease),
|
||||
transform var(--bsk-help-duration) var(--bsk-help-ease);
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
||||
display: flex;
|
||||
max-width: 280px;
|
||||
opacity: 1;
|
||||
transform: translateX(0);
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
@@ -471,15 +499,13 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
|
||||
.bsk-help-body {
|
||||
display: grid;
|
||||
grid-template-rows: 1fr;
|
||||
display: block;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
||||
position: absolute;
|
||||
grid-template-rows: 0fr;
|
||||
width: 0;
|
||||
height: 0;
|
||||
overflow: hidden;
|
||||
@@ -528,6 +554,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
.bsk-help-footer-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
@@ -539,7 +566,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: #4b5563;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.bsk-help-btn-continue {
|
||||
@@ -550,16 +578,30 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #fff;
|
||||
flex-shrink: 0;
|
||||
min-width: 0;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.bsk-help-header-actions .bsk-help-btn-cancel,
|
||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||
padding: 6px 12px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||
padding: 6px 14px;
|
||||
max-width: 180px;
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions {
|
||||
display: flex;
|
||||
max-width: none;
|
||||
}
|
||||
|
||||
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-cancel,
|
||||
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-continue {
|
||||
padding: 6px 10px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
||||
@@ -584,29 +626,31 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
`}</style>
|
||||
|
||||
{boxes.map((b, i) => (
|
||||
<div
|
||||
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
|
||||
key={i}
|
||||
data-slot="help-highlight"
|
||||
style={{
|
||||
position: "fixed",
|
||||
top: b.top,
|
||||
left: b.left,
|
||||
width: b.width,
|
||||
height: b.height,
|
||||
borderRadius: 6,
|
||||
zIndex: 2147483646,
|
||||
pointerEvents: "none",
|
||||
animation: "bsk-help-flash 1.2s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
{!isCompact &&
|
||||
boxes.map((b, i) => (
|
||||
<div
|
||||
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
|
||||
key={i}
|
||||
data-slot="help-highlight"
|
||||
style={{
|
||||
position: "fixed",
|
||||
top: b.top,
|
||||
left: b.left,
|
||||
width: b.width,
|
||||
height: b.height,
|
||||
borderRadius: 6,
|
||||
zIndex: 2147483646,
|
||||
pointerEvents: "none",
|
||||
animation: "bsk-help-flash 1.2s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
|
||||
<div
|
||||
ref={bannerRef}
|
||||
data-slot="help-request-banner"
|
||||
data-collapsed={collapsed ? "true" : "false"}
|
||||
data-display-mode={isCompact ? "compact" : "full"}
|
||||
data-anchored={panelPos ? "true" : "false"}
|
||||
data-dragging={dragging ? "true" : "false"}
|
||||
data-placement={panelPos?.placement}
|
||||
@@ -629,15 +673,17 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<div
|
||||
data-slot="help-drag-handle"
|
||||
className="bsk-help-drag-strip"
|
||||
role="img"
|
||||
aria-label={t("helpRequest.dragHandle")}
|
||||
onPointerDown={onHeaderPointerDown}
|
||||
>
|
||||
<span className="bsk-help-drag-pill" aria-hidden />
|
||||
</div>
|
||||
{!isCompact && (
|
||||
<div
|
||||
data-slot="help-drag-handle"
|
||||
className="bsk-help-drag-strip"
|
||||
role="img"
|
||||
aria-label={t("helpRequest.dragHandle")}
|
||||
onPointerDown={onHeaderPointerDown}
|
||||
>
|
||||
<span className="bsk-help-drag-pill" aria-hidden />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="bsk-help-header">
|
||||
<img
|
||||
@@ -645,77 +691,87 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
alt="browser-skill"
|
||||
style={{ width: 22, height: 22, borderRadius: 4 }}
|
||||
/>
|
||||
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
|
||||
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
|
||||
{isCompact ? (
|
||||
<span className="bsk-help-compact-title">{t("helpRequest.compactStatus")}</span>
|
||||
) : (
|
||||
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
|
||||
)}
|
||||
{!isCompact && (
|
||||
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? "help-cancel-button" : undefined}
|
||||
className="bsk-help-btn-cancel"
|
||||
tabIndex={collapsed ? 0 : -1}
|
||||
onClick={() => request.onCancel()}
|
||||
>
|
||||
{t("helpRequest.cancel")}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? "help-continue-button" : undefined}
|
||||
className="bsk-help-btn-continue"
|
||||
tabIndex={collapsed ? 0 : -1}
|
||||
onClick={() => request.onContinue(note)}
|
||||
>
|
||||
{t("helpRequest.continue")}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
{!isCompact && (
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? "help-cancel-button" : undefined}
|
||||
className="bsk-help-btn-cancel"
|
||||
tabIndex={collapsed ? 0 : -1}
|
||||
onClick={() => request.onCancel()}
|
||||
data-slot="help-collapse-toggle"
|
||||
className="bsk-help-collapse-toggle"
|
||||
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
|
||||
aria-expanded={!collapsed}
|
||||
onClick={() => setCollapsed((c) => !c)}
|
||||
>
|
||||
{t("helpRequest.cancel")}
|
||||
<span className="bsk-help-collapse-icon">
|
||||
<RiArrowDownSLine size={20} aria-hidden />
|
||||
</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? "help-continue-button" : undefined}
|
||||
className="bsk-help-btn-continue"
|
||||
tabIndex={collapsed ? 0 : -1}
|
||||
onClick={() => request.onContinue(note)}
|
||||
>
|
||||
{t("helpRequest.continue")}
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
data-slot="help-collapse-toggle"
|
||||
className="bsk-help-collapse-toggle"
|
||||
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
|
||||
aria-expanded={!collapsed}
|
||||
onClick={() => setCollapsed((c) => !c)}
|
||||
>
|
||||
<span className="bsk-help-collapse-icon">
|
||||
<RiArrowDownSLine size={20} aria-hidden />
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="bsk-help-body" aria-hidden={collapsed}>
|
||||
<div className="bsk-help-body-inner">
|
||||
<div className="bsk-help-body-content">
|
||||
<p className="bsk-help-prompt">{request.prompt}</p>
|
||||
<input
|
||||
type="text"
|
||||
className="bsk-help-note-input"
|
||||
aria-label={t("helpRequest.noteLabel")}
|
||||
placeholder={t("helpRequest.notePlaceholder")}
|
||||
value={note}
|
||||
tabIndex={collapsed ? -1 : 0}
|
||||
onChange={(e) => setNote(e.target.value)}
|
||||
/>
|
||||
<div className="bsk-help-footer-actions">
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? undefined : "help-cancel-button"}
|
||||
className="bsk-help-btn-cancel"
|
||||
{!isCompact && (
|
||||
<div className="bsk-help-body" aria-hidden={collapsed}>
|
||||
<div className="bsk-help-body-inner">
|
||||
<div className="bsk-help-body-content">
|
||||
<p className="bsk-help-prompt">{request.prompt}</p>
|
||||
<input
|
||||
type="text"
|
||||
className="bsk-help-note-input"
|
||||
aria-label={t("helpRequest.noteLabel")}
|
||||
placeholder={t("helpRequest.notePlaceholder")}
|
||||
value={note}
|
||||
tabIndex={collapsed ? -1 : 0}
|
||||
onClick={() => request.onCancel()}
|
||||
>
|
||||
{t("helpRequest.cancel")}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? undefined : "help-continue-button"}
|
||||
className="bsk-help-btn-continue"
|
||||
tabIndex={collapsed ? -1 : 0}
|
||||
onClick={() => request.onContinue(note)}
|
||||
>
|
||||
{t("helpRequest.continue")}
|
||||
</button>
|
||||
onChange={(e) => setNote(e.target.value)}
|
||||
/>
|
||||
<div className="bsk-help-footer-actions">
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? undefined : "help-cancel-button"}
|
||||
className="bsk-help-btn-cancel"
|
||||
tabIndex={collapsed ? -1 : 0}
|
||||
onClick={() => request.onCancel()}
|
||||
>
|
||||
{t("helpRequest.cancel")}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
data-slot={collapsed ? undefined : "help-continue-button"}
|
||||
className="bsk-help-btn-continue"
|
||||
tabIndex={collapsed ? -1 : 0}
|
||||
onClick={() => request.onContinue(note)}
|
||||
>
|
||||
{t("helpRequest.continue")}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
|
||||
@@ -38,6 +38,31 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders a compact status without full request controls", () => {
|
||||
const el = document.createElement("div");
|
||||
el.id = "login";
|
||||
el.getBoundingClientRect = () =>
|
||||
({ top: 10, left: 10, width: 100, height: 40, right: 110, bottom: 50 }) as DOMRect;
|
||||
document.body.append(el);
|
||||
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(screen.getByText(i18n.t("helpRequest.compactStatus", { ns: "extension" }))).toBeTruthy();
|
||||
expect(screen.queryByText("Please complete the captcha")).toBeNull();
|
||||
expect(screen.queryByRole("textbox")).toBeNull();
|
||||
expect(screen.queryByLabelText(i18n.t("helpRequest.collapse", { ns: "extension" }))).toBeNull();
|
||||
expect(container.querySelector("[data-slot='help-continue-button']")).toBeNull();
|
||||
expect(container.querySelector("[data-slot='help-cancel-button']")).toBeNull();
|
||||
expect(document.querySelectorAll("[data-slot='help-highlight']").length).toBe(0);
|
||||
|
||||
el.remove();
|
||||
});
|
||||
|
||||
it("renders a custom title when provided", () => {
|
||||
renderOverlay(baseRequest({ title: "Verify your identity" }));
|
||||
expect(screen.getByText("Verify your identity")).toBeTruthy();
|
||||
@@ -204,6 +229,15 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(styles).toContain("width: 0");
|
||||
});
|
||||
|
||||
it("uses natural body height in the expanded layout", () => {
|
||||
const { container } = renderOverlay(baseRequest());
|
||||
const styles = overlayStyles(container);
|
||||
|
||||
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
|
||||
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
|
||||
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
|
||||
});
|
||||
|
||||
it("keeps the same banner width when collapsed", () => {
|
||||
const { container } = renderOverlay(baseRequest());
|
||||
const styles = overlayStyles(container);
|
||||
|
||||
@@ -14,10 +14,14 @@ import {
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
HELP_RESPONSE,
|
||||
HELP_ACK,
|
||||
HELP_FINISH,
|
||||
HELP_QUERY,
|
||||
type HelpAckMessage,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
type HelpResponseMessage,
|
||||
isHelpCancelMessage,
|
||||
isHelpRequestMessage,
|
||||
} from "@/lib/help-bridge";
|
||||
@@ -55,8 +59,6 @@ export default defineContentScript({
|
||||
if (window.top !== window) return;
|
||||
|
||||
const overlays = new OverlayController();
|
||||
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
|
||||
null;
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
@@ -116,8 +118,7 @@ export default defineContentScript({
|
||||
function resetAgentOverlayState(sessionId: string) {
|
||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||
if (previousHelp) {
|
||||
activeHelpRespond?.("cancelled");
|
||||
activeHelpRespond = null;
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
@@ -156,7 +157,7 @@ export default defineContentScript({
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||
) => {
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
@@ -220,41 +221,30 @@ export default defineContentScript({
|
||||
if (isHelpCancelMessage(message)) {
|
||||
const state = overlays.snapshot();
|
||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||
activeHelpRespond?.("cancelled");
|
||||
overlays.clearAgentHelpRequest(message.requestId);
|
||||
renderOverlay();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isHelpRequestMessage(message)) {
|
||||
const helpMsg = message as HelpRequestMessage;
|
||||
let responded = false;
|
||||
const respond = (outcome: "continued" | "cancelled", note?: string) => {
|
||||
if (responded) return;
|
||||
responded = true;
|
||||
const reply: HelpResponseMessage = {
|
||||
type: HELP_RESPONSE,
|
||||
outcome,
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
sendResponse(reply);
|
||||
activeHelpRespond = null;
|
||||
overlays.clearAgentHelpRequest(helpMsg.requestId);
|
||||
renderOverlay();
|
||||
};
|
||||
const previousHelp = overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
|
||||
onCancel: () => respond("cancelled"),
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
if (previousHelp) {
|
||||
activeHelpRespond?.("cancelled");
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
activeHelpRespond = respond;
|
||||
renderOverlay();
|
||||
return true; // async sendResponse
|
||||
sendResponse({ type: HELP_ACK, ok: true });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (message.type === "borrow-request") {
|
||||
@@ -282,9 +272,60 @@ export default defineContentScript({
|
||||
return false;
|
||||
};
|
||||
|
||||
async function sendHelpFinish(
|
||||
requestId: string,
|
||||
outcome: "continued" | "cancelled",
|
||||
note?: string,
|
||||
): Promise<void> {
|
||||
const msg: HelpFinishMessage = {
|
||||
type: HELP_FINISH,
|
||||
requestId,
|
||||
outcome,
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
overlays.clearAgentHelpRequest(requestId);
|
||||
renderOverlay();
|
||||
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||
console.debug("[bsk overlay] help finish failed", err);
|
||||
});
|
||||
}
|
||||
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||
overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<boolean> {
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
try {
|
||||
const helpQuery = (await chrome.runtime.sendMessage({
|
||||
type: HELP_QUERY,
|
||||
})) as HelpQueryResponse | undefined;
|
||||
if (helpQuery?.active && helpQuery.request) {
|
||||
mountHelpRequest(helpQuery.request);
|
||||
renderOverlay();
|
||||
return true;
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] help query failed", err);
|
||||
}
|
||||
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function syncAgentOverlay(): Promise<void> {
|
||||
if (!(await anySessionLive())) return;
|
||||
try {
|
||||
const helpActive = await queryActiveHelpWithRetry();
|
||||
const reply = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_WHO_AM_I,
|
||||
})) as OverlayWhoAmIResponse | undefined;
|
||||
@@ -319,6 +360,10 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
if (!helpActive && overlays.snapshot().activeHelp === null) {
|
||||
void queryActiveHelpWithRetry();
|
||||
}
|
||||
|
||||
overlays.activateAgentSession(reply.sessionId);
|
||||
renderOverlay();
|
||||
} catch (err) {
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
HELP_ACK,
|
||||
HELP_CANCEL,
|
||||
HELP_FINISH,
|
||||
HELP_QUERY,
|
||||
HELP_REQUEST,
|
||||
HELP_RESPONSE,
|
||||
isHelpAckMessage,
|
||||
isHelpCancelMessage,
|
||||
isHelpFinishMessage,
|
||||
isHelpQueryMessage,
|
||||
isHelpRequestMessage,
|
||||
isHelpResponseMessage,
|
||||
} from "../help-bridge";
|
||||
|
||||
describe("help-bridge", () => {
|
||||
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
|
||||
expect(HELP_REQUEST).toBe("bsk-help-request");
|
||||
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
||||
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
||||
expect(HELP_ACK).toBe("bsk-help-ack");
|
||||
expect(HELP_QUERY).toBe("bsk-help-query");
|
||||
expect(HELP_FINISH).toBe("bsk-help-finish");
|
||||
});
|
||||
|
||||
it("type-guards a help-request message", () => {
|
||||
@@ -20,6 +30,7 @@ describe("help-bridge", () => {
|
||||
type: HELP_REQUEST,
|
||||
requestId: "r1",
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
@@ -55,4 +66,18 @@ describe("help-bridge", () => {
|
||||
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
||||
expect(isHelpCancelMessage(null)).toBe(false);
|
||||
});
|
||||
|
||||
it("type-guards lifecycle messages", () => {
|
||||
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
|
||||
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
|
||||
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,12 +6,15 @@
|
||||
* Mirrors the borrow-confirmation message style (see
|
||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||
* the content script replies once the user clicks Continue / Cancel.
|
||||
* the content script acks display, then later reports Done / Cancel.
|
||||
*/
|
||||
|
||||
export const HELP_REQUEST = "bsk-help-request";
|
||||
export const HELP_RESPONSE = "bsk-help-response";
|
||||
export const HELP_CANCEL = "bsk-help-cancel";
|
||||
export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
@@ -19,11 +22,18 @@ export interface HelpRequestMessage {
|
||||
prompt: string;
|
||||
/** Custom overlay title; omitted when the extension should use its default. */
|
||||
title?: string;
|
||||
/** Full task UI on the subject tab; compact status UI on related tabs. */
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||
selectors: string[];
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
export interface HelpAckMessage {
|
||||
type: typeof HELP_ACK;
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export interface HelpResponseMessage {
|
||||
type: typeof HELP_RESPONSE;
|
||||
outcome: "continued" | "cancelled";
|
||||
@@ -35,6 +45,22 @@ export interface HelpCancelMessage {
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface HelpQueryMessage {
|
||||
type: typeof HELP_QUERY;
|
||||
}
|
||||
|
||||
export interface HelpQueryResponse {
|
||||
active: boolean;
|
||||
request?: Omit<HelpRequestMessage, "type">;
|
||||
}
|
||||
|
||||
export interface HelpFinishMessage {
|
||||
type: typeof HELP_FINISH;
|
||||
requestId: string;
|
||||
outcome: "continued" | "cancelled";
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
if (typeof msg !== "object" || msg === null) {
|
||||
return false;
|
||||
@@ -45,6 +71,7 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
typeof m.requestId === "string" &&
|
||||
typeof m.prompt === "string" &&
|
||||
(m.title === undefined || typeof m.title === "string") &&
|
||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||
Array.isArray(m.selectors) &&
|
||||
m.selectors.every((selector) => typeof selector === "string") &&
|
||||
typeof m.timeoutMs === "number"
|
||||
@@ -58,3 +85,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
||||
}
|
||||
|
||||
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return (
|
||||
m.type === HELP_RESPONSE &&
|
||||
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||
(m.note === undefined || typeof m.note === "string")
|
||||
);
|
||||
}
|
||||
|
||||
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === HELP_ACK && m.ok === true;
|
||||
}
|
||||
|
||||
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === HELP_QUERY;
|
||||
}
|
||||
|
||||
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return (
|
||||
m.type === HELP_FINISH &&
|
||||
typeof m.requestId === "string" &&
|
||||
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||
(m.note === undefined || typeof m.note === "string")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,53 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { RequestHelpParams } from "@/transport/types";
|
||||
import { handleRequestHelp, type RequestHelpDeps } from "../human-loop";
|
||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||
|
||||
function chromeEvent<T extends (...args: never[]) => unknown>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
addListener: vi.fn((listener: T) => {
|
||||
listeners.add(listener);
|
||||
}),
|
||||
removeListener: vi.fn((listener: T) => {
|
||||
listeners.delete(listener);
|
||||
}),
|
||||
emit: (...args: Parameters<T>) => {
|
||||
for (const listener of [...listeners]) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function installHelpLifecycleChrome() {
|
||||
const runtimeOnMessage =
|
||||
chromeEvent<
|
||||
(
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: unknown) => void,
|
||||
) => unknown
|
||||
>();
|
||||
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
|
||||
const tabsOnCreated = chromeEvent<(tab: chrome.tabs.Tab) => unknown>();
|
||||
const tabsOnUpdated =
|
||||
chromeEvent<(tabId: number, changeInfo: chrome.tabs.TabChangeInfo) => unknown>();
|
||||
const webNavigationOnCompleted =
|
||||
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { onMessage: runtimeOnMessage },
|
||||
tabs: { onActivated: tabsOnActivated, onCreated: tabsOnCreated, onUpdated: tabsOnUpdated },
|
||||
webNavigation: { onCompleted: webNavigationOnCompleted },
|
||||
});
|
||||
|
||||
return {
|
||||
runtimeOnMessage,
|
||||
tabsOnActivated,
|
||||
tabsOnCreated,
|
||||
tabsOnUpdated,
|
||||
webNavigationOnCompleted,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||
const mgr = {
|
||||
@@ -27,14 +73,19 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
|
||||
windows: { update: vi.fn(async () => ({}) as never) },
|
||||
activateTab: vi.fn(async () => {}),
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
||||
watchTabNavigation: () => () => {},
|
||||
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
||||
notifications: null,
|
||||
autoAttachLifecycle: false,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe("handleRequestHelp", () => {
|
||||
afterEach(() => {
|
||||
resetHelpLifecycleForTests();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("rejects unknown session", async () => {
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
@@ -78,22 +129,29 @@ describe("handleRequestHelp", () => {
|
||||
expect(sentMsg.title).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns navigated when the tab navigates during the wait", async () => {
|
||||
const unwatch = vi.fn();
|
||||
it("does not complete merely because the tab navigates during the wait", async () => {
|
||||
vi.useFakeTimers();
|
||||
const deps = baseDeps({
|
||||
sendToTab: vi.fn(() => new Promise(() => {})),
|
||||
watchTabNavigation: vi.fn((_tabId, cb) => {
|
||||
queueMicrotask(() => cb());
|
||||
return unwatch;
|
||||
}),
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
deps,
|
||||
);
|
||||
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
|
||||
expect(unwatch).toHaveBeenCalled();
|
||||
try {
|
||||
const pending = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
let settled = false;
|
||||
void pending.then(() => {
|
||||
settled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns timed_out when the wait expires", async () => {
|
||||
@@ -115,6 +173,160 @@ describe("handleRequestHelp", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("returns completed when explicit completion criteria match", async () => {
|
||||
const deps = baseDeps({
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||
cdp: {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: true } };
|
||||
return {};
|
||||
}),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({
|
||||
tab_id: 5,
|
||||
completion_criteria: {
|
||||
any: [{ selector_exists: "#account-menu" }],
|
||||
stable_for_ms: 0,
|
||||
},
|
||||
}),
|
||||
deps,
|
||||
);
|
||||
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
|
||||
});
|
||||
|
||||
it("keeps user control active on new tabs without moving completion off the primary tab", async () => {
|
||||
vi.useFakeTimers();
|
||||
const chromeEvents = installHelpLifecycleChrome();
|
||||
const sendToTab = vi.fn(async () => ({ type: "bsk-help-ack", ok: true }));
|
||||
const deps = baseDeps({
|
||||
autoAttachLifecycle: undefined,
|
||||
sendToTab,
|
||||
tabsApi: {
|
||||
get: vi.fn(async (tabId: number) => ({
|
||||
id: tabId,
|
||||
windowId: 99,
|
||||
active: tabId === 6,
|
||||
url: tabId === 6 ? "https://app.example/reset/success" : "https://app.example/login",
|
||||
})) as never,
|
||||
query: vi.fn(async () => [{ id: 5, windowId: 99, active: true }] as never),
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const pending = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({
|
||||
tab_id: 5,
|
||||
timeout_ms: 1_000,
|
||||
completion_criteria: {
|
||||
any: [{ url_contains: "/reset/success" }],
|
||||
stable_for_ms: 0,
|
||||
},
|
||||
}),
|
||||
deps,
|
||||
);
|
||||
await vi.waitFor(() => expect(sendToTab).toHaveBeenCalledWith(5, expect.anything()));
|
||||
|
||||
chromeEvents.tabsOnActivated.emit({ tabId: 6, windowId: 99 });
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(sendToTab).toHaveBeenCalledWith(
|
||||
6,
|
||||
expect.objectContaining({
|
||||
type: "bsk-help-request",
|
||||
displayMode: "compact",
|
||||
selectors: [],
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
outcome: "timed_out",
|
||||
tab_id: 5,
|
||||
});
|
||||
expect(sendToTab).toHaveBeenCalledWith(
|
||||
6,
|
||||
expect.objectContaining({ type: "bsk-help-cancel" }),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("lets same-window tabs query active help after content-script load", async () => {
|
||||
const chromeEvents = installHelpLifecycleChrome();
|
||||
const ac = new AbortController();
|
||||
const cdpSend = vi.fn(async () => ({ root: { nodeId: 1 }, nodeIds: [] }));
|
||||
const deps = baseDeps({
|
||||
autoAttachLifecycle: undefined,
|
||||
signal: ac.signal,
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||
cdp: {
|
||||
send: cdpSend,
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
|
||||
const pending = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 60_000 }),
|
||||
deps,
|
||||
);
|
||||
await vi.waitFor(() => expect(deps.sendToTab).toHaveBeenCalledWith(5, expect.anything()));
|
||||
|
||||
const sameWindowResponse = vi.fn();
|
||||
chromeEvents.runtimeOnMessage.emit(
|
||||
{ type: "bsk-help-query" },
|
||||
{ tab: { id: 6 } as chrome.tabs.Tab },
|
||||
sameWindowResponse,
|
||||
);
|
||||
await vi.waitFor(() =>
|
||||
expect(sameWindowResponse).toHaveBeenCalledWith({
|
||||
active: true,
|
||||
request: expect.objectContaining({
|
||||
requestId: expect.any(String),
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: [],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const subjectResponse = vi.fn();
|
||||
chromeEvents.runtimeOnMessage.emit(
|
||||
{ type: "bsk-help-query" },
|
||||
{ tab: { id: 5 } as chrome.tabs.Tab },
|
||||
subjectResponse,
|
||||
);
|
||||
await vi.waitFor(() =>
|
||||
expect(subjectResponse).toHaveBeenCalledWith({
|
||||
active: true,
|
||||
request: expect.objectContaining({
|
||||
requestId: expect.any(String),
|
||||
prompt: "log in",
|
||||
displayMode: "full",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
ac.abort();
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
5,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
6,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
@@ -190,38 +402,40 @@ describe("handleRequestHelp", () => {
|
||||
expect("code" in res).toBe(false);
|
||||
});
|
||||
|
||||
it("returns protocol_error when sendToTab rejects", async () => {
|
||||
it("keeps waiting when the initial overlay delivery rejects", async () => {
|
||||
vi.useFakeTimers();
|
||||
const deps = baseDeps({
|
||||
sendToTab: vi.fn(async () => {
|
||||
throw new Error("no receiver");
|
||||
}),
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
deps,
|
||||
);
|
||||
expect("code" in res && res.code).toBe("protocol_error");
|
||||
try {
|
||||
const res = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns protocol_error for a malformed help response", async () => {
|
||||
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||
const res1 = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
undefinedReply,
|
||||
);
|
||||
expect("code" in res1 && res1.code).toBe("protocol_error");
|
||||
|
||||
const badOutcome = baseDeps({
|
||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
|
||||
});
|
||||
const res2 = await handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5 }),
|
||||
badOutcome,
|
||||
);
|
||||
expect("code" in res2 && res2.code).toBe("protocol_error");
|
||||
it("keeps waiting for explicit completion after a malformed help response", async () => {
|
||||
vi.useFakeTimers();
|
||||
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||
try {
|
||||
const res = handleRequestHelp(
|
||||
fakeManager("abcd", 99, 5),
|
||||
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("reports selector match status from CDP", async () => {
|
||||
|
||||
@@ -29,7 +29,7 @@ import type {
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
@@ -394,7 +394,6 @@ export class ToolDispatcher {
|
||||
await chrome.tabs.update(tabId, { active: true });
|
||||
},
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
watchTabNavigation: defaultWatchTabNavigation,
|
||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||
notifications: makeHelpNotifications(),
|
||||
notificationCopy: this.helpNotificationCopy?.(),
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
// `tool.request_help` — pause and ask the human to act on a tab.
|
||||
//
|
||||
// Brings the target tab to the foreground, resolves `ref` targets to a
|
||||
// temporary attribute selector via CDP, asks the content script to enter
|
||||
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
|
||||
// until the user clicks Continue / Cancel, the wait times out, or the
|
||||
// daemon cancels the RPC.
|
||||
// `tool.request_help` — pause automation and let the human control a tab.
|
||||
|
||||
import {
|
||||
HELP_ACK,
|
||||
HELP_CANCEL,
|
||||
HELP_FINISH,
|
||||
HELP_QUERY,
|
||||
HELP_REQUEST,
|
||||
HELP_RESPONSE,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
type HelpResponseMessage,
|
||||
isHelpAckMessage,
|
||||
isHelpFinishMessage,
|
||||
isHelpQueryMessage,
|
||||
isHelpResponseMessage,
|
||||
} from "@/lib/help-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
HelpCompletionCondition,
|
||||
HelpCompletionCriteria,
|
||||
HelpTarget,
|
||||
RequestHelpParams,
|
||||
RequestHelpResult,
|
||||
@@ -34,53 +37,59 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
/** Attribute the overlay highlights for resolved `ref` targets. */
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
const HELP_REARM_MAX_ATTEMPTS = 12;
|
||||
const HELP_REARM_RETRY_DELAY_MS = 400;
|
||||
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
|
||||
const COMPLETION_POLL_MS = 500;
|
||||
|
||||
export interface RequestHelpNotifications {
|
||||
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
||||
clear(id: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export type TabNavigationUnsubscribe = () => void;
|
||||
|
||||
export interface RequestHelpDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
windows: {
|
||||
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
||||
};
|
||||
/** Activate a tab inside its window. */
|
||||
activateTab(tabId: number): Promise<void>;
|
||||
/** Send the help-request to the tab's content script and await reply. */
|
||||
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
||||
/**
|
||||
* Watch for navigation on `tabId` (full page load or SPA URL change).
|
||||
* Invokes `onNavigated` once; returns unsubscribe.
|
||||
*/
|
||||
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
|
||||
cdp?: CdpRunner;
|
||||
/** Pass `null` to skip OS notifications (tests). */
|
||||
notifications: RequestHelpNotifications | null;
|
||||
notificationCopy?: { title: string; body: string };
|
||||
signal?: AbortSignal;
|
||||
autoAttachLifecycle?: boolean;
|
||||
}
|
||||
|
||||
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
|
||||
export function defaultWatchTabNavigation(
|
||||
tabId: number,
|
||||
onNavigated: () => void,
|
||||
): TabNavigationUnsubscribe {
|
||||
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
|
||||
if (updatedTabId !== tabId) return;
|
||||
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
|
||||
chrome.tabs.onUpdated.removeListener(listener);
|
||||
onNavigated();
|
||||
}
|
||||
};
|
||||
chrome.tabs.onUpdated.addListener(listener);
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
interface ActiveHelpRequest {
|
||||
ctx: SessionContext;
|
||||
requestId: string;
|
||||
primaryTabId: number;
|
||||
overlayTabIds: Set<number>;
|
||||
prompt: string;
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
completionCriteria?: HelpCompletionCriteria;
|
||||
deps: RequestHelpDeps;
|
||||
settled: boolean;
|
||||
resolve: (value: RequestHelpResult | RpcError) => void;
|
||||
timeoutTimer: ReturnType<typeof setTimeout>;
|
||||
completionTimer: ReturnType<typeof setInterval> | null;
|
||||
completionMatchedSince: number | null;
|
||||
abortHandler: (() => void) | null;
|
||||
}
|
||||
|
||||
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
|
||||
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
|
||||
let defaultDeps: RequestHelpDeps | null = null;
|
||||
function getDefaultDeps(): RequestHelpDeps {
|
||||
if (!defaultDeps) {
|
||||
@@ -91,7 +100,6 @@ function getDefaultDeps(): RequestHelpDeps {
|
||||
await chrome.tabs.update(tabId, { active: true });
|
||||
},
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
watchTabNavigation: defaultWatchTabNavigation,
|
||||
notifications: {
|
||||
create: (id, opts) =>
|
||||
new Promise((resolve, reject) =>
|
||||
@@ -113,11 +121,50 @@ function makeRequestId(tabId: number): string {
|
||||
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
|
||||
* content overlay can highlight it with a plain attribute selector.
|
||||
* Returns the selector on success, or null when the ref can't resolve.
|
||||
*/
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequestMessage {
|
||||
const isPrimaryTab = tabId === help.primaryTabId;
|
||||
return {
|
||||
type: HELP_REQUEST,
|
||||
requestId: help.requestId,
|
||||
prompt: help.prompt,
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: isPrimaryTab ? "full" : "compact",
|
||||
selectors: isPrimaryTab ? help.selectors : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
async function sendHelpRequestWithAck(
|
||||
tabId: number,
|
||||
msg: HelpRequestMessage,
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<HelpFinishMessage | null> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, msg);
|
||||
if (isHelpAckMessage(response)) return null;
|
||||
if (isHelpResponseMessage(response)) {
|
||||
return {
|
||||
type: HELP_FINISH,
|
||||
requestId: msg.requestId,
|
||||
outcome: response.outcome,
|
||||
...(response.note ? { note: response.note } : {}),
|
||||
};
|
||||
}
|
||||
lastError = new Error("content script did not ack HELP_REQUEST");
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
|
||||
}
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -145,7 +192,6 @@ async function tagRefTarget(
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns true when `selector` matches a live element in the page. */
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -163,7 +209,6 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove every help attribute we added (best-effort cleanup). */
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
@@ -180,34 +225,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// best-effort
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRequestHelp(
|
||||
manager: SessionManager,
|
||||
params: RequestHelpParams,
|
||||
deps: RequestHelpDeps = getDefaultDeps(),
|
||||
): Promise<RequestHelpResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "request_help");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (!params.prompt || typeof params.prompt !== "string") {
|
||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||
}
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "request_help aborted" };
|
||||
}
|
||||
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||
if (denied) return denied;
|
||||
const tabId = target.tabId;
|
||||
|
||||
// Resolve targets → selectors. `selector` passes through; `ref` is
|
||||
// tagged with an attribute via CDP and reported as its attr selector.
|
||||
const targets = params.targets ?? [];
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
const selectors: string[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
@@ -221,22 +248,20 @@ export async function handleRequestHelp(
|
||||
rootNodeId = undefined;
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < targets.length; i++) {
|
||||
const tgt = targets[i] as HelpTarget;
|
||||
|
||||
for (let i = 0; i < targets.length; i += 1) {
|
||||
const tgt = targets[i];
|
||||
if (tgt.selector) {
|
||||
selectors.push(tgt.selector);
|
||||
let matched: boolean;
|
||||
if (deps.cdp) {
|
||||
matched =
|
||||
rootNodeId !== undefined
|
||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||
: false;
|
||||
} else {
|
||||
matched = true;
|
||||
}
|
||||
const matched =
|
||||
deps.cdp && rootNodeId !== undefined
|
||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||
: deps.cdp
|
||||
? false
|
||||
: true;
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
@@ -247,7 +272,407 @@ export async function handleRequestHelp(
|
||||
}
|
||||
}
|
||||
|
||||
// Bring the tab to the foreground.
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.primaryTabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
if (help.deps.notifications) {
|
||||
await help.deps.notifications.clear(help.notificationId).catch(() => {});
|
||||
}
|
||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||
await Promise.all(
|
||||
[...tabsToCancel].map((tabId) =>
|
||||
Promise.all([
|
||||
clearRefTags(help.deps.cdp, tabId),
|
||||
help.deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {}),
|
||||
]),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function finishHelp(
|
||||
help: ActiveHelpRequest,
|
||||
value: RequestHelpResult | RpcError,
|
||||
notifyContent = true,
|
||||
): void {
|
||||
if (help.settled) return;
|
||||
help.settled = true;
|
||||
activeHelpRequests.delete(help.requestId);
|
||||
clearTimeout(help.timeoutTimer);
|
||||
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
|
||||
clearRearmTimer(help.primaryTabId);
|
||||
for (const tabId of help.overlayTabIds) clearRearmTimer(tabId);
|
||||
if (notifyContent) void cleanupHelp(help);
|
||||
help.resolve(value);
|
||||
}
|
||||
|
||||
function findHelpByKnownTabId(tabId: number): ActiveHelpRequest | null {
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
if (!help.settled && (help.primaryTabId === tabId || help.overlayTabIds.has(tabId))) {
|
||||
return help;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function findHelpForTab(
|
||||
tabId: number,
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<ActiveHelpRequest | null> {
|
||||
const direct = findHelpByKnownTabId(tabId);
|
||||
if (direct) return direct;
|
||||
|
||||
try {
|
||||
const tab = await deps.tabsApi.get(tabId);
|
||||
const windowId = tab.windowId;
|
||||
if (typeof windowId !== "number") return null;
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
if (!help.settled && help.ctx.agentWindowId === windowId) return help;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function clearRearmTimer(tabId: number): void {
|
||||
const timer = helpRearmTimers.get(tabId);
|
||||
if (!timer) return;
|
||||
clearTimeout(timer);
|
||||
helpRearmTimers.delete(tabId);
|
||||
}
|
||||
|
||||
async function sendCurrentHelpOverlay(help: ActiveHelpRequest): Promise<HelpFinishMessage | null> {
|
||||
const legacyFinish = await sendHelpRequestWithAck(
|
||||
help.primaryTabId,
|
||||
helpRequestMessage(help, help.primaryTabId),
|
||||
help.deps,
|
||||
);
|
||||
help.overlayTabIds.add(help.primaryTabId);
|
||||
return legacyFinish;
|
||||
}
|
||||
|
||||
async function refreshAndSendHelpOverlay(
|
||||
help: ActiveHelpRequest,
|
||||
tabId: number,
|
||||
): Promise<HelpFinishMessage | null> {
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
const legacyFinish = await sendHelpRequestWithAck(
|
||||
tabId,
|
||||
helpRequestMessage(help, tabId),
|
||||
help.deps,
|
||||
);
|
||||
help.overlayTabIds.add(tabId);
|
||||
return legacyFinish;
|
||||
}
|
||||
|
||||
async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise<boolean> {
|
||||
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
const legacyFinish = await refreshAndSendHelpOverlay(help, tabId);
|
||||
if (legacyFinish && !help.settled) {
|
||||
finishHelp(help, {
|
||||
outcome: legacyFinish.outcome,
|
||||
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||
tab_id: help.primaryTabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
|
||||
const existing = helpRearmTimers.get(tabId);
|
||||
if (existing) clearTimeout(existing);
|
||||
helpRearmTimers.set(
|
||||
tabId,
|
||||
setTimeout(() => {
|
||||
helpRearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const help = await findHelpForTab(tabId, deps);
|
||||
if (!help) return;
|
||||
await rearmHelp(help, tabId);
|
||||
})();
|
||||
}, HELP_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
}
|
||||
|
||||
let detachHelpObservation: (() => void) | null = null;
|
||||
|
||||
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
|
||||
if (detachHelpObservation) return;
|
||||
const detachRuntime = attachHelpRuntimeListener(deps);
|
||||
const detachTabs = attachHelpTabListener(deps);
|
||||
const detachNav = attachHelpNavigationListener(deps);
|
||||
detachHelpObservation = () => {
|
||||
detachRuntime();
|
||||
detachTabs();
|
||||
detachNav();
|
||||
};
|
||||
}
|
||||
|
||||
export function resetHelpLifecycleForTests(): void {
|
||||
detachHelpObservation?.();
|
||||
detachHelpObservation = null;
|
||||
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
|
||||
helpRearmTimers.clear();
|
||||
for (const help of activeHelpRequests.values()) {
|
||||
clearTimeout(help.timeoutTimer);
|
||||
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||
}
|
||||
activeHelpRequests.clear();
|
||||
}
|
||||
|
||||
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: HelpQueryResponse) => void,
|
||||
) => {
|
||||
if (isHelpFinishMessage(message)) {
|
||||
void finishHelpFromContent(message, sender, deps);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isHelpQueryMessage(message)) {
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId === undefined) {
|
||||
sendResponse({ active: false });
|
||||
return false;
|
||||
}
|
||||
void (async () => {
|
||||
const help = await findHelpForTab(tabId, deps);
|
||||
if (!help) {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
help.overlayTabIds.add(tabId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: help.requestId,
|
||||
prompt: help.prompt,
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
})();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
async function finishHelpFromContent(
|
||||
message: HelpFinishMessage,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<void> {
|
||||
const help = activeHelpRequests.get(message.requestId);
|
||||
if (!help || help.settled) return;
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId !== undefined) {
|
||||
const match = await findHelpForTab(tabId, deps);
|
||||
if (match !== help) return;
|
||||
}
|
||||
finishHelp(help, {
|
||||
outcome: message.outcome,
|
||||
...(message.note ? { note: message.note } : {}),
|
||||
tab_id: help.primaryTabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
}
|
||||
|
||||
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
|
||||
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||
if (typeof tab.id !== "number") return;
|
||||
scheduleRearmForTab(tab.id, deps);
|
||||
};
|
||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||
};
|
||||
chrome.tabs.onCreated?.addListener(onCreated);
|
||||
chrome.tabs.onActivated?.addListener(onActivated);
|
||||
return () => {
|
||||
chrome.tabs.onCreated?.removeListener(onCreated);
|
||||
chrome.tabs.onActivated?.removeListener(onActivated);
|
||||
};
|
||||
}
|
||||
|
||||
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
|
||||
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
|
||||
|
||||
if (chrome.webNavigation?.onCompleted) {
|
||||
const completedListener = (
|
||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
onMainFrameComplete(details.tabId);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||
}
|
||||
|
||||
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||
if (info.status !== "complete") return;
|
||||
onMainFrameComplete(tabId);
|
||||
};
|
||||
chrome.tabs.onUpdated?.addListener(listener);
|
||||
return () => chrome.tabs.onUpdated?.removeListener(listener);
|
||||
}
|
||||
|
||||
async function evaluateCompletionCondition(
|
||||
condition: HelpCompletionCondition,
|
||||
help: ActiveHelpRequest,
|
||||
): Promise<boolean> {
|
||||
if (condition.url_contains || condition.url_matches) {
|
||||
let url = "";
|
||||
try {
|
||||
const tab = await help.deps.tabsApi.get(help.primaryTabId);
|
||||
url = tab.url ?? "";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
|
||||
if (condition.url_matches) {
|
||||
try {
|
||||
if (!new RegExp(condition.url_matches).test(url)) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
condition.selector_exists ||
|
||||
condition.selector_missing ||
|
||||
condition.text_exists ||
|
||||
condition.text_missing
|
||||
) {
|
||||
if (!help.deps.cdp) return false;
|
||||
const expression = `(() => {
|
||||
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
|
||||
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
|
||||
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
|
||||
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
|
||||
if (selectorExists && !document.querySelector(selectorExists)) return false;
|
||||
if (selectorMissing && document.querySelector(selectorMissing)) return false;
|
||||
const text = document.body ? document.body.innerText || "" : "";
|
||||
if (textExists && !text.includes(textExists)) return false;
|
||||
if (textMissing && text.includes(textMissing)) return false;
|
||||
return true;
|
||||
})()`;
|
||||
try {
|
||||
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
|
||||
help.primaryTabId,
|
||||
"Runtime.evaluate",
|
||||
{ expression, returnByValue: true },
|
||||
);
|
||||
if (result.result?.value !== true) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
|
||||
const criteria = help.completionCriteria;
|
||||
if (!criteria) return false;
|
||||
const any = criteria.any ?? [];
|
||||
const all = criteria.all ?? [];
|
||||
const hasAny = any.length > 0;
|
||||
const hasAll = all.length > 0;
|
||||
if (!hasAny && !hasAll) return false;
|
||||
|
||||
if (hasAll) {
|
||||
for (const condition of all) {
|
||||
if (!(await evaluateCompletionCondition(condition, help))) return false;
|
||||
}
|
||||
}
|
||||
if (hasAny) {
|
||||
for (const condition of any) {
|
||||
if (await evaluateCompletionCondition(condition, help)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function startCompletionPolling(help: ActiveHelpRequest): void {
|
||||
if (!help.completionCriteria) return;
|
||||
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
|
||||
const check = async () => {
|
||||
if (help.settled) return;
|
||||
const matched = await completionCriteriaMatches(help);
|
||||
const now = Date.now();
|
||||
if (!matched) {
|
||||
help.completionMatchedSince = null;
|
||||
return;
|
||||
}
|
||||
help.completionMatchedSince ??= now;
|
||||
if (now - help.completionMatchedSince < stableMs) return;
|
||||
finishHelp(help, {
|
||||
outcome: "completed",
|
||||
completed_by: "system",
|
||||
tab_id: help.primaryTabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
};
|
||||
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
|
||||
void check();
|
||||
}
|
||||
|
||||
export async function handleRequestHelp(
|
||||
manager: SessionManager,
|
||||
params: RequestHelpParams,
|
||||
deps: RequestHelpDeps = getDefaultDeps(),
|
||||
): Promise<RequestHelpResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "request_help");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (!params.prompt || typeof params.prompt !== "string") {
|
||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||
}
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
|
||||
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||
if (denied) return denied;
|
||||
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
|
||||
@@ -255,7 +680,6 @@ export async function handleRequestHelp(
|
||||
const notificationId = `bsk-help:${requestId}`;
|
||||
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
||||
|
||||
// OS notification (best-effort).
|
||||
if (deps.notifications) {
|
||||
const copy = deps.notificationCopy ?? {
|
||||
title: "BrowserSkill: Agent needs your help",
|
||||
@@ -272,85 +696,59 @@ export async function handleRequestHelp(
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
const cleanup = async () => {
|
||||
await clearRefTags(deps.cdp, tabId);
|
||||
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
|
||||
// Retract the overlay in case we resolved via timeout / abort.
|
||||
await deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
|
||||
.catch(() => {});
|
||||
};
|
||||
return new Promise<RequestHelpResult | RpcError>((resolve) => {
|
||||
const timeoutTimer = setTimeout(() => {
|
||||
finishHelp(help, {
|
||||
outcome: "timed_out",
|
||||
tab_id: help.primaryTabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
const request: HelpRequestMessage = {
|
||||
type: HELP_REQUEST,
|
||||
requestId,
|
||||
prompt: params.prompt,
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
selectors,
|
||||
timeoutMs,
|
||||
};
|
||||
|
||||
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
|
||||
|
||||
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
|
||||
let settled = false;
|
||||
const unwatchNav = deps.watchTabNavigation(tabId, () => {
|
||||
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
|
||||
});
|
||||
|
||||
const finish = (value: RequestHelpResult | RpcError) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer !== null) clearTimeout(timer);
|
||||
unwatchNav();
|
||||
deps.signal?.removeEventListener("abort", onAbort);
|
||||
void cleanup();
|
||||
resolveOuter(value);
|
||||
const help: ActiveHelpRequest = {
|
||||
ctx,
|
||||
requestId,
|
||||
primaryTabId: tabId,
|
||||
overlayTabIds: new Set(),
|
||||
prompt: params.prompt,
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
completionCriteria: params.completion_criteria,
|
||||
deps,
|
||||
settled: false,
|
||||
resolve,
|
||||
timeoutTimer,
|
||||
completionTimer: null,
|
||||
completionMatchedSince: null,
|
||||
abortHandler: null,
|
||||
};
|
||||
|
||||
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
|
||||
}, timeoutMs) as unknown as number;
|
||||
|
||||
if (deps.signal) {
|
||||
if (deps.signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
deps.signal.addEventListener("abort", onAbort, { once: true });
|
||||
const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
|
||||
help.abortHandler = onAbort;
|
||||
activeHelpRequests.set(requestId, help);
|
||||
if (deps.signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
startCompletionPolling(help);
|
||||
|
||||
deps
|
||||
.sendToTab(tabId, request)
|
||||
.then((reply) => {
|
||||
const res = reply as HelpResponseMessage | undefined;
|
||||
if (
|
||||
!res ||
|
||||
res.type !== HELP_RESPONSE ||
|
||||
(res.outcome !== "continued" && res.outcome !== "cancelled")
|
||||
) {
|
||||
finish({
|
||||
code: "protocol_error",
|
||||
message: `invalid help response from tab ${tabId}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
finish({
|
||||
outcome: res.outcome,
|
||||
...(res.note ? { note: res.note } : {}),
|
||||
tab_id: tabId,
|
||||
resolved_targets: resolvedTargets,
|
||||
void sendCurrentHelpOverlay(help)
|
||||
.then((legacyFinish) => {
|
||||
if (!legacyFinish || help.settled) return;
|
||||
finishHelp(help, {
|
||||
outcome: legacyFinish.outcome,
|
||||
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||
tab_id: help.primaryTabId,
|
||||
resolved_targets: help.resolvedTargets,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
finish({
|
||||
code: "protocol_error",
|
||||
message: `failed to show help overlay on tab ${tabId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
});
|
||||
.catch(() => {
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -514,7 +514,22 @@ export interface HelpTarget {
|
||||
selector?: string;
|
||||
}
|
||||
|
||||
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated";
|
||||
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated";
|
||||
|
||||
export interface HelpCompletionCondition {
|
||||
url_contains?: string;
|
||||
url_matches?: string;
|
||||
selector_exists?: string;
|
||||
selector_missing?: string;
|
||||
text_exists?: string;
|
||||
text_missing?: string;
|
||||
}
|
||||
|
||||
export interface HelpCompletionCriteria {
|
||||
any?: HelpCompletionCondition[];
|
||||
all?: HelpCompletionCondition[];
|
||||
stable_for_ms?: number;
|
||||
}
|
||||
|
||||
export interface ResolvedTarget {
|
||||
matched: boolean;
|
||||
@@ -528,11 +543,13 @@ export interface RequestHelpParams {
|
||||
prompt: string;
|
||||
title?: string;
|
||||
targets?: HelpTarget[];
|
||||
completion_criteria?: HelpCompletionCriteria;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface RequestHelpResult {
|
||||
outcome: HelpOutcome;
|
||||
completed_by?: "system";
|
||||
note?: string;
|
||||
tab_id: number;
|
||||
resolved_targets?: ResolvedTarget[];
|
||||
|
||||
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
acts. The result `outcome` is one of:
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
@@ -7,7 +7,9 @@ use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{HelpTarget, RequestHelpParams, RequestHelpResult};
|
||||
use bsk_protocol::tools::{
|
||||
HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult,
|
||||
};
|
||||
use clap::Args;
|
||||
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
@@ -41,6 +43,10 @@ pub struct RequestHelpArgs {
|
||||
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
|
||||
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
|
||||
/// JSON success detector, e.g. '{"any":[{"url_contains":"/dashboard"}],"stable_for_ms":1000}'.
|
||||
#[arg(long = "completion-criteria")]
|
||||
pub completion_criteria: Option<String>,
|
||||
}
|
||||
|
||||
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
|
||||
@@ -68,6 +74,12 @@ pub fn parse_target(raw: &str) -> HelpTarget {
|
||||
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
|
||||
let completion_criteria = args
|
||||
.completion_criteria
|
||||
.as_deref()
|
||||
.map(serde_json::from_str::<HelpCompletionCriteria>)
|
||||
.transpose()
|
||||
.context("parse --completion-criteria JSON")?;
|
||||
let params = RequestHelpParams {
|
||||
session_id: args.session,
|
||||
tab_id: args.tab_id,
|
||||
@@ -78,6 +90,7 @@ pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
||||
} else {
|
||||
Some(targets)
|
||||
},
|
||||
completion_criteria,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply = call(info.sock_path, params, args.timeout)?;
|
||||
|
||||
@@ -7,6 +7,17 @@
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"completion_criteria": {
|
||||
"description": "Optional explicit success detector. Navigation by itself is not a completion signal; these criteria must match before the tool can auto-complete without the user clicking Done.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HelpCompletionCriteria"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"prompt": {
|
||||
"description": "Message shown to the user explaining what to do.",
|
||||
"type": "string"
|
||||
@@ -50,6 +61,78 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HelpCompletionCondition": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"selector_exists": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector_missing": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"text_exists": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"text_missing": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url_contains": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url_matches": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"HelpCompletionCriteria": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"all": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/HelpCompletionCondition"
|
||||
}
|
||||
},
|
||||
"any": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/HelpCompletionCondition"
|
||||
}
|
||||
},
|
||||
"stable_for_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"HelpTarget": {
|
||||
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
|
||||
"type": "object",
|
||||
|
||||
@@ -7,6 +7,16 @@
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"completed_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HelpCompletedBy"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"note": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -31,10 +41,16 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HelpCompletedBy": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"system"
|
||||
]
|
||||
},
|
||||
"HelpOutcome": {
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "User completed the step and clicked Continue.",
|
||||
"description": "User completed the step and clicked Done / return control.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"continued"
|
||||
@@ -55,7 +71,14 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "The page navigated while waiting (full reload or SPA URL change).",
|
||||
"description": "Explicit success criteria matched while the user had control.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"completed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "The page navigated while waiting (full reload or SPA URL change). Deprecated: navigation alone should not end a help request.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigated"
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
//! Lets an agent pause and ask the human to complete an in-page step
|
||||
//! (captcha, login, confirmation). The extension brings the target tab
|
||||
//! to the foreground, highlights the requested components, and blocks
|
||||
//! until the user clicks Continue / Cancel (or the call times out).
|
||||
//! until the user clicks Done / Cancel, explicit completion criteria match,
|
||||
//! or the call times out.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -41,6 +42,11 @@ pub struct RequestHelpParams {
|
||||
/// Optional components to scroll to + flash-highlight.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub targets: Option<Vec<HelpTarget>>,
|
||||
/// Optional explicit success detector. Navigation by itself is not a
|
||||
/// completion signal; these criteria must match before the tool can
|
||||
/// auto-complete without the user clicking Done.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completion_criteria: Option<HelpCompletionCriteria>,
|
||||
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
|
||||
/// enforced daemon-side via the generic tool timeout.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -48,16 +54,46 @@ pub struct RequestHelpParams {
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HelpCompletionCriteria {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub any: Option<Vec<HelpCompletionCondition>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub all: Option<Vec<HelpCompletionCondition>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 0))]
|
||||
pub stable_for_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HelpCompletionCondition {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url_contains: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url_matches: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector_exists: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector_missing: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_exists: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_missing: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HelpOutcome {
|
||||
/// User completed the step and clicked Continue.
|
||||
/// User completed the step and clicked Done / return control.
|
||||
Continued,
|
||||
/// User declined / aborted via the overlay's Cancel button.
|
||||
Cancelled,
|
||||
/// The wait expired before the user acted.
|
||||
TimedOut,
|
||||
/// Explicit success criteria matched while the user had control.
|
||||
Completed,
|
||||
/// The page navigated while waiting (full reload or SPA URL change).
|
||||
/// Deprecated: navigation alone should not end a help request.
|
||||
Navigated,
|
||||
}
|
||||
|
||||
@@ -67,11 +103,18 @@ impl HelpOutcome {
|
||||
Self::Continued => "continued",
|
||||
Self::Cancelled => "cancelled",
|
||||
Self::TimedOut => "timed_out",
|
||||
Self::Completed => "completed",
|
||||
Self::Navigated => "navigated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HelpCompletedBy {
|
||||
System,
|
||||
}
|
||||
|
||||
/// Per-target resolution status echoed back so the agent can tell which
|
||||
/// `ref` / `selector` actually matched a live element.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -87,6 +130,8 @@ pub struct ResolvedTarget {
|
||||
pub struct RequestHelpResult {
|
||||
pub outcome: HelpOutcome,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completed_by: Option<HelpCompletedBy>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub note: Option<String>,
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -123,6 +168,7 @@ mod tests {
|
||||
prompt: "log in".into(),
|
||||
title: None,
|
||||
targets: None,
|
||||
completion_criteria: None,
|
||||
timeout_ms: None,
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
@@ -140,6 +186,10 @@ mod tests {
|
||||
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
|
||||
json!("timed_out")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(HelpOutcome::Completed).unwrap(),
|
||||
json!("completed")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(HelpOutcome::Continued).unwrap(),
|
||||
json!("continued")
|
||||
@@ -158,6 +208,7 @@ mod tests {
|
||||
fn result_round_trips() {
|
||||
let r = RequestHelpResult {
|
||||
outcome: HelpOutcome::Continued,
|
||||
completed_by: None,
|
||||
note: Some("done".into()),
|
||||
tab_id: 7,
|
||||
resolved_targets: Some(vec![ResolvedTarget {
|
||||
|
||||
@@ -74,10 +74,11 @@
|
||||
},
|
||||
"helpRequest": {
|
||||
"title": "Agent needs your help",
|
||||
"compactStatus": "Agent is still waiting for you to finish this step",
|
||||
"noteLabel": "Optional note back to the agent",
|
||||
"notePlaceholder": "Add a note for the agent (optional)",
|
||||
"continue": "Continue",
|
||||
"cancel": "Cancel",
|
||||
"continue": "Done, return control",
|
||||
"cancel": "Cancel request",
|
||||
"collapse": "Collapse",
|
||||
"expand": "Expand",
|
||||
"dragHandle": "Drag to move",
|
||||
|
||||
@@ -74,10 +74,11 @@
|
||||
},
|
||||
"helpRequest": {
|
||||
"title": "Agent 需要你的帮助",
|
||||
"compactStatus": "当前 Agent 仍在等待你完成此步骤",
|
||||
"noteLabel": "给 Agent 的备注(可选)",
|
||||
"notePlaceholder": "给 Agent 留言(可选)",
|
||||
"continue": "继续",
|
||||
"cancel": "取消",
|
||||
"continue": "完成并交还控制权",
|
||||
"cancel": "取消请求",
|
||||
"collapse": "折叠",
|
||||
"expand": "展开",
|
||||
"dragHandle": "拖动以移动",
|
||||
|
||||
+10
-4
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
acts. The result `outcome` is one of:
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
Reference in New Issue
Block a user