mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Merge pull request #43 from Tencent/fix/early-control-return
fix(extension): fix early control return and preserve request-help handoff across tabs LGTM
This commit is contained in:
@@ -15,6 +15,8 @@ export interface HelpRequestData {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
/** Custom overlay title; falls back to i18n when omitted. */
|
/** Custom overlay title; falls back to i18n when omitted. */
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Full task UI on the subject tab; compact status UI on related tabs. */
|
||||||
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight. */
|
/** CSS selectors to scroll to + flash-highlight. */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
onContinue: (note: string) => void;
|
onContinue: (note: string) => void;
|
||||||
@@ -174,6 +176,8 @@ function measure(selectors: string[]): Box[] {
|
|||||||
|
|
||||||
export function HelpRequestOverlay({ request }: Props) {
|
export function HelpRequestOverlay({ request }: Props) {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
|
const isCompact = request?.displayMode === "compact";
|
||||||
|
const effectiveSelectors = isCompact ? [] : (request?.selectors ?? []);
|
||||||
const [note, setNote] = useState("");
|
const [note, setNote] = useState("");
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||||
@@ -207,8 +211,9 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
// Scroll the first matched target into view once per distinct request id.
|
// Scroll the first matched target into view once per distinct request id.
|
||||||
useLayoutEffect(() => {
|
useLayoutEffect(() => {
|
||||||
if (!request) return;
|
if (!request) return;
|
||||||
|
if (isCompact) return;
|
||||||
if (lastScrolledIdRef.current === request.id) return;
|
if (lastScrolledIdRef.current === request.id) return;
|
||||||
for (const sel of request.selectors) {
|
for (const sel of effectiveSelectors) {
|
||||||
const el = querySelectorSafe(sel);
|
const el = querySelectorSafe(sel);
|
||||||
if (el) {
|
if (el) {
|
||||||
lastScrolledIdRef.current = request.id;
|
lastScrolledIdRef.current = request.id;
|
||||||
@@ -216,7 +221,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, [request]);
|
}, [request, isCompact, effectiveSelectors]);
|
||||||
|
|
||||||
// Keep highlight boxes aligned with the page as it scrolls / resizes.
|
// Keep highlight boxes aligned with the page as it scrolls / resizes.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -225,7 +230,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const update = () => {
|
const update = () => {
|
||||||
const nextBoxes = measure(request.selectors);
|
const nextBoxes = measure(effectiveSelectors);
|
||||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||||
};
|
};
|
||||||
update();
|
update();
|
||||||
@@ -237,7 +242,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
window.removeEventListener("resize", update);
|
window.removeEventListener("resize", update);
|
||||||
window.clearInterval(interval);
|
window.clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [request]);
|
}, [request, effectiveSelectors]);
|
||||||
|
|
||||||
const onHeaderPointerDown = useCallback(
|
const onHeaderPointerDown = useCallback(
|
||||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||||
@@ -294,8 +299,12 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
setPanelPos(null);
|
setPanelPos(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (isCompact) {
|
||||||
|
setPanelPos(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (userMovedRef.current) return;
|
if (userMovedRef.current) return;
|
||||||
const measured = boxes.length > 0 ? boxes : measure(request.selectors);
|
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||||
const anchor = unionBox(measured);
|
const anchor = unionBox(measured);
|
||||||
const banner = bannerRef.current;
|
const banner = bannerRef.current;
|
||||||
if (!anchor || !banner) {
|
if (!anchor || !banner) {
|
||||||
@@ -319,7 +328,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||||
}, [request, boxes, collapsed]);
|
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||||
|
|
||||||
if (!request) return null;
|
if (!request) return null;
|
||||||
|
|
||||||
@@ -342,6 +351,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
z-index: 2147483647;
|
z-index: 2147483647;
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
justify-content: flex-start;
|
||||||
|
min-height: 0;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
width: var(--bsk-help-banner-width);
|
width: var(--bsk-help-banner-width);
|
||||||
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
||||||
@@ -359,16 +370,25 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
.bsk-help-banner[data-collapsed="true"] {
|
.bsk-help-banner[data-collapsed="true"] {
|
||||||
gap: 0;
|
gap: 0;
|
||||||
width: var(--bsk-help-banner-width);
|
width: var(--bsk-help-banner-width);
|
||||||
padding: 10px 12px;
|
padding: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] {
|
||||||
|
width: min(360px, calc(100vw - ${VIEWPORT_MARGIN * 2}px));
|
||||||
|
padding: 12px 14px;
|
||||||
|
gap: 0;
|
||||||
|
border-radius: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-drag-strip {
|
.bsk-help-drag-strip {
|
||||||
flex-shrink: 0;
|
position: absolute;
|
||||||
|
top: 6px;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
height: 12px;
|
height: 12px;
|
||||||
margin: -8px 0 -10px;
|
|
||||||
cursor: grab;
|
cursor: grab;
|
||||||
user-select: none;
|
user-select: none;
|
||||||
touch-action: none;
|
touch-action: none;
|
||||||
@@ -395,15 +415,20 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
||||||
margin: -6px 0 -8px;
|
top: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header {
|
.bsk-help-header {
|
||||||
|
flex-shrink: 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header {
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
.bsk-help-title {
|
.bsk-help-title {
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
@@ -414,31 +439,34 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
color: #111;
|
color: #111;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.bsk-help-compact-title {
|
||||||
|
flex: 1 1 auto;
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
font-size: 13px;
|
||||||
|
font-weight: 500;
|
||||||
|
color: #374151;
|
||||||
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-title {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-title {
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions {
|
.bsk-help-header-actions {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
display: flex;
|
display: none;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
margin-left: auto;
|
margin-left: auto;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
max-width: 0;
|
|
||||||
opacity: 0;
|
|
||||||
transform: translateX(8px);
|
|
||||||
pointer-events: none;
|
|
||||||
transition:
|
|
||||||
max-width var(--bsk-help-duration) var(--bsk-help-ease),
|
|
||||||
opacity 220ms var(--bsk-help-ease),
|
|
||||||
transform var(--bsk-help-duration) var(--bsk-help-ease);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
||||||
|
display: flex;
|
||||||
max-width: 280px;
|
max-width: 280px;
|
||||||
opacity: 1;
|
opacity: 1;
|
||||||
transform: translateX(0);
|
|
||||||
pointer-events: auto;
|
pointer-events: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -471,15 +499,13 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-body {
|
.bsk-help-body {
|
||||||
display: grid;
|
display: block;
|
||||||
grid-template-rows: 1fr;
|
flex-shrink: 0;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
grid-template-rows: 0fr;
|
|
||||||
width: 0;
|
width: 0;
|
||||||
height: 0;
|
height: 0;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -528,6 +554,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
.bsk-help-footer-actions {
|
.bsk-help-footer-actions {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
|
flex-wrap: wrap;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,7 +566,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
color: #4b5563;
|
color: #4b5563;
|
||||||
flex-shrink: 0;
|
min-width: 0;
|
||||||
|
white-space: normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-btn-continue {
|
.bsk-help-btn-continue {
|
||||||
@@ -550,16 +578,30 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: #fff;
|
color: #fff;
|
||||||
flex-shrink: 0;
|
min-width: 0;
|
||||||
|
white-space: normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions .bsk-help-btn-cancel,
|
.bsk-help-header-actions .bsk-help-btn-cancel,
|
||||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
padding: 6px 12px;
|
padding: 6px 12px;
|
||||||
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
padding: 6px 14px;
|
padding: 6px 14px;
|
||||||
|
max-width: 180px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions {
|
||||||
|
display: flex;
|
||||||
|
max-width: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-cancel,
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
|
padding: 6px 10px;
|
||||||
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
||||||
@@ -584,29 +626,31 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
`}</style>
|
`}</style>
|
||||||
|
|
||||||
{boxes.map((b, i) => (
|
{!isCompact &&
|
||||||
<div
|
boxes.map((b, i) => (
|
||||||
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
|
<div
|
||||||
key={i}
|
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
|
||||||
data-slot="help-highlight"
|
key={i}
|
||||||
style={{
|
data-slot="help-highlight"
|
||||||
position: "fixed",
|
style={{
|
||||||
top: b.top,
|
position: "fixed",
|
||||||
left: b.left,
|
top: b.top,
|
||||||
width: b.width,
|
left: b.left,
|
||||||
height: b.height,
|
width: b.width,
|
||||||
borderRadius: 6,
|
height: b.height,
|
||||||
zIndex: 2147483646,
|
borderRadius: 6,
|
||||||
pointerEvents: "none",
|
zIndex: 2147483646,
|
||||||
animation: "bsk-help-flash 1.2s ease-in-out infinite",
|
pointerEvents: "none",
|
||||||
}}
|
animation: "bsk-help-flash 1.2s ease-in-out infinite",
|
||||||
/>
|
}}
|
||||||
))}
|
/>
|
||||||
|
))}
|
||||||
|
|
||||||
<div
|
<div
|
||||||
ref={bannerRef}
|
ref={bannerRef}
|
||||||
data-slot="help-request-banner"
|
data-slot="help-request-banner"
|
||||||
data-collapsed={collapsed ? "true" : "false"}
|
data-collapsed={collapsed ? "true" : "false"}
|
||||||
|
data-display-mode={isCompact ? "compact" : "full"}
|
||||||
data-anchored={panelPos ? "true" : "false"}
|
data-anchored={panelPos ? "true" : "false"}
|
||||||
data-dragging={dragging ? "true" : "false"}
|
data-dragging={dragging ? "true" : "false"}
|
||||||
data-placement={panelPos?.placement}
|
data-placement={panelPos?.placement}
|
||||||
@@ -629,15 +673,17 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
: undefined
|
: undefined
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<div
|
{!isCompact && (
|
||||||
data-slot="help-drag-handle"
|
<div
|
||||||
className="bsk-help-drag-strip"
|
data-slot="help-drag-handle"
|
||||||
role="img"
|
className="bsk-help-drag-strip"
|
||||||
aria-label={t("helpRequest.dragHandle")}
|
role="img"
|
||||||
onPointerDown={onHeaderPointerDown}
|
aria-label={t("helpRequest.dragHandle")}
|
||||||
>
|
onPointerDown={onHeaderPointerDown}
|
||||||
<span className="bsk-help-drag-pill" aria-hidden />
|
>
|
||||||
</div>
|
<span className="bsk-help-drag-pill" aria-hidden />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="bsk-help-header">
|
<div className="bsk-help-header">
|
||||||
<img
|
<img
|
||||||
@@ -645,77 +691,87 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
alt="browser-skill"
|
alt="browser-skill"
|
||||||
style={{ width: 22, height: 22, borderRadius: 4 }}
|
style={{ width: 22, height: 22, borderRadius: 4 }}
|
||||||
/>
|
/>
|
||||||
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
|
{isCompact ? (
|
||||||
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
|
<span className="bsk-help-compact-title">{t("helpRequest.compactStatus")}</span>
|
||||||
|
) : (
|
||||||
|
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
|
||||||
|
)}
|
||||||
|
{!isCompact && (
|
||||||
|
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
data-slot={collapsed ? "help-cancel-button" : undefined}
|
||||||
|
className="bsk-help-btn-cancel"
|
||||||
|
tabIndex={collapsed ? 0 : -1}
|
||||||
|
onClick={() => request.onCancel()}
|
||||||
|
>
|
||||||
|
{t("helpRequest.cancel")}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
data-slot={collapsed ? "help-continue-button" : undefined}
|
||||||
|
className="bsk-help-btn-continue"
|
||||||
|
tabIndex={collapsed ? 0 : -1}
|
||||||
|
onClick={() => request.onContinue(note)}
|
||||||
|
>
|
||||||
|
{t("helpRequest.continue")}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{!isCompact && (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
data-slot={collapsed ? "help-cancel-button" : undefined}
|
data-slot="help-collapse-toggle"
|
||||||
className="bsk-help-btn-cancel"
|
className="bsk-help-collapse-toggle"
|
||||||
tabIndex={collapsed ? 0 : -1}
|
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
|
||||||
onClick={() => request.onCancel()}
|
aria-expanded={!collapsed}
|
||||||
|
onClick={() => setCollapsed((c) => !c)}
|
||||||
>
|
>
|
||||||
{t("helpRequest.cancel")}
|
<span className="bsk-help-collapse-icon">
|
||||||
|
<RiArrowDownSLine size={20} aria-hidden />
|
||||||
|
</span>
|
||||||
</button>
|
</button>
|
||||||
<button
|
)}
|
||||||
type="button"
|
|
||||||
data-slot={collapsed ? "help-continue-button" : undefined}
|
|
||||||
className="bsk-help-btn-continue"
|
|
||||||
tabIndex={collapsed ? 0 : -1}
|
|
||||||
onClick={() => request.onContinue(note)}
|
|
||||||
>
|
|
||||||
{t("helpRequest.continue")}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
data-slot="help-collapse-toggle"
|
|
||||||
className="bsk-help-collapse-toggle"
|
|
||||||
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
|
|
||||||
aria-expanded={!collapsed}
|
|
||||||
onClick={() => setCollapsed((c) => !c)}
|
|
||||||
>
|
|
||||||
<span className="bsk-help-collapse-icon">
|
|
||||||
<RiArrowDownSLine size={20} aria-hidden />
|
|
||||||
</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="bsk-help-body" aria-hidden={collapsed}>
|
{!isCompact && (
|
||||||
<div className="bsk-help-body-inner">
|
<div className="bsk-help-body" aria-hidden={collapsed}>
|
||||||
<div className="bsk-help-body-content">
|
<div className="bsk-help-body-inner">
|
||||||
<p className="bsk-help-prompt">{request.prompt}</p>
|
<div className="bsk-help-body-content">
|
||||||
<input
|
<p className="bsk-help-prompt">{request.prompt}</p>
|
||||||
type="text"
|
<input
|
||||||
className="bsk-help-note-input"
|
type="text"
|
||||||
aria-label={t("helpRequest.noteLabel")}
|
className="bsk-help-note-input"
|
||||||
placeholder={t("helpRequest.notePlaceholder")}
|
aria-label={t("helpRequest.noteLabel")}
|
||||||
value={note}
|
placeholder={t("helpRequest.notePlaceholder")}
|
||||||
tabIndex={collapsed ? -1 : 0}
|
value={note}
|
||||||
onChange={(e) => setNote(e.target.value)}
|
|
||||||
/>
|
|
||||||
<div className="bsk-help-footer-actions">
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
data-slot={collapsed ? undefined : "help-cancel-button"}
|
|
||||||
className="bsk-help-btn-cancel"
|
|
||||||
tabIndex={collapsed ? -1 : 0}
|
tabIndex={collapsed ? -1 : 0}
|
||||||
onClick={() => request.onCancel()}
|
onChange={(e) => setNote(e.target.value)}
|
||||||
>
|
/>
|
||||||
{t("helpRequest.cancel")}
|
<div className="bsk-help-footer-actions">
|
||||||
</button>
|
<button
|
||||||
<button
|
type="button"
|
||||||
type="button"
|
data-slot={collapsed ? undefined : "help-cancel-button"}
|
||||||
data-slot={collapsed ? undefined : "help-continue-button"}
|
className="bsk-help-btn-cancel"
|
||||||
className="bsk-help-btn-continue"
|
tabIndex={collapsed ? -1 : 0}
|
||||||
tabIndex={collapsed ? -1 : 0}
|
onClick={() => request.onCancel()}
|
||||||
onClick={() => request.onContinue(note)}
|
>
|
||||||
>
|
{t("helpRequest.cancel")}
|
||||||
{t("helpRequest.continue")}
|
</button>
|
||||||
</button>
|
<button
|
||||||
|
type="button"
|
||||||
|
data-slot={collapsed ? undefined : "help-continue-button"}
|
||||||
|
className="bsk-help-btn-continue"
|
||||||
|
tabIndex={collapsed ? -1 : 0}
|
||||||
|
onClick={() => request.onContinue(note)}
|
||||||
|
>
|
||||||
|
{t("helpRequest.continue")}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
)}
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -38,6 +38,31 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders a compact status without full request controls", () => {
|
||||||
|
const el = document.createElement("div");
|
||||||
|
el.id = "login";
|
||||||
|
el.getBoundingClientRect = () =>
|
||||||
|
({ top: 10, left: 10, width: 100, height: 40, right: 110, bottom: 50 }) as DOMRect;
|
||||||
|
document.body.append(el);
|
||||||
|
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({
|
||||||
|
displayMode: "compact",
|
||||||
|
selectors: ["#login"],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText(i18n.t("helpRequest.compactStatus", { ns: "extension" }))).toBeTruthy();
|
||||||
|
expect(screen.queryByText("Please complete the captcha")).toBeNull();
|
||||||
|
expect(screen.queryByRole("textbox")).toBeNull();
|
||||||
|
expect(screen.queryByLabelText(i18n.t("helpRequest.collapse", { ns: "extension" }))).toBeNull();
|
||||||
|
expect(container.querySelector("[data-slot='help-continue-button']")).toBeNull();
|
||||||
|
expect(container.querySelector("[data-slot='help-cancel-button']")).toBeNull();
|
||||||
|
expect(document.querySelectorAll("[data-slot='help-highlight']").length).toBe(0);
|
||||||
|
|
||||||
|
el.remove();
|
||||||
|
});
|
||||||
|
|
||||||
it("renders a custom title when provided", () => {
|
it("renders a custom title when provided", () => {
|
||||||
renderOverlay(baseRequest({ title: "Verify your identity" }));
|
renderOverlay(baseRequest({ title: "Verify your identity" }));
|
||||||
expect(screen.getByText("Verify your identity")).toBeTruthy();
|
expect(screen.getByText("Verify your identity")).toBeTruthy();
|
||||||
@@ -204,6 +229,15 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(styles).toContain("width: 0");
|
expect(styles).toContain("width: 0");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses natural body height in the expanded layout", () => {
|
||||||
|
const { container } = renderOverlay(baseRequest());
|
||||||
|
const styles = overlayStyles(container);
|
||||||
|
|
||||||
|
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
|
||||||
|
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
|
||||||
|
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the same banner width when collapsed", () => {
|
it("keeps the same banner width when collapsed", () => {
|
||||||
const { container } = renderOverlay(baseRequest());
|
const { container } = renderOverlay(baseRequest());
|
||||||
const styles = overlayStyles(container);
|
const styles = overlayStyles(container);
|
||||||
|
|||||||
@@ -14,10 +14,14 @@ import {
|
|||||||
type RecordCaptureController,
|
type RecordCaptureController,
|
||||||
} from "@/content/record-capture";
|
} from "@/content/record-capture";
|
||||||
import {
|
import {
|
||||||
HELP_RESPONSE,
|
HELP_ACK,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
|
type HelpAckMessage,
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
|
type HelpFinishMessage,
|
||||||
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
type HelpResponseMessage,
|
|
||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
@@ -55,8 +59,6 @@ export default defineContentScript({
|
|||||||
if (window.top !== window) return;
|
if (window.top !== window) return;
|
||||||
|
|
||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
|
|
||||||
null;
|
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
@@ -116,8 +118,7 @@ export default defineContentScript({
|
|||||||
function resetAgentOverlayState(sessionId: string) {
|
function resetAgentOverlayState(sessionId: string) {
|
||||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||||
if (previousHelp) {
|
if (previousHelp) {
|
||||||
activeHelpRespond?.("cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
activeHelpRespond = null;
|
|
||||||
}
|
}
|
||||||
recordCapture?.dispose();
|
recordCapture?.dispose();
|
||||||
recordCapture = null;
|
recordCapture = null;
|
||||||
@@ -156,7 +157,7 @@ export default defineContentScript({
|
|||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
|
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||||
) => {
|
) => {
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
const needsAsync = handleRecordContentMessage(
|
||||||
@@ -220,41 +221,30 @@ export default defineContentScript({
|
|||||||
if (isHelpCancelMessage(message)) {
|
if (isHelpCancelMessage(message)) {
|
||||||
const state = overlays.snapshot();
|
const state = overlays.snapshot();
|
||||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||||
activeHelpRespond?.("cancelled");
|
overlays.clearAgentHelpRequest(message.requestId);
|
||||||
|
renderOverlay();
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isHelpRequestMessage(message)) {
|
if (isHelpRequestMessage(message)) {
|
||||||
const helpMsg = message as HelpRequestMessage;
|
const helpMsg = message as HelpRequestMessage;
|
||||||
let responded = false;
|
|
||||||
const respond = (outcome: "continued" | "cancelled", note?: string) => {
|
|
||||||
if (responded) return;
|
|
||||||
responded = true;
|
|
||||||
const reply: HelpResponseMessage = {
|
|
||||||
type: HELP_RESPONSE,
|
|
||||||
outcome,
|
|
||||||
...(note ? { note } : {}),
|
|
||||||
};
|
|
||||||
sendResponse(reply);
|
|
||||||
activeHelpRespond = null;
|
|
||||||
overlays.clearAgentHelpRequest(helpMsg.requestId);
|
|
||||||
renderOverlay();
|
|
||||||
};
|
|
||||||
const previousHelp = overlays.setAgentHelpRequest({
|
const previousHelp = overlays.setAgentHelpRequest({
|
||||||
id: helpMsg.requestId,
|
id: helpMsg.requestId,
|
||||||
prompt: helpMsg.prompt,
|
prompt: helpMsg.prompt,
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||||
|
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||||
selectors: helpMsg.selectors,
|
selectors: helpMsg.selectors,
|
||||||
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
|
onContinue: (note: string) =>
|
||||||
onCancel: () => respond("cancelled"),
|
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||||
});
|
});
|
||||||
if (previousHelp) {
|
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||||
activeHelpRespond?.("cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
activeHelpRespond = respond;
|
|
||||||
renderOverlay();
|
renderOverlay();
|
||||||
return true; // async sendResponse
|
sendResponse({ type: HELP_ACK, ok: true });
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (message.type === "borrow-request") {
|
if (message.type === "borrow-request") {
|
||||||
@@ -282,9 +272,60 @@ export default defineContentScript({
|
|||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
async function sendHelpFinish(
|
||||||
|
requestId: string,
|
||||||
|
outcome: "continued" | "cancelled",
|
||||||
|
note?: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const msg: HelpFinishMessage = {
|
||||||
|
type: HELP_FINISH,
|
||||||
|
requestId,
|
||||||
|
outcome,
|
||||||
|
...(note ? { note } : {}),
|
||||||
|
};
|
||||||
|
overlays.clearAgentHelpRequest(requestId);
|
||||||
|
renderOverlay();
|
||||||
|
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||||
|
console.debug("[bsk overlay] help finish failed", err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||||
|
overlays.setAgentHelpRequest({
|
||||||
|
id: helpMsg.requestId,
|
||||||
|
prompt: helpMsg.prompt,
|
||||||
|
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||||
|
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||||
|
selectors: helpMsg.selectors,
|
||||||
|
onContinue: (note: string) =>
|
||||||
|
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function queryActiveHelpWithRetry(): Promise<boolean> {
|
||||||
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const helpQuery = (await chrome.runtime.sendMessage({
|
||||||
|
type: HELP_QUERY,
|
||||||
|
})) as HelpQueryResponse | undefined;
|
||||||
|
if (helpQuery?.active && helpQuery.request) {
|
||||||
|
mountHelpRequest(helpQuery.request);
|
||||||
|
renderOverlay();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk overlay] help query failed", err);
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
async function syncAgentOverlay(): Promise<void> {
|
async function syncAgentOverlay(): Promise<void> {
|
||||||
if (!(await anySessionLive())) return;
|
if (!(await anySessionLive())) return;
|
||||||
try {
|
try {
|
||||||
|
const helpActive = await queryActiveHelpWithRetry();
|
||||||
const reply = (await chrome.runtime.sendMessage({
|
const reply = (await chrome.runtime.sendMessage({
|
||||||
kind: OVERLAY_MSG_WHO_AM_I,
|
kind: OVERLAY_MSG_WHO_AM_I,
|
||||||
})) as OverlayWhoAmIResponse | undefined;
|
})) as OverlayWhoAmIResponse | undefined;
|
||||||
@@ -319,6 +360,10 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!helpActive && overlays.snapshot().activeHelp === null) {
|
||||||
|
void queryActiveHelpWithRetry();
|
||||||
|
}
|
||||||
|
|
||||||
overlays.activateAgentSession(reply.sessionId);
|
overlays.activateAgentSession(reply.sessionId);
|
||||||
renderOverlay();
|
renderOverlay();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -1,10 +1,17 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
|
HELP_ACK,
|
||||||
HELP_CANCEL,
|
HELP_CANCEL,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
HELP_RESPONSE,
|
HELP_RESPONSE,
|
||||||
|
isHelpAckMessage,
|
||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
|
isHelpFinishMessage,
|
||||||
|
isHelpQueryMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
|
isHelpResponseMessage,
|
||||||
} from "../help-bridge";
|
} from "../help-bridge";
|
||||||
|
|
||||||
describe("help-bridge", () => {
|
describe("help-bridge", () => {
|
||||||
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
|
|||||||
expect(HELP_REQUEST).toBe("bsk-help-request");
|
expect(HELP_REQUEST).toBe("bsk-help-request");
|
||||||
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
||||||
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
||||||
|
expect(HELP_ACK).toBe("bsk-help-ack");
|
||||||
|
expect(HELP_QUERY).toBe("bsk-help-query");
|
||||||
|
expect(HELP_FINISH).toBe("bsk-help-finish");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("type-guards a help-request message", () => {
|
it("type-guards a help-request message", () => {
|
||||||
@@ -20,6 +30,7 @@ describe("help-bridge", () => {
|
|||||||
type: HELP_REQUEST,
|
type: HELP_REQUEST,
|
||||||
requestId: "r1",
|
requestId: "r1",
|
||||||
prompt: "log in",
|
prompt: "log in",
|
||||||
|
displayMode: "compact",
|
||||||
selectors: ["#login"],
|
selectors: ["#login"],
|
||||||
timeoutMs: 1000,
|
timeoutMs: 1000,
|
||||||
}),
|
}),
|
||||||
@@ -55,4 +66,18 @@ describe("help-bridge", () => {
|
|||||||
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
||||||
expect(isHelpCancelMessage(null)).toBe(false);
|
expect(isHelpCancelMessage(null)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("type-guards lifecycle messages", () => {
|
||||||
|
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
|
||||||
|
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
|
||||||
|
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,12 +6,15 @@
|
|||||||
* Mirrors the borrow-confirmation message style (see
|
* Mirrors the borrow-confirmation message style (see
|
||||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||||
* the content script replies once the user clicks Continue / Cancel.
|
* the content script acks display, then later reports Done / Cancel.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const HELP_REQUEST = "bsk-help-request";
|
export const HELP_REQUEST = "bsk-help-request";
|
||||||
export const HELP_RESPONSE = "bsk-help-response";
|
export const HELP_RESPONSE = "bsk-help-response";
|
||||||
export const HELP_CANCEL = "bsk-help-cancel";
|
export const HELP_CANCEL = "bsk-help-cancel";
|
||||||
|
export const HELP_ACK = "bsk-help-ack";
|
||||||
|
export const HELP_QUERY = "bsk-help-query";
|
||||||
|
export const HELP_FINISH = "bsk-help-finish";
|
||||||
|
|
||||||
export interface HelpRequestMessage {
|
export interface HelpRequestMessage {
|
||||||
type: typeof HELP_REQUEST;
|
type: typeof HELP_REQUEST;
|
||||||
@@ -19,11 +22,18 @@ export interface HelpRequestMessage {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
/** Custom overlay title; omitted when the extension should use its default. */
|
/** Custom overlay title; omitted when the extension should use its default. */
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Full task UI on the subject tab; compact status UI on related tabs. */
|
||||||
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HelpAckMessage {
|
||||||
|
type: typeof HELP_ACK;
|
||||||
|
ok: true;
|
||||||
|
}
|
||||||
|
|
||||||
export interface HelpResponseMessage {
|
export interface HelpResponseMessage {
|
||||||
type: typeof HELP_RESPONSE;
|
type: typeof HELP_RESPONSE;
|
||||||
outcome: "continued" | "cancelled";
|
outcome: "continued" | "cancelled";
|
||||||
@@ -35,6 +45,22 @@ export interface HelpCancelMessage {
|
|||||||
requestId: string;
|
requestId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HelpQueryMessage {
|
||||||
|
type: typeof HELP_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpQueryResponse {
|
||||||
|
active: boolean;
|
||||||
|
request?: Omit<HelpRequestMessage, "type">;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpFinishMessage {
|
||||||
|
type: typeof HELP_FINISH;
|
||||||
|
requestId: string;
|
||||||
|
outcome: "continued" | "cancelled";
|
||||||
|
note?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||||
if (typeof msg !== "object" || msg === null) {
|
if (typeof msg !== "object" || msg === null) {
|
||||||
return false;
|
return false;
|
||||||
@@ -45,6 +71,7 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
|||||||
typeof m.requestId === "string" &&
|
typeof m.requestId === "string" &&
|
||||||
typeof m.prompt === "string" &&
|
typeof m.prompt === "string" &&
|
||||||
(m.title === undefined || typeof m.title === "string") &&
|
(m.title === undefined || typeof m.title === "string") &&
|
||||||
|
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||||
Array.isArray(m.selectors) &&
|
Array.isArray(m.selectors) &&
|
||||||
m.selectors.every((selector) => typeof selector === "string") &&
|
m.selectors.every((selector) => typeof selector === "string") &&
|
||||||
typeof m.timeoutMs === "number"
|
typeof m.timeoutMs === "number"
|
||||||
@@ -58,3 +85,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
|
|||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === HELP_RESPONSE &&
|
||||||
|
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||||
|
(m.note === undefined || typeof m.note === "string")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === HELP_ACK && m.ok === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === HELP_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === HELP_FINISH &&
|
||||||
|
typeof m.requestId === "string" &&
|
||||||
|
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||||
|
(m.note === undefined || typeof m.note === "string")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,53 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
import type { RequestHelpParams } from "@/transport/types";
|
import type { RequestHelpParams } from "@/transport/types";
|
||||||
import { handleRequestHelp, type RequestHelpDeps } from "../human-loop";
|
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||||
|
|
||||||
|
function chromeEvent<T extends (...args: never[]) => unknown>() {
|
||||||
|
const listeners = new Set<T>();
|
||||||
|
return {
|
||||||
|
addListener: vi.fn((listener: T) => {
|
||||||
|
listeners.add(listener);
|
||||||
|
}),
|
||||||
|
removeListener: vi.fn((listener: T) => {
|
||||||
|
listeners.delete(listener);
|
||||||
|
}),
|
||||||
|
emit: (...args: Parameters<T>) => {
|
||||||
|
for (const listener of [...listeners]) listener(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function installHelpLifecycleChrome() {
|
||||||
|
const runtimeOnMessage =
|
||||||
|
chromeEvent<
|
||||||
|
(
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: unknown) => void,
|
||||||
|
) => unknown
|
||||||
|
>();
|
||||||
|
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
|
||||||
|
const tabsOnCreated = chromeEvent<(tab: chrome.tabs.Tab) => unknown>();
|
||||||
|
const tabsOnUpdated =
|
||||||
|
chromeEvent<(tabId: number, changeInfo: chrome.tabs.TabChangeInfo) => unknown>();
|
||||||
|
const webNavigationOnCompleted =
|
||||||
|
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
|
||||||
|
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: { onMessage: runtimeOnMessage },
|
||||||
|
tabs: { onActivated: tabsOnActivated, onCreated: tabsOnCreated, onUpdated: tabsOnUpdated },
|
||||||
|
webNavigation: { onCompleted: webNavigationOnCompleted },
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
runtimeOnMessage,
|
||||||
|
tabsOnActivated,
|
||||||
|
tabsOnCreated,
|
||||||
|
tabsOnUpdated,
|
||||||
|
webNavigationOnCompleted,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||||
const mgr = {
|
const mgr = {
|
||||||
@@ -27,14 +73,19 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
|
|||||||
windows: { update: vi.fn(async () => ({}) as never) },
|
windows: { update: vi.fn(async () => ({}) as never) },
|
||||||
activateTab: vi.fn(async () => {}),
|
activateTab: vi.fn(async () => {}),
|
||||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
||||||
watchTabNavigation: () => () => {},
|
|
||||||
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
||||||
notifications: null,
|
notifications: null,
|
||||||
|
autoAttachLifecycle: false,
|
||||||
...over,
|
...over,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleRequestHelp", () => {
|
describe("handleRequestHelp", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
resetHelpLifecycleForTests();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects unknown session", async () => {
|
it("rejects unknown session", async () => {
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
fakeManager("abcd", 99, 5),
|
fakeManager("abcd", 99, 5),
|
||||||
@@ -78,22 +129,29 @@ describe("handleRequestHelp", () => {
|
|||||||
expect(sentMsg.title).toBeUndefined();
|
expect(sentMsg.title).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns navigated when the tab navigates during the wait", async () => {
|
it("does not complete merely because the tab navigates during the wait", async () => {
|
||||||
const unwatch = vi.fn();
|
vi.useFakeTimers();
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
sendToTab: vi.fn(() => new Promise(() => {})),
|
sendToTab: vi.fn(() => new Promise(() => {})),
|
||||||
watchTabNavigation: vi.fn((_tabId, cb) => {
|
|
||||||
queueMicrotask(() => cb());
|
|
||||||
return unwatch;
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
try {
|
||||||
fakeManager("abcd", 99, 5),
|
const pending = handleRequestHelp(
|
||||||
baseParams({ tab_id: 5 }),
|
fakeManager("abcd", 99, 5),
|
||||||
deps,
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
);
|
deps,
|
||||||
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
|
);
|
||||||
expect(unwatch).toHaveBeenCalled();
|
await vi.advanceTimersByTimeAsync(5);
|
||||||
|
let settled = false;
|
||||||
|
void pending.then(() => {
|
||||||
|
settled = true;
|
||||||
|
});
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns timed_out when the wait expires", async () => {
|
it("returns timed_out when the wait expires", async () => {
|
||||||
@@ -115,6 +173,160 @@ describe("handleRequestHelp", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns completed when explicit completion criteria match", async () => {
|
||||||
|
const deps = baseDeps({
|
||||||
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||||
|
cdp: {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Runtime.evaluate") return { result: { value: true } };
|
||||||
|
return {};
|
||||||
|
}),
|
||||||
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
|
});
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({
|
||||||
|
tab_id: 5,
|
||||||
|
completion_criteria: {
|
||||||
|
any: [{ selector_exists: "#account-menu" }],
|
||||||
|
stable_for_ms: 0,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps user control active on new tabs without moving completion off the primary tab", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const chromeEvents = installHelpLifecycleChrome();
|
||||||
|
const sendToTab = vi.fn(async () => ({ type: "bsk-help-ack", ok: true }));
|
||||||
|
const deps = baseDeps({
|
||||||
|
autoAttachLifecycle: undefined,
|
||||||
|
sendToTab,
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(async (tabId: number) => ({
|
||||||
|
id: tabId,
|
||||||
|
windowId: 99,
|
||||||
|
active: tabId === 6,
|
||||||
|
url: tabId === 6 ? "https://app.example/reset/success" : "https://app.example/login",
|
||||||
|
})) as never,
|
||||||
|
query: vi.fn(async () => [{ id: 5, windowId: 99, active: true }] as never),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const pending = handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({
|
||||||
|
tab_id: 5,
|
||||||
|
timeout_ms: 1_000,
|
||||||
|
completion_criteria: {
|
||||||
|
any: [{ url_contains: "/reset/success" }],
|
||||||
|
stable_for_ms: 0,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(sendToTab).toHaveBeenCalledWith(5, expect.anything()));
|
||||||
|
|
||||||
|
chromeEvents.tabsOnActivated.emit({ tabId: 6, windowId: 99 });
|
||||||
|
await vi.advanceTimersByTimeAsync(200);
|
||||||
|
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(
|
||||||
|
6,
|
||||||
|
expect.objectContaining({
|
||||||
|
type: "bsk-help-request",
|
||||||
|
displayMode: "compact",
|
||||||
|
selectors: [],
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
await expect(pending).resolves.toMatchObject({
|
||||||
|
outcome: "timed_out",
|
||||||
|
tab_id: 5,
|
||||||
|
});
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(
|
||||||
|
6,
|
||||||
|
expect.objectContaining({ type: "bsk-help-cancel" }),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets same-window tabs query active help after content-script load", async () => {
|
||||||
|
const chromeEvents = installHelpLifecycleChrome();
|
||||||
|
const ac = new AbortController();
|
||||||
|
const cdpSend = vi.fn(async () => ({ root: { nodeId: 1 }, nodeIds: [] }));
|
||||||
|
const deps = baseDeps({
|
||||||
|
autoAttachLifecycle: undefined,
|
||||||
|
signal: ac.signal,
|
||||||
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||||
|
cdp: {
|
||||||
|
send: cdpSend,
|
||||||
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const pending = handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({ tab_id: 5, timeout_ms: 60_000 }),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(deps.sendToTab).toHaveBeenCalledWith(5, expect.anything()));
|
||||||
|
|
||||||
|
const sameWindowResponse = vi.fn();
|
||||||
|
chromeEvents.runtimeOnMessage.emit(
|
||||||
|
{ type: "bsk-help-query" },
|
||||||
|
{ tab: { id: 6 } as chrome.tabs.Tab },
|
||||||
|
sameWindowResponse,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(sameWindowResponse).toHaveBeenCalledWith({
|
||||||
|
active: true,
|
||||||
|
request: expect.objectContaining({
|
||||||
|
requestId: expect.any(String),
|
||||||
|
prompt: "log in",
|
||||||
|
displayMode: "compact",
|
||||||
|
selectors: [],
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const subjectResponse = vi.fn();
|
||||||
|
chromeEvents.runtimeOnMessage.emit(
|
||||||
|
{ type: "bsk-help-query" },
|
||||||
|
{ tab: { id: 5 } as chrome.tabs.Tab },
|
||||||
|
subjectResponse,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(subjectResponse).toHaveBeenCalledWith({
|
||||||
|
active: true,
|
||||||
|
request: expect.objectContaining({
|
||||||
|
requestId: expect.any(String),
|
||||||
|
prompt: "log in",
|
||||||
|
displayMode: "full",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
ac.abort();
|
||||||
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
|
expect(cdpSend).toHaveBeenCalledWith(
|
||||||
|
5,
|
||||||
|
"DOM.querySelectorAll",
|
||||||
|
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||||
|
);
|
||||||
|
expect(cdpSend).toHaveBeenCalledWith(
|
||||||
|
6,
|
||||||
|
"DOM.querySelectorAll",
|
||||||
|
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("tags ref targets via CDP and reports them matched", async () => {
|
it("tags ref targets via CDP and reports them matched", async () => {
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
@@ -190,38 +402,40 @@ describe("handleRequestHelp", () => {
|
|||||||
expect("code" in res).toBe(false);
|
expect("code" in res).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns protocol_error when sendToTab rejects", async () => {
|
it("keeps waiting when the initial overlay delivery rejects", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
sendToTab: vi.fn(async () => {
|
sendToTab: vi.fn(async () => {
|
||||||
throw new Error("no receiver");
|
throw new Error("no receiver");
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
try {
|
||||||
fakeManager("abcd", 99, 5),
|
const res = handleRequestHelp(
|
||||||
baseParams({ tab_id: 5 }),
|
fakeManager("abcd", 99, 5),
|
||||||
deps,
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
);
|
deps,
|
||||||
expect("code" in res && res.code).toBe("protocol_error");
|
);
|
||||||
|
await vi.advanceTimersByTimeAsync(20);
|
||||||
|
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns protocol_error for a malformed help response", async () => {
|
it("keeps waiting for explicit completion after a malformed help response", async () => {
|
||||||
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
vi.useFakeTimers();
|
||||||
const res1 = await handleRequestHelp(
|
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||||
fakeManager("abcd", 99, 5),
|
try {
|
||||||
baseParams({ tab_id: 5 }),
|
const res = handleRequestHelp(
|
||||||
undefinedReply,
|
fakeManager("abcd", 99, 5),
|
||||||
);
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
expect("code" in res1 && res1.code).toBe("protocol_error");
|
deps,
|
||||||
|
);
|
||||||
const badOutcome = baseDeps({
|
await vi.advanceTimersByTimeAsync(20);
|
||||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
|
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||||
});
|
} finally {
|
||||||
const res2 = await handleRequestHelp(
|
vi.useRealTimers();
|
||||||
fakeManager("abcd", 99, 5),
|
}
|
||||||
baseParams({ tab_id: 5 }),
|
|
||||||
badOutcome,
|
|
||||||
);
|
|
||||||
expect("code" in res2 && res2.code).toBe("protocol_error");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("reports selector match status from CDP", async () => {
|
it("reports selector match status from CDP", async () => {
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ import type {
|
|||||||
import { isRequestFrame } from "@/transport/types";
|
import { isRequestFrame } from "@/transport/types";
|
||||||
import { handleConsole } from "./console";
|
import { handleConsole } from "./console";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
@@ -394,7 +394,6 @@ export class ToolDispatcher {
|
|||||||
await chrome.tabs.update(tabId, { active: true });
|
await chrome.tabs.update(tabId, { active: true });
|
||||||
},
|
},
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
watchTabNavigation: defaultWatchTabNavigation,
|
|
||||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||||
notifications: makeHelpNotifications(),
|
notifications: makeHelpNotifications(),
|
||||||
notificationCopy: this.helpNotificationCopy?.(),
|
notificationCopy: this.helpNotificationCopy?.(),
|
||||||
|
|||||||
@@ -1,21 +1,24 @@
|
|||||||
// `tool.request_help` — pause and ask the human to act on a tab.
|
// `tool.request_help` — pause automation and let the human control a tab.
|
||||||
//
|
|
||||||
// Brings the target tab to the foreground, resolves `ref` targets to a
|
|
||||||
// temporary attribute selector via CDP, asks the content script to enter
|
|
||||||
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
|
|
||||||
// until the user clicks Continue / Cancel, the wait times out, or the
|
|
||||||
// daemon cancels the RPC.
|
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
HELP_ACK,
|
||||||
HELP_CANCEL,
|
HELP_CANCEL,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
HELP_RESPONSE,
|
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
|
type HelpFinishMessage,
|
||||||
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
type HelpResponseMessage,
|
isHelpAckMessage,
|
||||||
|
isHelpFinishMessage,
|
||||||
|
isHelpQueryMessage,
|
||||||
|
isHelpResponseMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
|
HelpCompletionCondition,
|
||||||
|
HelpCompletionCriteria,
|
||||||
HelpTarget,
|
HelpTarget,
|
||||||
RequestHelpParams,
|
RequestHelpParams,
|
||||||
RequestHelpResult,
|
RequestHelpResult,
|
||||||
@@ -34,53 +37,59 @@ import {
|
|||||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||||
|
|
||||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||||
/** Attribute the overlay highlights for resolved `ref` targets. */
|
|
||||||
const HELP_ATTR = "data-bsk-help";
|
const HELP_ATTR = "data-bsk-help";
|
||||||
|
const HELP_SEND_RETRIES = 3;
|
||||||
|
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||||
|
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||||
|
const HELP_REARM_MAX_ATTEMPTS = 12;
|
||||||
|
const HELP_REARM_RETRY_DELAY_MS = 400;
|
||||||
|
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
|
||||||
|
const COMPLETION_POLL_MS = 500;
|
||||||
|
|
||||||
export interface RequestHelpNotifications {
|
export interface RequestHelpNotifications {
|
||||||
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
||||||
clear(id: string): Promise<boolean>;
|
clear(id: string): Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TabNavigationUnsubscribe = () => void;
|
|
||||||
|
|
||||||
export interface RequestHelpDeps {
|
export interface RequestHelpDeps {
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
windows: {
|
windows: {
|
||||||
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
||||||
};
|
};
|
||||||
/** Activate a tab inside its window. */
|
|
||||||
activateTab(tabId: number): Promise<void>;
|
activateTab(tabId: number): Promise<void>;
|
||||||
/** Send the help-request to the tab's content script and await reply. */
|
|
||||||
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
||||||
/**
|
|
||||||
* Watch for navigation on `tabId` (full page load or SPA URL change).
|
|
||||||
* Invokes `onNavigated` once; returns unsubscribe.
|
|
||||||
*/
|
|
||||||
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
|
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
/** Pass `null` to skip OS notifications (tests). */
|
|
||||||
notifications: RequestHelpNotifications | null;
|
notifications: RequestHelpNotifications | null;
|
||||||
notificationCopy?: { title: string; body: string };
|
notificationCopy?: { title: string; body: string };
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
autoAttachLifecycle?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
|
interface ActiveHelpRequest {
|
||||||
export function defaultWatchTabNavigation(
|
ctx: SessionContext;
|
||||||
tabId: number,
|
requestId: string;
|
||||||
onNavigated: () => void,
|
primaryTabId: number;
|
||||||
): TabNavigationUnsubscribe {
|
overlayTabIds: Set<number>;
|
||||||
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
|
prompt: string;
|
||||||
if (updatedTabId !== tabId) return;
|
title?: string;
|
||||||
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
|
targets: HelpTarget[];
|
||||||
chrome.tabs.onUpdated.removeListener(listener);
|
selectors: string[];
|
||||||
onNavigated();
|
timeoutMs: number;
|
||||||
}
|
notificationId: string;
|
||||||
};
|
resolvedTargets?: ResolvedTarget[];
|
||||||
chrome.tabs.onUpdated.addListener(listener);
|
completionCriteria?: HelpCompletionCriteria;
|
||||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
deps: RequestHelpDeps;
|
||||||
|
settled: boolean;
|
||||||
|
resolve: (value: RequestHelpResult | RpcError) => void;
|
||||||
|
timeoutTimer: ReturnType<typeof setTimeout>;
|
||||||
|
completionTimer: ReturnType<typeof setInterval> | null;
|
||||||
|
completionMatchedSince: number | null;
|
||||||
|
abortHandler: (() => void) | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
|
||||||
|
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||||
|
|
||||||
let defaultDeps: RequestHelpDeps | null = null;
|
let defaultDeps: RequestHelpDeps | null = null;
|
||||||
function getDefaultDeps(): RequestHelpDeps {
|
function getDefaultDeps(): RequestHelpDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
@@ -91,7 +100,6 @@ function getDefaultDeps(): RequestHelpDeps {
|
|||||||
await chrome.tabs.update(tabId, { active: true });
|
await chrome.tabs.update(tabId, { active: true });
|
||||||
},
|
},
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
watchTabNavigation: defaultWatchTabNavigation,
|
|
||||||
notifications: {
|
notifications: {
|
||||||
create: (id, opts) =>
|
create: (id, opts) =>
|
||||||
new Promise((resolve, reject) =>
|
new Promise((resolve, reject) =>
|
||||||
@@ -113,11 +121,50 @@ function makeRequestId(tabId: number): string {
|
|||||||
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function sleep(ms: number): Promise<void> {
|
||||||
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
* content overlay can highlight it with a plain attribute selector.
|
}
|
||||||
* Returns the selector on success, or null when the ref can't resolve.
|
|
||||||
*/
|
function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequestMessage {
|
||||||
|
const isPrimaryTab = tabId === help.primaryTabId;
|
||||||
|
return {
|
||||||
|
type: HELP_REQUEST,
|
||||||
|
requestId: help.requestId,
|
||||||
|
prompt: help.prompt,
|
||||||
|
...(help.title ? { title: help.title } : {}),
|
||||||
|
displayMode: isPrimaryTab ? "full" : "compact",
|
||||||
|
selectors: isPrimaryTab ? help.selectors : [],
|
||||||
|
timeoutMs: help.timeoutMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendHelpRequestWithAck(
|
||||||
|
tabId: number,
|
||||||
|
msg: HelpRequestMessage,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<HelpFinishMessage | null> {
|
||||||
|
let lastError: unknown;
|
||||||
|
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const response = await deps.sendToTab(tabId, msg);
|
||||||
|
if (isHelpAckMessage(response)) return null;
|
||||||
|
if (isHelpResponseMessage(response)) {
|
||||||
|
return {
|
||||||
|
type: HELP_FINISH,
|
||||||
|
requestId: msg.requestId,
|
||||||
|
outcome: response.outcome,
|
||||||
|
...(response.note ? { note: response.note } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
lastError = new Error("content script did not ack HELP_REQUEST");
|
||||||
|
} catch (err) {
|
||||||
|
lastError = err;
|
||||||
|
}
|
||||||
|
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
throw lastError ?? new Error("failed to show help overlay");
|
||||||
|
}
|
||||||
|
|
||||||
async function tagRefTarget(
|
async function tagRefTarget(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -145,7 +192,6 @@ async function tagRefTarget(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns true when `selector` matches a live element in the page. */
|
|
||||||
async function selectorExists(
|
async function selectorExists(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -163,7 +209,6 @@ async function selectorExists(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Remove every help attribute we added (best-effort cleanup). */
|
|
||||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||||
if (!cdp) return;
|
if (!cdp) return;
|
||||||
try {
|
try {
|
||||||
@@ -180,34 +225,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<
|
|||||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// best-effort
|
// Best-effort cleanup.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function handleRequestHelp(
|
async function resolveHelpTargets(
|
||||||
manager: SessionManager,
|
ctx: ReturnType<SessionManager["get"]>,
|
||||||
params: RequestHelpParams,
|
tabId: number,
|
||||||
deps: RequestHelpDeps = getDefaultDeps(),
|
targets: HelpTarget[],
|
||||||
): Promise<RequestHelpResult | RpcError> {
|
deps: RequestHelpDeps,
|
||||||
const ctxOrErr = lookupSession(manager, params, "request_help");
|
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
|
||||||
const ctx = ctxOrErr;
|
|
||||||
if (!params.prompt || typeof params.prompt !== "string") {
|
|
||||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
|
||||||
}
|
|
||||||
if (deps.signal?.aborted) {
|
|
||||||
return { code: "cancelled", message: "request_help aborted" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
|
||||||
if (isRpcError(target)) return target;
|
|
||||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
|
||||||
if (denied) return denied;
|
|
||||||
const tabId = target.tabId;
|
|
||||||
|
|
||||||
// Resolve targets → selectors. `selector` passes through; `ref` is
|
|
||||||
// tagged with an attribute via CDP and reported as its attr selector.
|
|
||||||
const targets = params.targets ?? [];
|
|
||||||
const selectors: string[] = [];
|
const selectors: string[] = [];
|
||||||
const resolved: ResolvedTarget[] = [];
|
const resolved: ResolvedTarget[] = [];
|
||||||
let rootNodeId: number | undefined;
|
let rootNodeId: number | undefined;
|
||||||
@@ -221,22 +248,20 @@ export async function handleRequestHelp(
|
|||||||
rootNodeId = undefined;
|
rootNodeId = undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (let i = 0; i < targets.length; i++) {
|
|
||||||
const tgt = targets[i] as HelpTarget;
|
for (let i = 0; i < targets.length; i += 1) {
|
||||||
|
const tgt = targets[i];
|
||||||
if (tgt.selector) {
|
if (tgt.selector) {
|
||||||
selectors.push(tgt.selector);
|
selectors.push(tgt.selector);
|
||||||
let matched: boolean;
|
const matched =
|
||||||
if (deps.cdp) {
|
deps.cdp && rootNodeId !== undefined
|
||||||
matched =
|
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||||
rootNodeId !== undefined
|
: deps.cdp
|
||||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
? false
|
||||||
: false;
|
: true;
|
||||||
} else {
|
|
||||||
matched = true;
|
|
||||||
}
|
|
||||||
resolved.push({ matched, selector: tgt.selector });
|
resolved.push({ matched, selector: tgt.selector });
|
||||||
} else if (tgt.ref) {
|
} else if (tgt.ref) {
|
||||||
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
|
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||||
const backendNodeId = looked?.backendNodeId ?? null;
|
const backendNodeId = looked?.backendNodeId ?? null;
|
||||||
let sel: string | null = null;
|
let sel: string | null = null;
|
||||||
if (backendNodeId !== null && deps.cdp) {
|
if (backendNodeId !== null && deps.cdp) {
|
||||||
@@ -247,7 +272,407 @@ export async function handleRequestHelp(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bring the tab to the foreground.
|
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||||
|
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||||
|
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||||
|
help.ctx,
|
||||||
|
help.primaryTabId,
|
||||||
|
help.targets,
|
||||||
|
help.deps,
|
||||||
|
);
|
||||||
|
help.selectors = selectors;
|
||||||
|
help.resolvedTargets = resolvedTargets;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||||
|
if (help.deps.notifications) {
|
||||||
|
await help.deps.notifications.clear(help.notificationId).catch(() => {});
|
||||||
|
}
|
||||||
|
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||||
|
await Promise.all(
|
||||||
|
[...tabsToCancel].map((tabId) =>
|
||||||
|
Promise.all([
|
||||||
|
clearRefTags(help.deps.cdp, tabId),
|
||||||
|
help.deps
|
||||||
|
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||||
|
.catch(() => {}),
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function finishHelp(
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
value: RequestHelpResult | RpcError,
|
||||||
|
notifyContent = true,
|
||||||
|
): void {
|
||||||
|
if (help.settled) return;
|
||||||
|
help.settled = true;
|
||||||
|
activeHelpRequests.delete(help.requestId);
|
||||||
|
clearTimeout(help.timeoutTimer);
|
||||||
|
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||||
|
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
|
||||||
|
clearRearmTimer(help.primaryTabId);
|
||||||
|
for (const tabId of help.overlayTabIds) clearRearmTimer(tabId);
|
||||||
|
if (notifyContent) void cleanupHelp(help);
|
||||||
|
help.resolve(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findHelpByKnownTabId(tabId: number): ActiveHelpRequest | null {
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
if (!help.settled && (help.primaryTabId === tabId || help.overlayTabIds.has(tabId))) {
|
||||||
|
return help;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findHelpForTab(
|
||||||
|
tabId: number,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<ActiveHelpRequest | null> {
|
||||||
|
const direct = findHelpByKnownTabId(tabId);
|
||||||
|
if (direct) return direct;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const tab = await deps.tabsApi.get(tabId);
|
||||||
|
const windowId = tab.windowId;
|
||||||
|
if (typeof windowId !== "number") return null;
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
if (!help.settled && help.ctx.agentWindowId === windowId) return help;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearRearmTimer(tabId: number): void {
|
||||||
|
const timer = helpRearmTimers.get(tabId);
|
||||||
|
if (!timer) return;
|
||||||
|
clearTimeout(timer);
|
||||||
|
helpRearmTimers.delete(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendCurrentHelpOverlay(help: ActiveHelpRequest): Promise<HelpFinishMessage | null> {
|
||||||
|
const legacyFinish = await sendHelpRequestWithAck(
|
||||||
|
help.primaryTabId,
|
||||||
|
helpRequestMessage(help, help.primaryTabId),
|
||||||
|
help.deps,
|
||||||
|
);
|
||||||
|
help.overlayTabIds.add(help.primaryTabId);
|
||||||
|
return legacyFinish;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshAndSendHelpOverlay(
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<HelpFinishMessage | null> {
|
||||||
|
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||||
|
const legacyFinish = await sendHelpRequestWithAck(
|
||||||
|
tabId,
|
||||||
|
helpRequestMessage(help, tabId),
|
||||||
|
help.deps,
|
||||||
|
);
|
||||||
|
help.overlayTabIds.add(tabId);
|
||||||
|
return legacyFinish;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise<boolean> {
|
||||||
|
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const legacyFinish = await refreshAndSendHelpOverlay(help, tabId);
|
||||||
|
if (legacyFinish && !help.settled) {
|
||||||
|
finishHelp(help, {
|
||||||
|
outcome: legacyFinish.outcome,
|
||||||
|
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||||
|
tab_id: help.primaryTabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
|
||||||
|
const existing = helpRearmTimers.get(tabId);
|
||||||
|
if (existing) clearTimeout(existing);
|
||||||
|
helpRearmTimers.set(
|
||||||
|
tabId,
|
||||||
|
setTimeout(() => {
|
||||||
|
helpRearmTimers.delete(tabId);
|
||||||
|
void (async () => {
|
||||||
|
const help = await findHelpForTab(tabId, deps);
|
||||||
|
if (!help) return;
|
||||||
|
await rearmHelp(help, tabId);
|
||||||
|
})();
|
||||||
|
}, HELP_REARM_DEBOUNCE_MS),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let detachHelpObservation: (() => void) | null = null;
|
||||||
|
|
||||||
|
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
|
||||||
|
if (detachHelpObservation) return;
|
||||||
|
const detachRuntime = attachHelpRuntimeListener(deps);
|
||||||
|
const detachTabs = attachHelpTabListener(deps);
|
||||||
|
const detachNav = attachHelpNavigationListener(deps);
|
||||||
|
detachHelpObservation = () => {
|
||||||
|
detachRuntime();
|
||||||
|
detachTabs();
|
||||||
|
detachNav();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resetHelpLifecycleForTests(): void {
|
||||||
|
detachHelpObservation?.();
|
||||||
|
detachHelpObservation = null;
|
||||||
|
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
|
||||||
|
helpRearmTimers.clear();
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
clearTimeout(help.timeoutTimer);
|
||||||
|
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||||
|
}
|
||||||
|
activeHelpRequests.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const listener = (
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: HelpQueryResponse) => void,
|
||||||
|
) => {
|
||||||
|
if (isHelpFinishMessage(message)) {
|
||||||
|
void finishHelpFromContent(message, sender, deps);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isHelpQueryMessage(message)) {
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
if (tabId === undefined) {
|
||||||
|
sendResponse({ active: false });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
void (async () => {
|
||||||
|
const help = await findHelpForTab(tabId, deps);
|
||||||
|
if (!help) {
|
||||||
|
sendResponse({ active: false });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||||
|
help.overlayTabIds.add(tabId);
|
||||||
|
sendResponse({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: help.requestId,
|
||||||
|
prompt: help.prompt,
|
||||||
|
...(help.title ? { title: help.title } : {}),
|
||||||
|
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||||
|
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||||
|
timeoutMs: help.timeoutMs,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
chrome.runtime.onMessage.addListener(listener);
|
||||||
|
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finishHelpFromContent(
|
||||||
|
message: HelpFinishMessage,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
const help = activeHelpRequests.get(message.requestId);
|
||||||
|
if (!help || help.settled) return;
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
if (tabId !== undefined) {
|
||||||
|
const match = await findHelpForTab(tabId, deps);
|
||||||
|
if (match !== help) return;
|
||||||
|
}
|
||||||
|
finishHelp(help, {
|
||||||
|
outcome: message.outcome,
|
||||||
|
...(message.note ? { note: message.note } : {}),
|
||||||
|
tab_id: help.primaryTabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||||
|
if (typeof tab.id !== "number") return;
|
||||||
|
scheduleRearmForTab(tab.id, deps);
|
||||||
|
};
|
||||||
|
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||||
|
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||||
|
};
|
||||||
|
chrome.tabs.onCreated?.addListener(onCreated);
|
||||||
|
chrome.tabs.onActivated?.addListener(onActivated);
|
||||||
|
return () => {
|
||||||
|
chrome.tabs.onCreated?.removeListener(onCreated);
|
||||||
|
chrome.tabs.onActivated?.removeListener(onActivated);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
|
||||||
|
|
||||||
|
if (chrome.webNavigation?.onCompleted) {
|
||||||
|
const completedListener = (
|
||||||
|
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||||
|
) => {
|
||||||
|
if (details.frameId !== 0) return;
|
||||||
|
onMainFrameComplete(details.tabId);
|
||||||
|
};
|
||||||
|
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||||
|
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||||
|
}
|
||||||
|
|
||||||
|
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||||
|
if (info.status !== "complete") return;
|
||||||
|
onMainFrameComplete(tabId);
|
||||||
|
};
|
||||||
|
chrome.tabs.onUpdated?.addListener(listener);
|
||||||
|
return () => chrome.tabs.onUpdated?.removeListener(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evaluateCompletionCondition(
|
||||||
|
condition: HelpCompletionCondition,
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
): Promise<boolean> {
|
||||||
|
if (condition.url_contains || condition.url_matches) {
|
||||||
|
let url = "";
|
||||||
|
try {
|
||||||
|
const tab = await help.deps.tabsApi.get(help.primaryTabId);
|
||||||
|
url = tab.url ?? "";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
|
||||||
|
if (condition.url_matches) {
|
||||||
|
try {
|
||||||
|
if (!new RegExp(condition.url_matches).test(url)) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
condition.selector_exists ||
|
||||||
|
condition.selector_missing ||
|
||||||
|
condition.text_exists ||
|
||||||
|
condition.text_missing
|
||||||
|
) {
|
||||||
|
if (!help.deps.cdp) return false;
|
||||||
|
const expression = `(() => {
|
||||||
|
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
|
||||||
|
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
|
||||||
|
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
|
||||||
|
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
|
||||||
|
if (selectorExists && !document.querySelector(selectorExists)) return false;
|
||||||
|
if (selectorMissing && document.querySelector(selectorMissing)) return false;
|
||||||
|
const text = document.body ? document.body.innerText || "" : "";
|
||||||
|
if (textExists && !text.includes(textExists)) return false;
|
||||||
|
if (textMissing && text.includes(textMissing)) return false;
|
||||||
|
return true;
|
||||||
|
})()`;
|
||||||
|
try {
|
||||||
|
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
|
||||||
|
help.primaryTabId,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{ expression, returnByValue: true },
|
||||||
|
);
|
||||||
|
if (result.result?.value !== true) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
|
||||||
|
const criteria = help.completionCriteria;
|
||||||
|
if (!criteria) return false;
|
||||||
|
const any = criteria.any ?? [];
|
||||||
|
const all = criteria.all ?? [];
|
||||||
|
const hasAny = any.length > 0;
|
||||||
|
const hasAll = all.length > 0;
|
||||||
|
if (!hasAny && !hasAll) return false;
|
||||||
|
|
||||||
|
if (hasAll) {
|
||||||
|
for (const condition of all) {
|
||||||
|
if (!(await evaluateCompletionCondition(condition, help))) return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (hasAny) {
|
||||||
|
for (const condition of any) {
|
||||||
|
if (await evaluateCompletionCondition(condition, help)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function startCompletionPolling(help: ActiveHelpRequest): void {
|
||||||
|
if (!help.completionCriteria) return;
|
||||||
|
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
|
||||||
|
const check = async () => {
|
||||||
|
if (help.settled) return;
|
||||||
|
const matched = await completionCriteriaMatches(help);
|
||||||
|
const now = Date.now();
|
||||||
|
if (!matched) {
|
||||||
|
help.completionMatchedSince = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
help.completionMatchedSince ??= now;
|
||||||
|
if (now - help.completionMatchedSince < stableMs) return;
|
||||||
|
finishHelp(help, {
|
||||||
|
outcome: "completed",
|
||||||
|
completed_by: "system",
|
||||||
|
tab_id: help.primaryTabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
|
||||||
|
void check();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleRequestHelp(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: RequestHelpParams,
|
||||||
|
deps: RequestHelpDeps = getDefaultDeps(),
|
||||||
|
): Promise<RequestHelpResult | RpcError> {
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "request_help");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
if (!params.prompt || typeof params.prompt !== "string") {
|
||||||
|
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||||
|
}
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
|
||||||
|
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||||
|
if (denied) return denied;
|
||||||
|
|
||||||
|
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||||
|
const tabId = target.tabId;
|
||||||
|
const initialTargets = params.targets ?? [];
|
||||||
|
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||||
|
|
||||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||||
await deps.activateTab(tabId).catch(() => {});
|
await deps.activateTab(tabId).catch(() => {});
|
||||||
|
|
||||||
@@ -255,7 +680,6 @@ export async function handleRequestHelp(
|
|||||||
const notificationId = `bsk-help:${requestId}`;
|
const notificationId = `bsk-help:${requestId}`;
|
||||||
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
||||||
|
|
||||||
// OS notification (best-effort).
|
|
||||||
if (deps.notifications) {
|
if (deps.notifications) {
|
||||||
const copy = deps.notificationCopy ?? {
|
const copy = deps.notificationCopy ?? {
|
||||||
title: "BrowserSkill: Agent needs your help",
|
title: "BrowserSkill: Agent needs your help",
|
||||||
@@ -272,85 +696,59 @@ export async function handleRequestHelp(
|
|||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
const cleanup = async () => {
|
return new Promise<RequestHelpResult | RpcError>((resolve) => {
|
||||||
await clearRefTags(deps.cdp, tabId);
|
const timeoutTimer = setTimeout(() => {
|
||||||
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
|
finishHelp(help, {
|
||||||
// Retract the overlay in case we resolved via timeout / abort.
|
outcome: "timed_out",
|
||||||
await deps
|
tab_id: help.primaryTabId,
|
||||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
|
resolved_targets: help.resolvedTargets,
|
||||||
.catch(() => {});
|
});
|
||||||
};
|
}, timeoutMs);
|
||||||
|
|
||||||
const request: HelpRequestMessage = {
|
const help: ActiveHelpRequest = {
|
||||||
type: HELP_REQUEST,
|
ctx,
|
||||||
requestId,
|
requestId,
|
||||||
prompt: params.prompt,
|
primaryTabId: tabId,
|
||||||
...(params.title ? { title: params.title } : {}),
|
overlayTabIds: new Set(),
|
||||||
selectors,
|
prompt: params.prompt,
|
||||||
timeoutMs,
|
...(params.title ? { title: params.title } : {}),
|
||||||
};
|
targets: initialTargets,
|
||||||
|
selectors,
|
||||||
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
|
timeoutMs,
|
||||||
|
notificationId,
|
||||||
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
|
resolvedTargets,
|
||||||
let settled = false;
|
completionCriteria: params.completion_criteria,
|
||||||
const unwatchNav = deps.watchTabNavigation(tabId, () => {
|
deps,
|
||||||
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
|
settled: false,
|
||||||
});
|
resolve,
|
||||||
|
timeoutTimer,
|
||||||
const finish = (value: RequestHelpResult | RpcError) => {
|
completionTimer: null,
|
||||||
if (settled) return;
|
completionMatchedSince: null,
|
||||||
settled = true;
|
abortHandler: null,
|
||||||
if (timer !== null) clearTimeout(timer);
|
|
||||||
unwatchNav();
|
|
||||||
deps.signal?.removeEventListener("abort", onAbort);
|
|
||||||
void cleanup();
|
|
||||||
resolveOuter(value);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
|
const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
|
||||||
|
help.abortHandler = onAbort;
|
||||||
const timer = setTimeout(() => {
|
activeHelpRequests.set(requestId, help);
|
||||||
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
|
if (deps.signal?.aborted) {
|
||||||
}, timeoutMs) as unknown as number;
|
onAbort();
|
||||||
|
return;
|
||||||
if (deps.signal) {
|
|
||||||
if (deps.signal.aborted) {
|
|
||||||
onAbort();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
deps.signal.addEventListener("abort", onAbort, { once: true });
|
|
||||||
}
|
}
|
||||||
|
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
startCompletionPolling(help);
|
||||||
|
|
||||||
deps
|
void sendCurrentHelpOverlay(help)
|
||||||
.sendToTab(tabId, request)
|
.then((legacyFinish) => {
|
||||||
.then((reply) => {
|
if (!legacyFinish || help.settled) return;
|
||||||
const res = reply as HelpResponseMessage | undefined;
|
finishHelp(help, {
|
||||||
if (
|
outcome: legacyFinish.outcome,
|
||||||
!res ||
|
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||||
res.type !== HELP_RESPONSE ||
|
tab_id: help.primaryTabId,
|
||||||
(res.outcome !== "continued" && res.outcome !== "cancelled")
|
resolved_targets: help.resolvedTargets,
|
||||||
) {
|
|
||||||
finish({
|
|
||||||
code: "protocol_error",
|
|
||||||
message: `invalid help response from tab ${tabId}`,
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
finish({
|
|
||||||
outcome: res.outcome,
|
|
||||||
...(res.note ? { note: res.note } : {}),
|
|
||||||
tab_id: tabId,
|
|
||||||
resolved_targets: resolvedTargets,
|
|
||||||
});
|
});
|
||||||
})
|
})
|
||||||
.catch((err) => {
|
.catch(() => {
|
||||||
finish({
|
scheduleRearmForTab(tabId, deps);
|
||||||
code: "protocol_error",
|
|
||||||
message: `failed to show help overlay on tab ${tabId}: ${
|
|
||||||
err instanceof Error ? err.message : String(err)
|
|
||||||
}`,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -514,7 +514,22 @@ export interface HelpTarget {
|
|||||||
selector?: string;
|
selector?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated";
|
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated";
|
||||||
|
|
||||||
|
export interface HelpCompletionCondition {
|
||||||
|
url_contains?: string;
|
||||||
|
url_matches?: string;
|
||||||
|
selector_exists?: string;
|
||||||
|
selector_missing?: string;
|
||||||
|
text_exists?: string;
|
||||||
|
text_missing?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpCompletionCriteria {
|
||||||
|
any?: HelpCompletionCondition[];
|
||||||
|
all?: HelpCompletionCondition[];
|
||||||
|
stable_for_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface ResolvedTarget {
|
export interface ResolvedTarget {
|
||||||
matched: boolean;
|
matched: boolean;
|
||||||
@@ -528,11 +543,13 @@ export interface RequestHelpParams {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
targets?: HelpTarget[];
|
targets?: HelpTarget[];
|
||||||
|
completion_criteria?: HelpCompletionCriteria;
|
||||||
timeout_ms?: number;
|
timeout_ms?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RequestHelpResult {
|
export interface RequestHelpResult {
|
||||||
outcome: HelpOutcome;
|
outcome: HelpOutcome;
|
||||||
|
completed_by?: "system";
|
||||||
note?: string;
|
note?: string;
|
||||||
tab_id: number;
|
tab_id: number;
|
||||||
resolved_targets?: ResolvedTarget[];
|
resolved_targets?: ResolvedTarget[];
|
||||||
|
|||||||
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
|||||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||||
confirm an important action, pause and ask:
|
confirm an important action, pause and ask:
|
||||||
|
|
||||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||||
|
|
||||||
- `--prompt` (required): what the user should do.
|
- `--prompt` (required): what the user should do.
|
||||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
|||||||
prompt with no `--target` for cases where there is genuinely no specific
|
prompt with no `--target` for cases where there is genuinely no specific
|
||||||
element to point at (e.g. "wait for the page to finish loading").
|
element to point at (e.g. "wait for the page to finish loading").
|
||||||
- `--timeout` (default `5m`): how long to wait.
|
- `--timeout` (default `5m`): how long to wait.
|
||||||
|
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||||
|
when there is a concrete post-help success signal, e.g.
|
||||||
|
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||||
|
|
||||||
The target tab is brought to the foreground; the page stays interactive
|
The target tab is brought to the foreground; the page stays interactive
|
||||||
while the agent control mask is hidden. The call blocks until the user
|
while the agent control mask is hidden. The call blocks until the user
|
||||||
acts. The result `outcome` is one of:
|
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||||
|
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||||
|
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||||
|
|
||||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||||
- `timed_out` — nobody acted within the timeout.
|
- `timed_out` — nobody acted within the timeout.
|
||||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||||
|
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||||
|
|
||||||
`note` carries any text the user typed back. `resolved_targets` reports
|
`note` carries any text the user typed back. `resolved_targets` reports
|
||||||
which refs/selectors matched a live element.
|
which refs/selectors matched a live element.
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use bsk_protocol::Method;
|
use bsk_protocol::Method;
|
||||||
use bsk_protocol::tools::{HelpTarget, RequestHelpParams, RequestHelpResult};
|
use bsk_protocol::tools::{
|
||||||
|
HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult,
|
||||||
|
};
|
||||||
use clap::Args;
|
use clap::Args;
|
||||||
|
|
||||||
use crate::cli::ensure_daemon::ensure_daemon;
|
use crate::cli::ensure_daemon::ensure_daemon;
|
||||||
@@ -41,6 +43,10 @@ pub struct RequestHelpArgs {
|
|||||||
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
|
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
|
||||||
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
|
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
|
||||||
pub timeout: u32,
|
pub timeout: u32,
|
||||||
|
|
||||||
|
/// JSON success detector, e.g. '{"any":[{"url_contains":"/dashboard"}],"stable_for_ms":1000}'.
|
||||||
|
#[arg(long = "completion-criteria")]
|
||||||
|
pub completion_criteria: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
|
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
|
||||||
@@ -68,6 +74,12 @@ pub fn parse_target(raw: &str) -> HelpTarget {
|
|||||||
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
||||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||||
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
|
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
|
||||||
|
let completion_criteria = args
|
||||||
|
.completion_criteria
|
||||||
|
.as_deref()
|
||||||
|
.map(serde_json::from_str::<HelpCompletionCriteria>)
|
||||||
|
.transpose()
|
||||||
|
.context("parse --completion-criteria JSON")?;
|
||||||
let params = RequestHelpParams {
|
let params = RequestHelpParams {
|
||||||
session_id: args.session,
|
session_id: args.session,
|
||||||
tab_id: args.tab_id,
|
tab_id: args.tab_id,
|
||||||
@@ -78,6 +90,7 @@ pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
|||||||
} else {
|
} else {
|
||||||
Some(targets)
|
Some(targets)
|
||||||
},
|
},
|
||||||
|
completion_criteria,
|
||||||
timeout_ms: Some(args.timeout),
|
timeout_ms: Some(args.timeout),
|
||||||
};
|
};
|
||||||
let reply = call(info.sock_path, params, args.timeout)?;
|
let reply = call(info.sock_path, params, args.timeout)?;
|
||||||
|
|||||||
@@ -7,6 +7,17 @@
|
|||||||
"session_id"
|
"session_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"completion_criteria": {
|
||||||
|
"description": "Optional explicit success detector. Navigation by itself is not a completion signal; these criteria must match before the tool can auto-complete without the user clicking Done.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/HelpCompletionCriteria"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"prompt": {
|
"prompt": {
|
||||||
"description": "Message shown to the user explaining what to do.",
|
"description": "Message shown to the user explaining what to do.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
@@ -50,6 +61,78 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"HelpCompletionCondition": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"selector_exists": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"selector_missing": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"text_exists": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"text_missing": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"url_contains": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"url_matches": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"HelpCompletionCriteria": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"all": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/HelpCompletionCondition"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"any": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/HelpCompletionCondition"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"stable_for_ms": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"HelpTarget": {
|
"HelpTarget": {
|
||||||
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
|
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -7,6 +7,16 @@
|
|||||||
"tab_id"
|
"tab_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"completed_by": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/HelpCompletedBy"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"note": {
|
"note": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
@@ -31,10 +41,16 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"HelpCompletedBy": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"system"
|
||||||
|
]
|
||||||
|
},
|
||||||
"HelpOutcome": {
|
"HelpOutcome": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
"description": "User completed the step and clicked Continue.",
|
"description": "User completed the step and clicked Done / return control.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": [
|
"enum": [
|
||||||
"continued"
|
"continued"
|
||||||
@@ -55,7 +71,14 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "The page navigated while waiting (full reload or SPA URL change).",
|
"description": "Explicit success criteria matched while the user had control.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"completed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "The page navigated while waiting (full reload or SPA URL change). Deprecated: navigation alone should not end a help request.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": [
|
"enum": [
|
||||||
"navigated"
|
"navigated"
|
||||||
|
|||||||
@@ -3,7 +3,8 @@
|
|||||||
//! Lets an agent pause and ask the human to complete an in-page step
|
//! Lets an agent pause and ask the human to complete an in-page step
|
||||||
//! (captcha, login, confirmation). The extension brings the target tab
|
//! (captcha, login, confirmation). The extension brings the target tab
|
||||||
//! to the foreground, highlights the requested components, and blocks
|
//! to the foreground, highlights the requested components, and blocks
|
||||||
//! until the user clicks Continue / Cancel (or the call times out).
|
//! until the user clicks Done / Cancel, explicit completion criteria match,
|
||||||
|
//! or the call times out.
|
||||||
|
|
||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -41,6 +42,11 @@ pub struct RequestHelpParams {
|
|||||||
/// Optional components to scroll to + flash-highlight.
|
/// Optional components to scroll to + flash-highlight.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub targets: Option<Vec<HelpTarget>>,
|
pub targets: Option<Vec<HelpTarget>>,
|
||||||
|
/// Optional explicit success detector. Navigation by itself is not a
|
||||||
|
/// completion signal; these criteria must match before the tool can
|
||||||
|
/// auto-complete without the user clicking Done.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub completion_criteria: Option<HelpCompletionCriteria>,
|
||||||
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
|
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
|
||||||
/// enforced daemon-side via the generic tool timeout.
|
/// enforced daemon-side via the generic tool timeout.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
@@ -48,16 +54,46 @@ pub struct RequestHelpParams {
|
|||||||
pub timeout_ms: Option<u32>,
|
pub timeout_ms: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HelpCompletionCriteria {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub any: Option<Vec<HelpCompletionCondition>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub all: Option<Vec<HelpCompletionCondition>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[schemars(range(min = 0))]
|
||||||
|
pub stable_for_ms: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HelpCompletionCondition {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub url_contains: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub url_matches: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector_exists: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector_missing: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub text_exists: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub text_missing: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum HelpOutcome {
|
pub enum HelpOutcome {
|
||||||
/// User completed the step and clicked Continue.
|
/// User completed the step and clicked Done / return control.
|
||||||
Continued,
|
Continued,
|
||||||
/// User declined / aborted via the overlay's Cancel button.
|
/// User declined / aborted via the overlay's Cancel button.
|
||||||
Cancelled,
|
Cancelled,
|
||||||
/// The wait expired before the user acted.
|
/// The wait expired before the user acted.
|
||||||
TimedOut,
|
TimedOut,
|
||||||
|
/// Explicit success criteria matched while the user had control.
|
||||||
|
Completed,
|
||||||
/// The page navigated while waiting (full reload or SPA URL change).
|
/// The page navigated while waiting (full reload or SPA URL change).
|
||||||
|
/// Deprecated: navigation alone should not end a help request.
|
||||||
Navigated,
|
Navigated,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,11 +103,18 @@ impl HelpOutcome {
|
|||||||
Self::Continued => "continued",
|
Self::Continued => "continued",
|
||||||
Self::Cancelled => "cancelled",
|
Self::Cancelled => "cancelled",
|
||||||
Self::TimedOut => "timed_out",
|
Self::TimedOut => "timed_out",
|
||||||
|
Self::Completed => "completed",
|
||||||
Self::Navigated => "navigated",
|
Self::Navigated => "navigated",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum HelpCompletedBy {
|
||||||
|
System,
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-target resolution status echoed back so the agent can tell which
|
/// Per-target resolution status echoed back so the agent can tell which
|
||||||
/// `ref` / `selector` actually matched a live element.
|
/// `ref` / `selector` actually matched a live element.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
@@ -87,6 +130,8 @@ pub struct ResolvedTarget {
|
|||||||
pub struct RequestHelpResult {
|
pub struct RequestHelpResult {
|
||||||
pub outcome: HelpOutcome,
|
pub outcome: HelpOutcome,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub completed_by: Option<HelpCompletedBy>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub note: Option<String>,
|
pub note: Option<String>,
|
||||||
pub tab_id: i64,
|
pub tab_id: i64,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
@@ -123,6 +168,7 @@ mod tests {
|
|||||||
prompt: "log in".into(),
|
prompt: "log in".into(),
|
||||||
title: None,
|
title: None,
|
||||||
targets: None,
|
targets: None,
|
||||||
|
completion_criteria: None,
|
||||||
timeout_ms: None,
|
timeout_ms: None,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&p).unwrap();
|
let v = serde_json::to_value(&p).unwrap();
|
||||||
@@ -140,6 +186,10 @@ mod tests {
|
|||||||
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
|
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
|
||||||
json!("timed_out")
|
json!("timed_out")
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(HelpOutcome::Completed).unwrap(),
|
||||||
|
json!("completed")
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::to_value(HelpOutcome::Continued).unwrap(),
|
serde_json::to_value(HelpOutcome::Continued).unwrap(),
|
||||||
json!("continued")
|
json!("continued")
|
||||||
@@ -158,6 +208,7 @@ mod tests {
|
|||||||
fn result_round_trips() {
|
fn result_round_trips() {
|
||||||
let r = RequestHelpResult {
|
let r = RequestHelpResult {
|
||||||
outcome: HelpOutcome::Continued,
|
outcome: HelpOutcome::Continued,
|
||||||
|
completed_by: None,
|
||||||
note: Some("done".into()),
|
note: Some("done".into()),
|
||||||
tab_id: 7,
|
tab_id: 7,
|
||||||
resolved_targets: Some(vec![ResolvedTarget {
|
resolved_targets: Some(vec![ResolvedTarget {
|
||||||
|
|||||||
@@ -74,10 +74,11 @@
|
|||||||
},
|
},
|
||||||
"helpRequest": {
|
"helpRequest": {
|
||||||
"title": "Agent needs your help",
|
"title": "Agent needs your help",
|
||||||
|
"compactStatus": "Agent is still waiting for you to finish this step",
|
||||||
"noteLabel": "Optional note back to the agent",
|
"noteLabel": "Optional note back to the agent",
|
||||||
"notePlaceholder": "Add a note for the agent (optional)",
|
"notePlaceholder": "Add a note for the agent (optional)",
|
||||||
"continue": "Continue",
|
"continue": "Done, return control",
|
||||||
"cancel": "Cancel",
|
"cancel": "Cancel request",
|
||||||
"collapse": "Collapse",
|
"collapse": "Collapse",
|
||||||
"expand": "Expand",
|
"expand": "Expand",
|
||||||
"dragHandle": "Drag to move",
|
"dragHandle": "Drag to move",
|
||||||
|
|||||||
@@ -74,10 +74,11 @@
|
|||||||
},
|
},
|
||||||
"helpRequest": {
|
"helpRequest": {
|
||||||
"title": "Agent 需要你的帮助",
|
"title": "Agent 需要你的帮助",
|
||||||
|
"compactStatus": "当前 Agent 仍在等待你完成此步骤",
|
||||||
"noteLabel": "给 Agent 的备注(可选)",
|
"noteLabel": "给 Agent 的备注(可选)",
|
||||||
"notePlaceholder": "给 Agent 留言(可选)",
|
"notePlaceholder": "给 Agent 留言(可选)",
|
||||||
"continue": "继续",
|
"continue": "完成并交还控制权",
|
||||||
"cancel": "取消",
|
"cancel": "取消请求",
|
||||||
"collapse": "折叠",
|
"collapse": "折叠",
|
||||||
"expand": "展开",
|
"expand": "展开",
|
||||||
"dragHandle": "拖动以移动",
|
"dragHandle": "拖动以移动",
|
||||||
|
|||||||
+10
-4
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
|||||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||||
confirm an important action, pause and ask:
|
confirm an important action, pause and ask:
|
||||||
|
|
||||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||||
|
|
||||||
- `--prompt` (required): what the user should do.
|
- `--prompt` (required): what the user should do.
|
||||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
|||||||
prompt with no `--target` for cases where there is genuinely no specific
|
prompt with no `--target` for cases where there is genuinely no specific
|
||||||
element to point at (e.g. "wait for the page to finish loading").
|
element to point at (e.g. "wait for the page to finish loading").
|
||||||
- `--timeout` (default `5m`): how long to wait.
|
- `--timeout` (default `5m`): how long to wait.
|
||||||
|
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||||
|
when there is a concrete post-help success signal, e.g.
|
||||||
|
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||||
|
|
||||||
The target tab is brought to the foreground; the page stays interactive
|
The target tab is brought to the foreground; the page stays interactive
|
||||||
while the agent control mask is hidden. The call blocks until the user
|
while the agent control mask is hidden. The call blocks until the user
|
||||||
acts. The result `outcome` is one of:
|
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||||
|
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||||
|
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||||
|
|
||||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||||
- `timed_out` — nobody acted within the timeout.
|
- `timed_out` — nobody acted within the timeout.
|
||||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||||
|
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||||
|
|
||||||
`note` carries any text the user typed back. `resolved_targets` reports
|
`note` carries any text the user typed back. `resolved_targets` reports
|
||||||
which refs/selectors matched a live element.
|
which refs/selectors matched a live element.
|
||||||
|
|||||||
Reference in New Issue
Block a user