mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(runtime): fix early control return and update lifecycle
This commit is contained in:
@@ -342,6 +342,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
z-index: 2147483647;
|
z-index: 2147483647;
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
justify-content: flex-start;
|
||||||
|
min-height: 0;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
width: var(--bsk-help-banner-width);
|
width: var(--bsk-help-banner-width);
|
||||||
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
||||||
@@ -359,16 +361,18 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
.bsk-help-banner[data-collapsed="true"] {
|
.bsk-help-banner[data-collapsed="true"] {
|
||||||
gap: 0;
|
gap: 0;
|
||||||
width: var(--bsk-help-banner-width);
|
width: var(--bsk-help-banner-width);
|
||||||
padding: 10px 12px;
|
padding: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-drag-strip {
|
.bsk-help-drag-strip {
|
||||||
flex-shrink: 0;
|
position: absolute;
|
||||||
|
top: 6px;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
height: 12px;
|
height: 12px;
|
||||||
margin: -8px 0 -10px;
|
|
||||||
cursor: grab;
|
cursor: grab;
|
||||||
user-select: none;
|
user-select: none;
|
||||||
touch-action: none;
|
touch-action: none;
|
||||||
@@ -395,10 +399,11 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
||||||
margin: -6px 0 -8px;
|
top: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header {
|
.bsk-help-header {
|
||||||
|
flex-shrink: 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
@@ -420,25 +425,17 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
|
|
||||||
.bsk-help-header-actions {
|
.bsk-help-header-actions {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
display: flex;
|
display: none;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
margin-left: auto;
|
margin-left: auto;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
max-width: 0;
|
|
||||||
opacity: 0;
|
|
||||||
transform: translateX(8px);
|
|
||||||
pointer-events: none;
|
|
||||||
transition:
|
|
||||||
max-width var(--bsk-help-duration) var(--bsk-help-ease),
|
|
||||||
opacity 220ms var(--bsk-help-ease),
|
|
||||||
transform var(--bsk-help-duration) var(--bsk-help-ease);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
||||||
|
display: flex;
|
||||||
max-width: 280px;
|
max-width: 280px;
|
||||||
opacity: 1;
|
opacity: 1;
|
||||||
transform: translateX(0);
|
|
||||||
pointer-events: auto;
|
pointer-events: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -471,15 +468,13 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-body {
|
.bsk-help-body {
|
||||||
display: grid;
|
display: block;
|
||||||
grid-template-rows: 1fr;
|
flex-shrink: 0;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
grid-template-rows: 0fr;
|
|
||||||
width: 0;
|
width: 0;
|
||||||
height: 0;
|
height: 0;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -528,6 +523,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
.bsk-help-footer-actions {
|
.bsk-help-footer-actions {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
|
flex-wrap: wrap;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,7 +535,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
color: #4b5563;
|
color: #4b5563;
|
||||||
flex-shrink: 0;
|
min-width: 0;
|
||||||
|
white-space: normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-btn-continue {
|
.bsk-help-btn-continue {
|
||||||
@@ -550,16 +547,19 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: #fff;
|
color: #fff;
|
||||||
flex-shrink: 0;
|
min-width: 0;
|
||||||
|
white-space: normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions .bsk-help-btn-cancel,
|
.bsk-help-header-actions .bsk-help-btn-cancel,
|
||||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
padding: 6px 12px;
|
padding: 6px 12px;
|
||||||
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
padding: 6px 14px;
|
padding: 6px 14px;
|
||||||
|
max-width: 180px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
||||||
|
|||||||
@@ -204,6 +204,15 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(styles).toContain("width: 0");
|
expect(styles).toContain("width: 0");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses natural body height in the expanded layout", () => {
|
||||||
|
const { container } = renderOverlay(baseRequest());
|
||||||
|
const styles = overlayStyles(container);
|
||||||
|
|
||||||
|
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
|
||||||
|
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
|
||||||
|
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the same banner width when collapsed", () => {
|
it("keeps the same banner width when collapsed", () => {
|
||||||
const { container } = renderOverlay(baseRequest());
|
const { container } = renderOverlay(baseRequest());
|
||||||
const styles = overlayStyles(container);
|
const styles = overlayStyles(container);
|
||||||
|
|||||||
@@ -14,10 +14,14 @@ import {
|
|||||||
type RecordCaptureController,
|
type RecordCaptureController,
|
||||||
} from "@/content/record-capture";
|
} from "@/content/record-capture";
|
||||||
import {
|
import {
|
||||||
HELP_RESPONSE,
|
HELP_ACK,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
|
type HelpAckMessage,
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
|
type HelpFinishMessage,
|
||||||
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
type HelpResponseMessage,
|
|
||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
@@ -55,8 +59,6 @@ export default defineContentScript({
|
|||||||
if (window.top !== window) return;
|
if (window.top !== window) return;
|
||||||
|
|
||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
|
|
||||||
null;
|
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
@@ -116,8 +118,7 @@ export default defineContentScript({
|
|||||||
function resetAgentOverlayState(sessionId: string) {
|
function resetAgentOverlayState(sessionId: string) {
|
||||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||||
if (previousHelp) {
|
if (previousHelp) {
|
||||||
activeHelpRespond?.("cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
activeHelpRespond = null;
|
|
||||||
}
|
}
|
||||||
recordCapture?.dispose();
|
recordCapture?.dispose();
|
||||||
recordCapture = null;
|
recordCapture = null;
|
||||||
@@ -156,7 +157,7 @@ export default defineContentScript({
|
|||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
|
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||||
) => {
|
) => {
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
const needsAsync = handleRecordContentMessage(
|
||||||
@@ -220,41 +221,29 @@ export default defineContentScript({
|
|||||||
if (isHelpCancelMessage(message)) {
|
if (isHelpCancelMessage(message)) {
|
||||||
const state = overlays.snapshot();
|
const state = overlays.snapshot();
|
||||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||||
activeHelpRespond?.("cancelled");
|
overlays.clearAgentHelpRequest(message.requestId);
|
||||||
|
renderOverlay();
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isHelpRequestMessage(message)) {
|
if (isHelpRequestMessage(message)) {
|
||||||
const helpMsg = message as HelpRequestMessage;
|
const helpMsg = message as HelpRequestMessage;
|
||||||
let responded = false;
|
|
||||||
const respond = (outcome: "continued" | "cancelled", note?: string) => {
|
|
||||||
if (responded) return;
|
|
||||||
responded = true;
|
|
||||||
const reply: HelpResponseMessage = {
|
|
||||||
type: HELP_RESPONSE,
|
|
||||||
outcome,
|
|
||||||
...(note ? { note } : {}),
|
|
||||||
};
|
|
||||||
sendResponse(reply);
|
|
||||||
activeHelpRespond = null;
|
|
||||||
overlays.clearAgentHelpRequest(helpMsg.requestId);
|
|
||||||
renderOverlay();
|
|
||||||
};
|
|
||||||
const previousHelp = overlays.setAgentHelpRequest({
|
const previousHelp = overlays.setAgentHelpRequest({
|
||||||
id: helpMsg.requestId,
|
id: helpMsg.requestId,
|
||||||
prompt: helpMsg.prompt,
|
prompt: helpMsg.prompt,
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||||
selectors: helpMsg.selectors,
|
selectors: helpMsg.selectors,
|
||||||
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
|
onContinue: (note: string) =>
|
||||||
onCancel: () => respond("cancelled"),
|
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||||
});
|
});
|
||||||
if (previousHelp) {
|
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||||
activeHelpRespond?.("cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
activeHelpRespond = respond;
|
|
||||||
renderOverlay();
|
renderOverlay();
|
||||||
return true; // async sendResponse
|
sendResponse({ type: HELP_ACK, ok: true });
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (message.type === "borrow-request") {
|
if (message.type === "borrow-request") {
|
||||||
@@ -282,9 +271,59 @@ export default defineContentScript({
|
|||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
async function sendHelpFinish(
|
||||||
|
requestId: string,
|
||||||
|
outcome: "continued" | "cancelled",
|
||||||
|
note?: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const msg: HelpFinishMessage = {
|
||||||
|
type: HELP_FINISH,
|
||||||
|
requestId,
|
||||||
|
outcome,
|
||||||
|
...(note ? { note } : {}),
|
||||||
|
};
|
||||||
|
overlays.clearAgentHelpRequest(requestId);
|
||||||
|
renderOverlay();
|
||||||
|
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||||
|
console.debug("[bsk overlay] help finish failed", err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||||
|
overlays.setAgentHelpRequest({
|
||||||
|
id: helpMsg.requestId,
|
||||||
|
prompt: helpMsg.prompt,
|
||||||
|
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||||
|
selectors: helpMsg.selectors,
|
||||||
|
onContinue: (note: string) =>
|
||||||
|
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function queryActiveHelpWithRetry(): Promise<boolean> {
|
||||||
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const helpQuery = (await chrome.runtime.sendMessage({
|
||||||
|
type: HELP_QUERY,
|
||||||
|
})) as HelpQueryResponse | undefined;
|
||||||
|
if (helpQuery?.active && helpQuery.request) {
|
||||||
|
mountHelpRequest(helpQuery.request);
|
||||||
|
renderOverlay();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk overlay] help query failed", err);
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
async function syncAgentOverlay(): Promise<void> {
|
async function syncAgentOverlay(): Promise<void> {
|
||||||
if (!(await anySessionLive())) return;
|
if (!(await anySessionLive())) return;
|
||||||
try {
|
try {
|
||||||
|
const helpActive = await queryActiveHelpWithRetry();
|
||||||
const reply = (await chrome.runtime.sendMessage({
|
const reply = (await chrome.runtime.sendMessage({
|
||||||
kind: OVERLAY_MSG_WHO_AM_I,
|
kind: OVERLAY_MSG_WHO_AM_I,
|
||||||
})) as OverlayWhoAmIResponse | undefined;
|
})) as OverlayWhoAmIResponse | undefined;
|
||||||
@@ -319,6 +358,10 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!helpActive && overlays.snapshot().activeHelp === null) {
|
||||||
|
void queryActiveHelpWithRetry();
|
||||||
|
}
|
||||||
|
|
||||||
overlays.activateAgentSession(reply.sessionId);
|
overlays.activateAgentSession(reply.sessionId);
|
||||||
renderOverlay();
|
renderOverlay();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -1,10 +1,17 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
|
HELP_ACK,
|
||||||
HELP_CANCEL,
|
HELP_CANCEL,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
HELP_RESPONSE,
|
HELP_RESPONSE,
|
||||||
|
isHelpAckMessage,
|
||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
|
isHelpFinishMessage,
|
||||||
|
isHelpQueryMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
|
isHelpResponseMessage,
|
||||||
} from "../help-bridge";
|
} from "../help-bridge";
|
||||||
|
|
||||||
describe("help-bridge", () => {
|
describe("help-bridge", () => {
|
||||||
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
|
|||||||
expect(HELP_REQUEST).toBe("bsk-help-request");
|
expect(HELP_REQUEST).toBe("bsk-help-request");
|
||||||
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
||||||
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
||||||
|
expect(HELP_ACK).toBe("bsk-help-ack");
|
||||||
|
expect(HELP_QUERY).toBe("bsk-help-query");
|
||||||
|
expect(HELP_FINISH).toBe("bsk-help-finish");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("type-guards a help-request message", () => {
|
it("type-guards a help-request message", () => {
|
||||||
@@ -55,4 +65,18 @@ describe("help-bridge", () => {
|
|||||||
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
||||||
expect(isHelpCancelMessage(null)).toBe(false);
|
expect(isHelpCancelMessage(null)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("type-guards lifecycle messages", () => {
|
||||||
|
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
|
||||||
|
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
|
||||||
|
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,12 +6,15 @@
|
|||||||
* Mirrors the borrow-confirmation message style (see
|
* Mirrors the borrow-confirmation message style (see
|
||||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||||
* the content script replies once the user clicks Continue / Cancel.
|
* the content script acks display, then later reports Done / Cancel.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const HELP_REQUEST = "bsk-help-request";
|
export const HELP_REQUEST = "bsk-help-request";
|
||||||
export const HELP_RESPONSE = "bsk-help-response";
|
export const HELP_RESPONSE = "bsk-help-response";
|
||||||
export const HELP_CANCEL = "bsk-help-cancel";
|
export const HELP_CANCEL = "bsk-help-cancel";
|
||||||
|
export const HELP_ACK = "bsk-help-ack";
|
||||||
|
export const HELP_QUERY = "bsk-help-query";
|
||||||
|
export const HELP_FINISH = "bsk-help-finish";
|
||||||
|
|
||||||
export interface HelpRequestMessage {
|
export interface HelpRequestMessage {
|
||||||
type: typeof HELP_REQUEST;
|
type: typeof HELP_REQUEST;
|
||||||
@@ -24,6 +27,11 @@ export interface HelpRequestMessage {
|
|||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HelpAckMessage {
|
||||||
|
type: typeof HELP_ACK;
|
||||||
|
ok: true;
|
||||||
|
}
|
||||||
|
|
||||||
export interface HelpResponseMessage {
|
export interface HelpResponseMessage {
|
||||||
type: typeof HELP_RESPONSE;
|
type: typeof HELP_RESPONSE;
|
||||||
outcome: "continued" | "cancelled";
|
outcome: "continued" | "cancelled";
|
||||||
@@ -35,6 +43,22 @@ export interface HelpCancelMessage {
|
|||||||
requestId: string;
|
requestId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HelpQueryMessage {
|
||||||
|
type: typeof HELP_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpQueryResponse {
|
||||||
|
active: boolean;
|
||||||
|
request?: Omit<HelpRequestMessage, "type">;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpFinishMessage {
|
||||||
|
type: typeof HELP_FINISH;
|
||||||
|
requestId: string;
|
||||||
|
outcome: "continued" | "cancelled";
|
||||||
|
note?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||||
if (typeof msg !== "object" || msg === null) {
|
if (typeof msg !== "object" || msg === null) {
|
||||||
return false;
|
return false;
|
||||||
@@ -58,3 +82,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
|
|||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === HELP_RESPONSE &&
|
||||||
|
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||||
|
(m.note === undefined || typeof m.note === "string")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === HELP_ACK && m.ok === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === HELP_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === HELP_FINISH &&
|
||||||
|
typeof m.requestId === "string" &&
|
||||||
|
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||||
|
(m.note === undefined || typeof m.note === "string")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,9 +27,9 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
|
|||||||
windows: { update: vi.fn(async () => ({}) as never) },
|
windows: { update: vi.fn(async () => ({}) as never) },
|
||||||
activateTab: vi.fn(async () => {}),
|
activateTab: vi.fn(async () => {}),
|
||||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
||||||
watchTabNavigation: () => () => {},
|
|
||||||
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
||||||
notifications: null,
|
notifications: null,
|
||||||
|
autoAttachLifecycle: false,
|
||||||
...over,
|
...over,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -78,22 +78,29 @@ describe("handleRequestHelp", () => {
|
|||||||
expect(sentMsg.title).toBeUndefined();
|
expect(sentMsg.title).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns navigated when the tab navigates during the wait", async () => {
|
it("does not complete merely because the tab navigates during the wait", async () => {
|
||||||
const unwatch = vi.fn();
|
vi.useFakeTimers();
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
sendToTab: vi.fn(() => new Promise(() => {})),
|
sendToTab: vi.fn(() => new Promise(() => {})),
|
||||||
watchTabNavigation: vi.fn((_tabId, cb) => {
|
|
||||||
queueMicrotask(() => cb());
|
|
||||||
return unwatch;
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
try {
|
||||||
fakeManager("abcd", 99, 5),
|
const pending = handleRequestHelp(
|
||||||
baseParams({ tab_id: 5 }),
|
fakeManager("abcd", 99, 5),
|
||||||
deps,
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
);
|
deps,
|
||||||
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
|
);
|
||||||
expect(unwatch).toHaveBeenCalled();
|
await vi.advanceTimersByTimeAsync(5);
|
||||||
|
let settled = false;
|
||||||
|
void pending.then(() => {
|
||||||
|
settled = true;
|
||||||
|
});
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns timed_out when the wait expires", async () => {
|
it("returns timed_out when the wait expires", async () => {
|
||||||
@@ -115,6 +122,30 @@ describe("handleRequestHelp", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns completed when explicit completion criteria match", async () => {
|
||||||
|
const deps = baseDeps({
|
||||||
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||||
|
cdp: {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Runtime.evaluate") return { result: { value: true } };
|
||||||
|
return {};
|
||||||
|
}),
|
||||||
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
|
});
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({
|
||||||
|
tab_id: 5,
|
||||||
|
completion_criteria: {
|
||||||
|
any: [{ selector_exists: "#account-menu" }],
|
||||||
|
stable_for_ms: 0,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
|
||||||
|
});
|
||||||
|
|
||||||
it("tags ref targets via CDP and reports them matched", async () => {
|
it("tags ref targets via CDP and reports them matched", async () => {
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
@@ -190,38 +221,40 @@ describe("handleRequestHelp", () => {
|
|||||||
expect("code" in res).toBe(false);
|
expect("code" in res).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns protocol_error when sendToTab rejects", async () => {
|
it("keeps waiting when the initial overlay delivery rejects", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
sendToTab: vi.fn(async () => {
|
sendToTab: vi.fn(async () => {
|
||||||
throw new Error("no receiver");
|
throw new Error("no receiver");
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
try {
|
||||||
fakeManager("abcd", 99, 5),
|
const res = handleRequestHelp(
|
||||||
baseParams({ tab_id: 5 }),
|
fakeManager("abcd", 99, 5),
|
||||||
deps,
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
);
|
deps,
|
||||||
expect("code" in res && res.code).toBe("protocol_error");
|
);
|
||||||
|
await vi.advanceTimersByTimeAsync(20);
|
||||||
|
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns protocol_error for a malformed help response", async () => {
|
it("keeps waiting for explicit completion after a malformed help response", async () => {
|
||||||
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
vi.useFakeTimers();
|
||||||
const res1 = await handleRequestHelp(
|
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||||
fakeManager("abcd", 99, 5),
|
try {
|
||||||
baseParams({ tab_id: 5 }),
|
const res = handleRequestHelp(
|
||||||
undefinedReply,
|
fakeManager("abcd", 99, 5),
|
||||||
);
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
expect("code" in res1 && res1.code).toBe("protocol_error");
|
deps,
|
||||||
|
);
|
||||||
const badOutcome = baseDeps({
|
await vi.advanceTimersByTimeAsync(20);
|
||||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
|
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||||
});
|
} finally {
|
||||||
const res2 = await handleRequestHelp(
|
vi.useRealTimers();
|
||||||
fakeManager("abcd", 99, 5),
|
}
|
||||||
baseParams({ tab_id: 5 }),
|
|
||||||
badOutcome,
|
|
||||||
);
|
|
||||||
expect("code" in res2 && res2.code).toBe("protocol_error");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("reports selector match status from CDP", async () => {
|
it("reports selector match status from CDP", async () => {
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ import type {
|
|||||||
import { isRequestFrame } from "@/transport/types";
|
import { isRequestFrame } from "@/transport/types";
|
||||||
import { handleConsole } from "./console";
|
import { handleConsole } from "./console";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
@@ -394,7 +394,6 @@ export class ToolDispatcher {
|
|||||||
await chrome.tabs.update(tabId, { active: true });
|
await chrome.tabs.update(tabId, { active: true });
|
||||||
},
|
},
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
watchTabNavigation: defaultWatchTabNavigation,
|
|
||||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||||
notifications: makeHelpNotifications(),
|
notifications: makeHelpNotifications(),
|
||||||
notificationCopy: this.helpNotificationCopy?.(),
|
notificationCopy: this.helpNotificationCopy?.(),
|
||||||
|
|||||||
@@ -1,21 +1,24 @@
|
|||||||
// `tool.request_help` — pause and ask the human to act on a tab.
|
// `tool.request_help` — pause automation and let the human control a tab.
|
||||||
//
|
|
||||||
// Brings the target tab to the foreground, resolves `ref` targets to a
|
|
||||||
// temporary attribute selector via CDP, asks the content script to enter
|
|
||||||
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
|
|
||||||
// until the user clicks Continue / Cancel, the wait times out, or the
|
|
||||||
// daemon cancels the RPC.
|
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
HELP_ACK,
|
||||||
HELP_CANCEL,
|
HELP_CANCEL,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
HELP_RESPONSE,
|
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
|
type HelpFinishMessage,
|
||||||
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
type HelpResponseMessage,
|
isHelpAckMessage,
|
||||||
|
isHelpFinishMessage,
|
||||||
|
isHelpQueryMessage,
|
||||||
|
isHelpResponseMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
|
HelpCompletionCondition,
|
||||||
|
HelpCompletionCriteria,
|
||||||
HelpTarget,
|
HelpTarget,
|
||||||
RequestHelpParams,
|
RequestHelpParams,
|
||||||
RequestHelpResult,
|
RequestHelpResult,
|
||||||
@@ -34,53 +37,60 @@ import {
|
|||||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||||
|
|
||||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||||
/** Attribute the overlay highlights for resolved `ref` targets. */
|
|
||||||
const HELP_ATTR = "data-bsk-help";
|
const HELP_ATTR = "data-bsk-help";
|
||||||
|
const HELP_SEND_RETRIES = 3;
|
||||||
|
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||||
|
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||||
|
const HELP_REARM_MAX_ATTEMPTS = 12;
|
||||||
|
const HELP_REARM_RETRY_DELAY_MS = 400;
|
||||||
|
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
|
||||||
|
const COMPLETION_POLL_MS = 500;
|
||||||
|
|
||||||
export interface RequestHelpNotifications {
|
export interface RequestHelpNotifications {
|
||||||
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
||||||
clear(id: string): Promise<boolean>;
|
clear(id: string): Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TabNavigationUnsubscribe = () => void;
|
|
||||||
|
|
||||||
export interface RequestHelpDeps {
|
export interface RequestHelpDeps {
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
windows: {
|
windows: {
|
||||||
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
||||||
};
|
};
|
||||||
/** Activate a tab inside its window. */
|
|
||||||
activateTab(tabId: number): Promise<void>;
|
activateTab(tabId: number): Promise<void>;
|
||||||
/** Send the help-request to the tab's content script and await reply. */
|
|
||||||
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
||||||
/**
|
|
||||||
* Watch for navigation on `tabId` (full page load or SPA URL change).
|
|
||||||
* Invokes `onNavigated` once; returns unsubscribe.
|
|
||||||
*/
|
|
||||||
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
|
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
/** Pass `null` to skip OS notifications (tests). */
|
|
||||||
notifications: RequestHelpNotifications | null;
|
notifications: RequestHelpNotifications | null;
|
||||||
notificationCopy?: { title: string; body: string };
|
notificationCopy?: { title: string; body: string };
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
autoAttachLifecycle?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
|
interface ActiveHelpRequest {
|
||||||
export function defaultWatchTabNavigation(
|
sessionId: string;
|
||||||
tabId: number,
|
ctx: SessionContext;
|
||||||
onNavigated: () => void,
|
requestId: string;
|
||||||
): TabNavigationUnsubscribe {
|
tabId: number;
|
||||||
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
|
agentWindowId: number;
|
||||||
if (updatedTabId !== tabId) return;
|
prompt: string;
|
||||||
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
|
title?: string;
|
||||||
chrome.tabs.onUpdated.removeListener(listener);
|
targets: HelpTarget[];
|
||||||
onNavigated();
|
selectors: string[];
|
||||||
}
|
timeoutMs: number;
|
||||||
};
|
notificationId: string;
|
||||||
chrome.tabs.onUpdated.addListener(listener);
|
resolvedTargets?: ResolvedTarget[];
|
||||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
completionCriteria?: HelpCompletionCriteria;
|
||||||
|
deps: RequestHelpDeps;
|
||||||
|
settled: boolean;
|
||||||
|
resolve: (value: RequestHelpResult | RpcError) => void;
|
||||||
|
timeoutTimer: ReturnType<typeof setTimeout>;
|
||||||
|
completionTimer: ReturnType<typeof setInterval> | null;
|
||||||
|
completionMatchedSince: number | null;
|
||||||
|
abortHandler: (() => void) | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
|
||||||
|
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||||
|
|
||||||
let defaultDeps: RequestHelpDeps | null = null;
|
let defaultDeps: RequestHelpDeps | null = null;
|
||||||
function getDefaultDeps(): RequestHelpDeps {
|
function getDefaultDeps(): RequestHelpDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
@@ -91,7 +101,6 @@ function getDefaultDeps(): RequestHelpDeps {
|
|||||||
await chrome.tabs.update(tabId, { active: true });
|
await chrome.tabs.update(tabId, { active: true });
|
||||||
},
|
},
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
watchTabNavigation: defaultWatchTabNavigation,
|
|
||||||
notifications: {
|
notifications: {
|
||||||
create: (id, opts) =>
|
create: (id, opts) =>
|
||||||
new Promise((resolve, reject) =>
|
new Promise((resolve, reject) =>
|
||||||
@@ -113,11 +122,48 @@ function makeRequestId(tabId: number): string {
|
|||||||
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function sleep(ms: number): Promise<void> {
|
||||||
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
* content overlay can highlight it with a plain attribute selector.
|
}
|
||||||
* Returns the selector on success, or null when the ref can't resolve.
|
|
||||||
*/
|
function helpRequestMessage(help: ActiveHelpRequest): HelpRequestMessage {
|
||||||
|
return {
|
||||||
|
type: HELP_REQUEST,
|
||||||
|
requestId: help.requestId,
|
||||||
|
prompt: help.prompt,
|
||||||
|
...(help.title ? { title: help.title } : {}),
|
||||||
|
selectors: help.selectors,
|
||||||
|
timeoutMs: help.timeoutMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendHelpRequestWithAck(
|
||||||
|
tabId: number,
|
||||||
|
msg: HelpRequestMessage,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<HelpFinishMessage | null> {
|
||||||
|
let lastError: unknown;
|
||||||
|
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const response = await deps.sendToTab(tabId, msg);
|
||||||
|
if (isHelpAckMessage(response)) return null;
|
||||||
|
if (isHelpResponseMessage(response)) {
|
||||||
|
return {
|
||||||
|
type: HELP_FINISH,
|
||||||
|
requestId: msg.requestId,
|
||||||
|
outcome: response.outcome,
|
||||||
|
...(response.note ? { note: response.note } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
lastError = new Error("content script did not ack HELP_REQUEST");
|
||||||
|
} catch (err) {
|
||||||
|
lastError = err;
|
||||||
|
}
|
||||||
|
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
throw lastError ?? new Error("failed to show help overlay");
|
||||||
|
}
|
||||||
|
|
||||||
async function tagRefTarget(
|
async function tagRefTarget(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -145,7 +191,6 @@ async function tagRefTarget(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns true when `selector` matches a live element in the page. */
|
|
||||||
async function selectorExists(
|
async function selectorExists(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -163,7 +208,6 @@ async function selectorExists(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Remove every help attribute we added (best-effort cleanup). */
|
|
||||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||||
if (!cdp) return;
|
if (!cdp) return;
|
||||||
try {
|
try {
|
||||||
@@ -180,34 +224,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<
|
|||||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// best-effort
|
// Best-effort cleanup.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function handleRequestHelp(
|
async function resolveHelpTargets(
|
||||||
manager: SessionManager,
|
ctx: ReturnType<SessionManager["get"]>,
|
||||||
params: RequestHelpParams,
|
tabId: number,
|
||||||
deps: RequestHelpDeps = getDefaultDeps(),
|
targets: HelpTarget[],
|
||||||
): Promise<RequestHelpResult | RpcError> {
|
deps: RequestHelpDeps,
|
||||||
const ctxOrErr = lookupSession(manager, params, "request_help");
|
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
|
||||||
const ctx = ctxOrErr;
|
|
||||||
if (!params.prompt || typeof params.prompt !== "string") {
|
|
||||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
|
||||||
}
|
|
||||||
if (deps.signal?.aborted) {
|
|
||||||
return { code: "cancelled", message: "request_help aborted" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
|
||||||
if (isRpcError(target)) return target;
|
|
||||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
|
||||||
if (denied) return denied;
|
|
||||||
const tabId = target.tabId;
|
|
||||||
|
|
||||||
// Resolve targets → selectors. `selector` passes through; `ref` is
|
|
||||||
// tagged with an attribute via CDP and reported as its attr selector.
|
|
||||||
const targets = params.targets ?? [];
|
|
||||||
const selectors: string[] = [];
|
const selectors: string[] = [];
|
||||||
const resolved: ResolvedTarget[] = [];
|
const resolved: ResolvedTarget[] = [];
|
||||||
let rootNodeId: number | undefined;
|
let rootNodeId: number | undefined;
|
||||||
@@ -221,22 +247,20 @@ export async function handleRequestHelp(
|
|||||||
rootNodeId = undefined;
|
rootNodeId = undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (let i = 0; i < targets.length; i++) {
|
|
||||||
const tgt = targets[i] as HelpTarget;
|
for (let i = 0; i < targets.length; i += 1) {
|
||||||
|
const tgt = targets[i];
|
||||||
if (tgt.selector) {
|
if (tgt.selector) {
|
||||||
selectors.push(tgt.selector);
|
selectors.push(tgt.selector);
|
||||||
let matched: boolean;
|
const matched =
|
||||||
if (deps.cdp) {
|
deps.cdp && rootNodeId !== undefined
|
||||||
matched =
|
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||||
rootNodeId !== undefined
|
: deps.cdp
|
||||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
? false
|
||||||
: false;
|
: true;
|
||||||
} else {
|
|
||||||
matched = true;
|
|
||||||
}
|
|
||||||
resolved.push({ matched, selector: tgt.selector });
|
resolved.push({ matched, selector: tgt.selector });
|
||||||
} else if (tgt.ref) {
|
} else if (tgt.ref) {
|
||||||
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
|
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||||
const backendNodeId = looked?.backendNodeId ?? null;
|
const backendNodeId = looked?.backendNodeId ?? null;
|
||||||
let sel: string | null = null;
|
let sel: string | null = null;
|
||||||
if (backendNodeId !== null && deps.cdp) {
|
if (backendNodeId !== null && deps.cdp) {
|
||||||
@@ -247,7 +271,385 @@ export async function handleRequestHelp(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bring the tab to the foreground.
|
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||||
|
await clearRefTags(help.deps.cdp, help.tabId);
|
||||||
|
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||||
|
help.ctx,
|
||||||
|
help.tabId,
|
||||||
|
help.targets,
|
||||||
|
help.deps,
|
||||||
|
);
|
||||||
|
help.selectors = selectors;
|
||||||
|
help.resolvedTargets = resolvedTargets;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||||
|
await clearRefTags(help.deps.cdp, help.tabId);
|
||||||
|
if (help.deps.notifications) {
|
||||||
|
await help.deps.notifications.clear(help.notificationId).catch(() => {});
|
||||||
|
}
|
||||||
|
await help.deps
|
||||||
|
.sendToTab(help.tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||||
|
.catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
function finishHelp(
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
value: RequestHelpResult | RpcError,
|
||||||
|
notifyContent = true,
|
||||||
|
): void {
|
||||||
|
if (help.settled) return;
|
||||||
|
help.settled = true;
|
||||||
|
activeHelpRequests.delete(help.requestId);
|
||||||
|
clearTimeout(help.timeoutTimer);
|
||||||
|
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||||
|
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
|
||||||
|
clearRearmTimer(help.tabId);
|
||||||
|
if (notifyContent) void cleanupHelp(help);
|
||||||
|
help.resolve(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findHelpByTabId(tabId: number): ActiveHelpRequest | null {
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
if (!help.settled && help.tabId === tabId) return help;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findHelpForTab(
|
||||||
|
tabId: number,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<ActiveHelpRequest | null> {
|
||||||
|
const direct = findHelpByTabId(tabId);
|
||||||
|
if (direct) return direct;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const tab = await deps.tabsApi.get(tabId);
|
||||||
|
const windowId = tab.windowId;
|
||||||
|
if (typeof windowId !== "number") return null;
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
if (!help.settled && help.agentWindowId === windowId) return help;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearRearmTimer(tabId: number): void {
|
||||||
|
const timer = helpRearmTimers.get(tabId);
|
||||||
|
if (!timer) return;
|
||||||
|
clearTimeout(timer);
|
||||||
|
helpRearmTimers.delete(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rearmHelp(help: ActiveHelpRequest, targetTabId: number): Promise<boolean> {
|
||||||
|
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
|
try {
|
||||||
|
help.tabId = targetTabId;
|
||||||
|
await refreshHelpTargets(help);
|
||||||
|
const msg = helpRequestMessage(help);
|
||||||
|
const legacyFinish = await sendHelpRequestWithAck(targetTabId, msg, help.deps);
|
||||||
|
if (legacyFinish && !help.settled) {
|
||||||
|
finishHelp(help, {
|
||||||
|
outcome: legacyFinish.outcome,
|
||||||
|
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||||
|
tab_id: help.tabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
|
||||||
|
const existing = helpRearmTimers.get(tabId);
|
||||||
|
if (existing) clearTimeout(existing);
|
||||||
|
helpRearmTimers.set(
|
||||||
|
tabId,
|
||||||
|
setTimeout(() => {
|
||||||
|
helpRearmTimers.delete(tabId);
|
||||||
|
void (async () => {
|
||||||
|
const help = await findHelpForTab(tabId, deps);
|
||||||
|
if (help) await rearmHelp(help, tabId);
|
||||||
|
})();
|
||||||
|
}, HELP_REARM_DEBOUNCE_MS),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let detachHelpObservation: (() => void) | null = null;
|
||||||
|
|
||||||
|
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
|
||||||
|
if (detachHelpObservation) return;
|
||||||
|
const detachRuntime = attachHelpRuntimeListener(deps);
|
||||||
|
const detachTabs = attachHelpTabListener(deps);
|
||||||
|
const detachNav = attachHelpNavigationListener(deps);
|
||||||
|
detachHelpObservation = () => {
|
||||||
|
detachRuntime();
|
||||||
|
detachTabs();
|
||||||
|
detachNav();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resetHelpLifecycleForTests(): void {
|
||||||
|
detachHelpObservation?.();
|
||||||
|
detachHelpObservation = null;
|
||||||
|
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
|
||||||
|
helpRearmTimers.clear();
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
clearTimeout(help.timeoutTimer);
|
||||||
|
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||||
|
}
|
||||||
|
activeHelpRequests.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const listener = (
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: HelpQueryResponse) => void,
|
||||||
|
) => {
|
||||||
|
if (isHelpFinishMessage(message)) {
|
||||||
|
void finishHelpFromContent(message, sender, deps);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isHelpQueryMessage(message)) {
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
if (tabId === undefined) {
|
||||||
|
sendResponse({ active: false });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
void (async () => {
|
||||||
|
const help = await findHelpForTab(tabId, deps);
|
||||||
|
if (!help) {
|
||||||
|
sendResponse({ active: false });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
help.tabId = tabId;
|
||||||
|
await refreshHelpTargets(help);
|
||||||
|
sendResponse({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: help.requestId,
|
||||||
|
prompt: help.prompt,
|
||||||
|
...(help.title ? { title: help.title } : {}),
|
||||||
|
selectors: help.selectors,
|
||||||
|
timeoutMs: help.timeoutMs,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
chrome.runtime.onMessage.addListener(listener);
|
||||||
|
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finishHelpFromContent(
|
||||||
|
message: HelpFinishMessage,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
const help = activeHelpRequests.get(message.requestId);
|
||||||
|
if (!help || help.settled) return;
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
if (tabId !== undefined) {
|
||||||
|
const match = await findHelpForTab(tabId, deps);
|
||||||
|
if (match !== help) return;
|
||||||
|
}
|
||||||
|
finishHelp(
|
||||||
|
help,
|
||||||
|
{
|
||||||
|
outcome: message.outcome,
|
||||||
|
...(message.note ? { note: message.note } : {}),
|
||||||
|
tab_id: help.tabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
},
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||||
|
const tabId = tab.id;
|
||||||
|
const windowId = tab.windowId;
|
||||||
|
if (tabId === undefined || windowId === undefined) return;
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
if (!help.settled && help.agentWindowId === windowId) {
|
||||||
|
scheduleRearmForTab(tabId, deps);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||||
|
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||||
|
};
|
||||||
|
chrome.tabs.onCreated?.addListener(onCreated);
|
||||||
|
chrome.tabs.onActivated?.addListener(onActivated);
|
||||||
|
return () => {
|
||||||
|
chrome.tabs.onCreated?.removeListener(onCreated);
|
||||||
|
chrome.tabs.onActivated?.removeListener(onActivated);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
|
||||||
|
|
||||||
|
if (chrome.webNavigation?.onCompleted) {
|
||||||
|
const completedListener = (
|
||||||
|
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||||
|
) => {
|
||||||
|
if (details.frameId !== 0) return;
|
||||||
|
onMainFrameComplete(details.tabId);
|
||||||
|
};
|
||||||
|
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||||
|
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||||
|
}
|
||||||
|
|
||||||
|
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||||
|
if (info.status !== "complete") return;
|
||||||
|
onMainFrameComplete(tabId);
|
||||||
|
};
|
||||||
|
chrome.tabs.onUpdated?.addListener(listener);
|
||||||
|
return () => chrome.tabs.onUpdated?.removeListener(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evaluateCompletionCondition(
|
||||||
|
condition: HelpCompletionCondition,
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
): Promise<boolean> {
|
||||||
|
if (condition.url_contains || condition.url_matches) {
|
||||||
|
let url = "";
|
||||||
|
try {
|
||||||
|
const tab = await help.deps.tabsApi.get(help.tabId);
|
||||||
|
url = tab.url ?? "";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
|
||||||
|
if (condition.url_matches) {
|
||||||
|
try {
|
||||||
|
if (!new RegExp(condition.url_matches).test(url)) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
condition.selector_exists ||
|
||||||
|
condition.selector_missing ||
|
||||||
|
condition.text_exists ||
|
||||||
|
condition.text_missing
|
||||||
|
) {
|
||||||
|
if (!help.deps.cdp) return false;
|
||||||
|
const expression = `(() => {
|
||||||
|
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
|
||||||
|
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
|
||||||
|
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
|
||||||
|
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
|
||||||
|
if (selectorExists && !document.querySelector(selectorExists)) return false;
|
||||||
|
if (selectorMissing && document.querySelector(selectorMissing)) return false;
|
||||||
|
const text = document.body ? document.body.innerText || "" : "";
|
||||||
|
if (textExists && !text.includes(textExists)) return false;
|
||||||
|
if (textMissing && text.includes(textMissing)) return false;
|
||||||
|
return true;
|
||||||
|
})()`;
|
||||||
|
try {
|
||||||
|
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
|
||||||
|
help.tabId,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{ expression, returnByValue: true },
|
||||||
|
);
|
||||||
|
if (result.result?.value !== true) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
|
||||||
|
const criteria = help.completionCriteria;
|
||||||
|
if (!criteria) return false;
|
||||||
|
const any = criteria.any ?? [];
|
||||||
|
const all = criteria.all ?? [];
|
||||||
|
const hasAny = any.length > 0;
|
||||||
|
const hasAll = all.length > 0;
|
||||||
|
if (!hasAny && !hasAll) return false;
|
||||||
|
|
||||||
|
if (hasAll) {
|
||||||
|
for (const condition of all) {
|
||||||
|
if (!(await evaluateCompletionCondition(condition, help))) return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (hasAny) {
|
||||||
|
for (const condition of any) {
|
||||||
|
if (await evaluateCompletionCondition(condition, help)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function startCompletionPolling(help: ActiveHelpRequest): void {
|
||||||
|
if (!help.completionCriteria) return;
|
||||||
|
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
|
||||||
|
const check = async () => {
|
||||||
|
if (help.settled) return;
|
||||||
|
const matched = await completionCriteriaMatches(help);
|
||||||
|
const now = Date.now();
|
||||||
|
if (!matched) {
|
||||||
|
help.completionMatchedSince = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
help.completionMatchedSince ??= now;
|
||||||
|
if (now - help.completionMatchedSince < stableMs) return;
|
||||||
|
finishHelp(help, {
|
||||||
|
outcome: "completed",
|
||||||
|
completed_by: "system",
|
||||||
|
tab_id: help.tabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
|
||||||
|
void check();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleRequestHelp(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: RequestHelpParams,
|
||||||
|
deps: RequestHelpDeps = getDefaultDeps(),
|
||||||
|
): Promise<RequestHelpResult | RpcError> {
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "request_help");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
if (!params.prompt || typeof params.prompt !== "string") {
|
||||||
|
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||||
|
}
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
|
||||||
|
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||||
|
if (denied) return denied;
|
||||||
|
|
||||||
|
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||||
|
const tabId = target.tabId;
|
||||||
|
const initialTargets = params.targets ?? [];
|
||||||
|
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||||
|
|
||||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||||
await deps.activateTab(tabId).catch(() => {});
|
await deps.activateTab(tabId).catch(() => {});
|
||||||
|
|
||||||
@@ -255,7 +657,6 @@ export async function handleRequestHelp(
|
|||||||
const notificationId = `bsk-help:${requestId}`;
|
const notificationId = `bsk-help:${requestId}`;
|
||||||
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
||||||
|
|
||||||
// OS notification (best-effort).
|
|
||||||
if (deps.notifications) {
|
if (deps.notifications) {
|
||||||
const copy = deps.notificationCopy ?? {
|
const copy = deps.notificationCopy ?? {
|
||||||
title: "BrowserSkill: Agent needs your help",
|
title: "BrowserSkill: Agent needs your help",
|
||||||
@@ -272,85 +673,60 @@ export async function handleRequestHelp(
|
|||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
const cleanup = async () => {
|
return new Promise<RequestHelpResult | RpcError>((resolve) => {
|
||||||
await clearRefTags(deps.cdp, tabId);
|
const timeoutTimer = setTimeout(() => {
|
||||||
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
|
finishHelp(help, {
|
||||||
// Retract the overlay in case we resolved via timeout / abort.
|
outcome: "timed_out",
|
||||||
await deps
|
tab_id: help.tabId,
|
||||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
|
resolved_targets: help.resolvedTargets,
|
||||||
.catch(() => {});
|
});
|
||||||
};
|
}, timeoutMs);
|
||||||
|
|
||||||
const request: HelpRequestMessage = {
|
const help: ActiveHelpRequest = {
|
||||||
type: HELP_REQUEST,
|
sessionId: params.session_id,
|
||||||
requestId,
|
ctx,
|
||||||
prompt: params.prompt,
|
requestId,
|
||||||
...(params.title ? { title: params.title } : {}),
|
tabId,
|
||||||
selectors,
|
agentWindowId: ctx.agentWindowId,
|
||||||
timeoutMs,
|
prompt: params.prompt,
|
||||||
};
|
...(params.title ? { title: params.title } : {}),
|
||||||
|
targets: initialTargets,
|
||||||
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
|
selectors,
|
||||||
|
timeoutMs,
|
||||||
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
|
notificationId,
|
||||||
let settled = false;
|
resolvedTargets,
|
||||||
const unwatchNav = deps.watchTabNavigation(tabId, () => {
|
completionCriteria: params.completion_criteria,
|
||||||
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
|
deps,
|
||||||
});
|
settled: false,
|
||||||
|
resolve,
|
||||||
const finish = (value: RequestHelpResult | RpcError) => {
|
timeoutTimer,
|
||||||
if (settled) return;
|
completionTimer: null,
|
||||||
settled = true;
|
completionMatchedSince: null,
|
||||||
if (timer !== null) clearTimeout(timer);
|
abortHandler: null,
|
||||||
unwatchNav();
|
|
||||||
deps.signal?.removeEventListener("abort", onAbort);
|
|
||||||
void cleanup();
|
|
||||||
resolveOuter(value);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
|
const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" });
|
||||||
|
help.abortHandler = onAbort;
|
||||||
const timer = setTimeout(() => {
|
activeHelpRequests.set(requestId, help);
|
||||||
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
|
if (deps.signal?.aborted) {
|
||||||
}, timeoutMs) as unknown as number;
|
onAbort();
|
||||||
|
return;
|
||||||
if (deps.signal) {
|
|
||||||
if (deps.signal.aborted) {
|
|
||||||
onAbort();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
deps.signal.addEventListener("abort", onAbort, { once: true });
|
|
||||||
}
|
}
|
||||||
|
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
startCompletionPolling(help);
|
||||||
|
|
||||||
deps
|
void sendHelpRequestWithAck(tabId, helpRequestMessage(help), deps)
|
||||||
.sendToTab(tabId, request)
|
.then((legacyFinish) => {
|
||||||
.then((reply) => {
|
if (!legacyFinish || help.settled) return;
|
||||||
const res = reply as HelpResponseMessage | undefined;
|
finishHelp(help, {
|
||||||
if (
|
outcome: legacyFinish.outcome,
|
||||||
!res ||
|
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||||
res.type !== HELP_RESPONSE ||
|
tab_id: help.tabId,
|
||||||
(res.outcome !== "continued" && res.outcome !== "cancelled")
|
resolved_targets: help.resolvedTargets,
|
||||||
) {
|
|
||||||
finish({
|
|
||||||
code: "protocol_error",
|
|
||||||
message: `invalid help response from tab ${tabId}`,
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
finish({
|
|
||||||
outcome: res.outcome,
|
|
||||||
...(res.note ? { note: res.note } : {}),
|
|
||||||
tab_id: tabId,
|
|
||||||
resolved_targets: resolvedTargets,
|
|
||||||
});
|
});
|
||||||
})
|
})
|
||||||
.catch((err) => {
|
.catch(() => {
|
||||||
finish({
|
scheduleRearmForTab(tabId, deps);
|
||||||
code: "protocol_error",
|
|
||||||
message: `failed to show help overlay on tab ${tabId}: ${
|
|
||||||
err instanceof Error ? err.message : String(err)
|
|
||||||
}`,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -514,7 +514,22 @@ export interface HelpTarget {
|
|||||||
selector?: string;
|
selector?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated";
|
export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated";
|
||||||
|
|
||||||
|
export interface HelpCompletionCondition {
|
||||||
|
url_contains?: string;
|
||||||
|
url_matches?: string;
|
||||||
|
selector_exists?: string;
|
||||||
|
selector_missing?: string;
|
||||||
|
text_exists?: string;
|
||||||
|
text_missing?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpCompletionCriteria {
|
||||||
|
any?: HelpCompletionCondition[];
|
||||||
|
all?: HelpCompletionCondition[];
|
||||||
|
stable_for_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface ResolvedTarget {
|
export interface ResolvedTarget {
|
||||||
matched: boolean;
|
matched: boolean;
|
||||||
@@ -528,11 +543,13 @@ export interface RequestHelpParams {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
targets?: HelpTarget[];
|
targets?: HelpTarget[];
|
||||||
|
completion_criteria?: HelpCompletionCriteria;
|
||||||
timeout_ms?: number;
|
timeout_ms?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RequestHelpResult {
|
export interface RequestHelpResult {
|
||||||
outcome: HelpOutcome;
|
outcome: HelpOutcome;
|
||||||
|
completed_by?: "system";
|
||||||
note?: string;
|
note?: string;
|
||||||
tab_id: number;
|
tab_id: number;
|
||||||
resolved_targets?: ResolvedTarget[];
|
resolved_targets?: ResolvedTarget[];
|
||||||
|
|||||||
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
|||||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||||
confirm an important action, pause and ask:
|
confirm an important action, pause and ask:
|
||||||
|
|
||||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||||
|
|
||||||
- `--prompt` (required): what the user should do.
|
- `--prompt` (required): what the user should do.
|
||||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
|||||||
prompt with no `--target` for cases where there is genuinely no specific
|
prompt with no `--target` for cases where there is genuinely no specific
|
||||||
element to point at (e.g. "wait for the page to finish loading").
|
element to point at (e.g. "wait for the page to finish loading").
|
||||||
- `--timeout` (default `5m`): how long to wait.
|
- `--timeout` (default `5m`): how long to wait.
|
||||||
|
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||||
|
when there is a concrete post-help success signal, e.g.
|
||||||
|
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||||
|
|
||||||
The target tab is brought to the foreground; the page stays interactive
|
The target tab is brought to the foreground; the page stays interactive
|
||||||
while the agent control mask is hidden. The call blocks until the user
|
while the agent control mask is hidden. The call blocks until the user
|
||||||
acts. The result `outcome` is one of:
|
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||||
|
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||||
|
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||||
|
|
||||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||||
- `timed_out` — nobody acted within the timeout.
|
- `timed_out` — nobody acted within the timeout.
|
||||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||||
|
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||||
|
|
||||||
`note` carries any text the user typed back. `resolved_targets` reports
|
`note` carries any text the user typed back. `resolved_targets` reports
|
||||||
which refs/selectors matched a live element.
|
which refs/selectors matched a live element.
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use bsk_protocol::Method;
|
use bsk_protocol::Method;
|
||||||
use bsk_protocol::tools::{HelpTarget, RequestHelpParams, RequestHelpResult};
|
use bsk_protocol::tools::{
|
||||||
|
HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult,
|
||||||
|
};
|
||||||
use clap::Args;
|
use clap::Args;
|
||||||
|
|
||||||
use crate::cli::ensure_daemon::ensure_daemon;
|
use crate::cli::ensure_daemon::ensure_daemon;
|
||||||
@@ -41,6 +43,10 @@ pub struct RequestHelpArgs {
|
|||||||
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
|
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
|
||||||
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
|
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
|
||||||
pub timeout: u32,
|
pub timeout: u32,
|
||||||
|
|
||||||
|
/// JSON success detector, e.g. '{"any":[{"url_contains":"/dashboard"}],"stable_for_ms":1000}'.
|
||||||
|
#[arg(long = "completion-criteria")]
|
||||||
|
pub completion_criteria: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
|
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
|
||||||
@@ -68,6 +74,12 @@ pub fn parse_target(raw: &str) -> HelpTarget {
|
|||||||
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
||||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||||
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
|
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
|
||||||
|
let completion_criteria = args
|
||||||
|
.completion_criteria
|
||||||
|
.as_deref()
|
||||||
|
.map(|raw| serde_json::from_str::<HelpCompletionCriteria>(raw))
|
||||||
|
.transpose()
|
||||||
|
.context("parse --completion-criteria JSON")?;
|
||||||
let params = RequestHelpParams {
|
let params = RequestHelpParams {
|
||||||
session_id: args.session,
|
session_id: args.session,
|
||||||
tab_id: args.tab_id,
|
tab_id: args.tab_id,
|
||||||
@@ -78,6 +90,7 @@ pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
|||||||
} else {
|
} else {
|
||||||
Some(targets)
|
Some(targets)
|
||||||
},
|
},
|
||||||
|
completion_criteria,
|
||||||
timeout_ms: Some(args.timeout),
|
timeout_ms: Some(args.timeout),
|
||||||
};
|
};
|
||||||
let reply = call(info.sock_path, params, args.timeout)?;
|
let reply = call(info.sock_path, params, args.timeout)?;
|
||||||
|
|||||||
@@ -7,6 +7,17 @@
|
|||||||
"session_id"
|
"session_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"completion_criteria": {
|
||||||
|
"description": "Optional explicit success detector. Navigation by itself is not a completion signal; these criteria must match before the tool can auto-complete without the user clicking Done.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/HelpCompletionCriteria"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"prompt": {
|
"prompt": {
|
||||||
"description": "Message shown to the user explaining what to do.",
|
"description": "Message shown to the user explaining what to do.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
@@ -50,6 +61,78 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"HelpCompletionCondition": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"selector_exists": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"selector_missing": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"text_exists": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"text_missing": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"url_contains": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"url_matches": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"HelpCompletionCriteria": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"all": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/HelpCompletionCondition"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"any": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/HelpCompletionCondition"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"stable_for_ms": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"HelpTarget": {
|
"HelpTarget": {
|
||||||
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
|
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -7,6 +7,16 @@
|
|||||||
"tab_id"
|
"tab_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"completed_by": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/HelpCompletedBy"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"note": {
|
"note": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
@@ -31,10 +41,16 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"HelpCompletedBy": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"system"
|
||||||
|
]
|
||||||
|
},
|
||||||
"HelpOutcome": {
|
"HelpOutcome": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
"description": "User completed the step and clicked Continue.",
|
"description": "User completed the step and clicked Done / return control.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": [
|
"enum": [
|
||||||
"continued"
|
"continued"
|
||||||
@@ -55,7 +71,14 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "The page navigated while waiting (full reload or SPA URL change).",
|
"description": "Explicit success criteria matched while the user had control.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"completed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "The page navigated while waiting (full reload or SPA URL change). Deprecated: navigation alone should not end a help request.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": [
|
"enum": [
|
||||||
"navigated"
|
"navigated"
|
||||||
|
|||||||
@@ -3,7 +3,8 @@
|
|||||||
//! Lets an agent pause and ask the human to complete an in-page step
|
//! Lets an agent pause and ask the human to complete an in-page step
|
||||||
//! (captcha, login, confirmation). The extension brings the target tab
|
//! (captcha, login, confirmation). The extension brings the target tab
|
||||||
//! to the foreground, highlights the requested components, and blocks
|
//! to the foreground, highlights the requested components, and blocks
|
||||||
//! until the user clicks Continue / Cancel (or the call times out).
|
//! until the user clicks Done / Cancel, explicit completion criteria match,
|
||||||
|
//! or the call times out.
|
||||||
|
|
||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -41,6 +42,11 @@ pub struct RequestHelpParams {
|
|||||||
/// Optional components to scroll to + flash-highlight.
|
/// Optional components to scroll to + flash-highlight.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub targets: Option<Vec<HelpTarget>>,
|
pub targets: Option<Vec<HelpTarget>>,
|
||||||
|
/// Optional explicit success detector. Navigation by itself is not a
|
||||||
|
/// completion signal; these criteria must match before the tool can
|
||||||
|
/// auto-complete without the user clicking Done.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub completion_criteria: Option<HelpCompletionCriteria>,
|
||||||
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
|
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
|
||||||
/// enforced daemon-side via the generic tool timeout.
|
/// enforced daemon-side via the generic tool timeout.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
@@ -48,16 +54,46 @@ pub struct RequestHelpParams {
|
|||||||
pub timeout_ms: Option<u32>,
|
pub timeout_ms: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HelpCompletionCriteria {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub any: Option<Vec<HelpCompletionCondition>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub all: Option<Vec<HelpCompletionCondition>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[schemars(range(min = 0))]
|
||||||
|
pub stable_for_ms: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HelpCompletionCondition {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub url_contains: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub url_matches: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector_exists: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector_missing: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub text_exists: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub text_missing: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum HelpOutcome {
|
pub enum HelpOutcome {
|
||||||
/// User completed the step and clicked Continue.
|
/// User completed the step and clicked Done / return control.
|
||||||
Continued,
|
Continued,
|
||||||
/// User declined / aborted via the overlay's Cancel button.
|
/// User declined / aborted via the overlay's Cancel button.
|
||||||
Cancelled,
|
Cancelled,
|
||||||
/// The wait expired before the user acted.
|
/// The wait expired before the user acted.
|
||||||
TimedOut,
|
TimedOut,
|
||||||
|
/// Explicit success criteria matched while the user had control.
|
||||||
|
Completed,
|
||||||
/// The page navigated while waiting (full reload or SPA URL change).
|
/// The page navigated while waiting (full reload or SPA URL change).
|
||||||
|
/// Deprecated: navigation alone should not end a help request.
|
||||||
Navigated,
|
Navigated,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,11 +103,18 @@ impl HelpOutcome {
|
|||||||
Self::Continued => "continued",
|
Self::Continued => "continued",
|
||||||
Self::Cancelled => "cancelled",
|
Self::Cancelled => "cancelled",
|
||||||
Self::TimedOut => "timed_out",
|
Self::TimedOut => "timed_out",
|
||||||
|
Self::Completed => "completed",
|
||||||
Self::Navigated => "navigated",
|
Self::Navigated => "navigated",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum HelpCompletedBy {
|
||||||
|
System,
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-target resolution status echoed back so the agent can tell which
|
/// Per-target resolution status echoed back so the agent can tell which
|
||||||
/// `ref` / `selector` actually matched a live element.
|
/// `ref` / `selector` actually matched a live element.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
@@ -87,6 +130,8 @@ pub struct ResolvedTarget {
|
|||||||
pub struct RequestHelpResult {
|
pub struct RequestHelpResult {
|
||||||
pub outcome: HelpOutcome,
|
pub outcome: HelpOutcome,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub completed_by: Option<HelpCompletedBy>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub note: Option<String>,
|
pub note: Option<String>,
|
||||||
pub tab_id: i64,
|
pub tab_id: i64,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
@@ -123,6 +168,7 @@ mod tests {
|
|||||||
prompt: "log in".into(),
|
prompt: "log in".into(),
|
||||||
title: None,
|
title: None,
|
||||||
targets: None,
|
targets: None,
|
||||||
|
completion_criteria: None,
|
||||||
timeout_ms: None,
|
timeout_ms: None,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&p).unwrap();
|
let v = serde_json::to_value(&p).unwrap();
|
||||||
@@ -140,6 +186,10 @@ mod tests {
|
|||||||
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
|
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
|
||||||
json!("timed_out")
|
json!("timed_out")
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(HelpOutcome::Completed).unwrap(),
|
||||||
|
json!("completed")
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::to_value(HelpOutcome::Continued).unwrap(),
|
serde_json::to_value(HelpOutcome::Continued).unwrap(),
|
||||||
json!("continued")
|
json!("continued")
|
||||||
@@ -158,6 +208,7 @@ mod tests {
|
|||||||
fn result_round_trips() {
|
fn result_round_trips() {
|
||||||
let r = RequestHelpResult {
|
let r = RequestHelpResult {
|
||||||
outcome: HelpOutcome::Continued,
|
outcome: HelpOutcome::Continued,
|
||||||
|
completed_by: None,
|
||||||
note: Some("done".into()),
|
note: Some("done".into()),
|
||||||
tab_id: 7,
|
tab_id: 7,
|
||||||
resolved_targets: Some(vec![ResolvedTarget {
|
resolved_targets: Some(vec![ResolvedTarget {
|
||||||
|
|||||||
@@ -76,8 +76,8 @@
|
|||||||
"title": "Agent needs your help",
|
"title": "Agent needs your help",
|
||||||
"noteLabel": "Optional note back to the agent",
|
"noteLabel": "Optional note back to the agent",
|
||||||
"notePlaceholder": "Add a note for the agent (optional)",
|
"notePlaceholder": "Add a note for the agent (optional)",
|
||||||
"continue": "Continue",
|
"continue": "Done, return control",
|
||||||
"cancel": "Cancel",
|
"cancel": "Cancel request",
|
||||||
"collapse": "Collapse",
|
"collapse": "Collapse",
|
||||||
"expand": "Expand",
|
"expand": "Expand",
|
||||||
"dragHandle": "Drag to move",
|
"dragHandle": "Drag to move",
|
||||||
|
|||||||
@@ -76,8 +76,8 @@
|
|||||||
"title": "Agent 需要你的帮助",
|
"title": "Agent 需要你的帮助",
|
||||||
"noteLabel": "给 Agent 的备注(可选)",
|
"noteLabel": "给 Agent 的备注(可选)",
|
||||||
"notePlaceholder": "给 Agent 留言(可选)",
|
"notePlaceholder": "给 Agent 留言(可选)",
|
||||||
"continue": "继续",
|
"continue": "完成并交还控制权",
|
||||||
"cancel": "取消",
|
"cancel": "取消请求",
|
||||||
"collapse": "折叠",
|
"collapse": "折叠",
|
||||||
"expand": "展开",
|
"expand": "展开",
|
||||||
"dragHandle": "拖动以移动",
|
"dragHandle": "拖动以移动",
|
||||||
|
|||||||
+10
-4
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
|||||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||||
confirm an important action, pause and ask:
|
confirm an important action, pause and ask:
|
||||||
|
|
||||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||||
|
|
||||||
- `--prompt` (required): what the user should do.
|
- `--prompt` (required): what the user should do.
|
||||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
|||||||
prompt with no `--target` for cases where there is genuinely no specific
|
prompt with no `--target` for cases where there is genuinely no specific
|
||||||
element to point at (e.g. "wait for the page to finish loading").
|
element to point at (e.g. "wait for the page to finish loading").
|
||||||
- `--timeout` (default `5m`): how long to wait.
|
- `--timeout` (default `5m`): how long to wait.
|
||||||
|
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||||
|
when there is a concrete post-help success signal, e.g.
|
||||||
|
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||||
|
|
||||||
The target tab is brought to the foreground; the page stays interactive
|
The target tab is brought to the foreground; the page stays interactive
|
||||||
while the agent control mask is hidden. The call blocks until the user
|
while the agent control mask is hidden. The call blocks until the user
|
||||||
acts. The result `outcome` is one of:
|
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||||
|
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||||
|
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||||
|
|
||||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||||
- `timed_out` — nobody acted within the timeout.
|
- `timed_out` — nobody acted within the timeout.
|
||||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||||
|
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||||
|
|
||||||
`note` carries any text the user typed back. `resolved_targets` reports
|
`note` carries any text the user typed back. `resolved_targets` reports
|
||||||
which refs/selectors matched a live element.
|
which refs/selectors matched a live element.
|
||||||
|
|||||||
Reference in New Issue
Block a user