mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(runtime): fix early control return and update lifecycle
This commit is contained in:
@@ -178,7 +178,7 @@ Details and flags: **`bsk <cmd> --help`**
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Continue" \
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
@@ -193,15 +193,21 @@ confirm an important action, pause and ask:
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
acts. The result `outcome` is one of:
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Continue (treat as confirm).
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
@@ -7,7 +7,9 @@ use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{HelpTarget, RequestHelpParams, RequestHelpResult};
|
||||
use bsk_protocol::tools::{
|
||||
HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult,
|
||||
};
|
||||
use clap::Args;
|
||||
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
@@ -41,6 +43,10 @@ pub struct RequestHelpArgs {
|
||||
/// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`.
|
||||
#[arg(long, default_value = "5m", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
|
||||
/// JSON success detector, e.g. '{"any":[{"url_contains":"/dashboard"}],"stable_for_ms":1000}'.
|
||||
#[arg(long = "completion-criteria")]
|
||||
pub completion_criteria: Option<String>,
|
||||
}
|
||||
|
||||
/// Classify a `--target` value as a ref or a selector. Refs are `@eN`
|
||||
@@ -68,6 +74,12 @@ pub fn parse_target(raw: &str) -> HelpTarget {
|
||||
pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let targets: Vec<HelpTarget> = args.target.iter().map(|t| parse_target(t)).collect();
|
||||
let completion_criteria = args
|
||||
.completion_criteria
|
||||
.as_deref()
|
||||
.map(|raw| serde_json::from_str::<HelpCompletionCriteria>(raw))
|
||||
.transpose()
|
||||
.context("parse --completion-criteria JSON")?;
|
||||
let params = RequestHelpParams {
|
||||
session_id: args.session,
|
||||
tab_id: args.tab_id,
|
||||
@@ -78,6 +90,7 @@ pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> {
|
||||
} else {
|
||||
Some(targets)
|
||||
},
|
||||
completion_criteria,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply = call(info.sock_path, params, args.timeout)?;
|
||||
|
||||
@@ -7,6 +7,17 @@
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"completion_criteria": {
|
||||
"description": "Optional explicit success detector. Navigation by itself is not a completion signal; these criteria must match before the tool can auto-complete without the user clicking Done.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HelpCompletionCriteria"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"prompt": {
|
||||
"description": "Message shown to the user explaining what to do.",
|
||||
"type": "string"
|
||||
@@ -50,6 +61,78 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HelpCompletionCondition": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"selector_exists": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector_missing": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"text_exists": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"text_missing": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url_contains": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url_matches": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"HelpCompletionCriteria": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"all": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/HelpCompletionCondition"
|
||||
}
|
||||
},
|
||||
"any": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/HelpCompletionCondition"
|
||||
}
|
||||
},
|
||||
"stable_for_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"HelpTarget": {
|
||||
"description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].",
|
||||
"type": "object",
|
||||
|
||||
@@ -7,6 +7,16 @@
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"completed_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HelpCompletedBy"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"note": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -31,10 +41,16 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HelpCompletedBy": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"system"
|
||||
]
|
||||
},
|
||||
"HelpOutcome": {
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "User completed the step and clicked Continue.",
|
||||
"description": "User completed the step and clicked Done / return control.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"continued"
|
||||
@@ -55,7 +71,14 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "The page navigated while waiting (full reload or SPA URL change).",
|
||||
"description": "Explicit success criteria matched while the user had control.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"completed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "The page navigated while waiting (full reload or SPA URL change). Deprecated: navigation alone should not end a help request.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigated"
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
//! Lets an agent pause and ask the human to complete an in-page step
|
||||
//! (captcha, login, confirmation). The extension brings the target tab
|
||||
//! to the foreground, highlights the requested components, and blocks
|
||||
//! until the user clicks Continue / Cancel (or the call times out).
|
||||
//! until the user clicks Done / Cancel, explicit completion criteria match,
|
||||
//! or the call times out.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -41,6 +42,11 @@ pub struct RequestHelpParams {
|
||||
/// Optional components to scroll to + flash-highlight.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub targets: Option<Vec<HelpTarget>>,
|
||||
/// Optional explicit success detector. Navigation by itself is not a
|
||||
/// completion signal; these criteria must match before the tool can
|
||||
/// auto-complete without the user clicking Done.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completion_criteria: Option<HelpCompletionCriteria>,
|
||||
/// Hard upper bound on the wait. Defaults to 300000 (5 min),
|
||||
/// enforced daemon-side via the generic tool timeout.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -48,16 +54,46 @@ pub struct RequestHelpParams {
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HelpCompletionCriteria {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub any: Option<Vec<HelpCompletionCondition>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub all: Option<Vec<HelpCompletionCondition>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 0))]
|
||||
pub stable_for_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HelpCompletionCondition {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url_contains: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url_matches: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector_exists: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector_missing: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_exists: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_missing: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HelpOutcome {
|
||||
/// User completed the step and clicked Continue.
|
||||
/// User completed the step and clicked Done / return control.
|
||||
Continued,
|
||||
/// User declined / aborted via the overlay's Cancel button.
|
||||
Cancelled,
|
||||
/// The wait expired before the user acted.
|
||||
TimedOut,
|
||||
/// Explicit success criteria matched while the user had control.
|
||||
Completed,
|
||||
/// The page navigated while waiting (full reload or SPA URL change).
|
||||
/// Deprecated: navigation alone should not end a help request.
|
||||
Navigated,
|
||||
}
|
||||
|
||||
@@ -67,11 +103,18 @@ impl HelpOutcome {
|
||||
Self::Continued => "continued",
|
||||
Self::Cancelled => "cancelled",
|
||||
Self::TimedOut => "timed_out",
|
||||
Self::Completed => "completed",
|
||||
Self::Navigated => "navigated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HelpCompletedBy {
|
||||
System,
|
||||
}
|
||||
|
||||
/// Per-target resolution status echoed back so the agent can tell which
|
||||
/// `ref` / `selector` actually matched a live element.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -87,6 +130,8 @@ pub struct ResolvedTarget {
|
||||
pub struct RequestHelpResult {
|
||||
pub outcome: HelpOutcome,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completed_by: Option<HelpCompletedBy>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub note: Option<String>,
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -123,6 +168,7 @@ mod tests {
|
||||
prompt: "log in".into(),
|
||||
title: None,
|
||||
targets: None,
|
||||
completion_criteria: None,
|
||||
timeout_ms: None,
|
||||
};
|
||||
let v = serde_json::to_value(&p).unwrap();
|
||||
@@ -140,6 +186,10 @@ mod tests {
|
||||
serde_json::to_value(HelpOutcome::TimedOut).unwrap(),
|
||||
json!("timed_out")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(HelpOutcome::Completed).unwrap(),
|
||||
json!("completed")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(HelpOutcome::Continued).unwrap(),
|
||||
json!("continued")
|
||||
@@ -158,6 +208,7 @@ mod tests {
|
||||
fn result_round_trips() {
|
||||
let r = RequestHelpResult {
|
||||
outcome: HelpOutcome::Continued,
|
||||
completed_by: None,
|
||||
note: Some("done".into()),
|
||||
tab_id: 7,
|
||||
resolved_targets: Some(vec![ResolvedTarget {
|
||||
|
||||
Reference in New Issue
Block a user