mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(record): capture interactions in embedded frames
Track recording per document so injectable child frames are captured while partial frame failures remain visible in exported traces. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
@@ -8,8 +8,8 @@ use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::tools::{
|
||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
||||
RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||
FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult,
|
||||
RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, Method};
|
||||
use clap::{Args, Subcommand};
|
||||
@@ -550,6 +550,13 @@ fn render_finish(
|
||||
trace_path.display(),
|
||||
pages_dir.display()
|
||||
);
|
||||
if let Some(frame_capture) = &trace.frame_capture {
|
||||
if matches!(frame_capture.status, FrameCaptureStatus::Partial) {
|
||||
eprintln!(
|
||||
"warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -584,6 +591,7 @@ mod tests {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![TraceState {
|
||||
id: state_id.into(),
|
||||
url: "https://example.com/".into(),
|
||||
|
||||
@@ -96,6 +96,7 @@ fn sample_trace() -> Trace {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![TraceState {
|
||||
id: "s1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
|
||||
@@ -16,6 +16,16 @@
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"frame_capture": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/FrameCaptureInfo"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -64,6 +74,77 @@
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"FrameCaptureFailure": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"frame_id",
|
||||
"reason"
|
||||
],
|
||||
"properties": {
|
||||
"detail": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"document_id": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frame_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/FrameCaptureFailureReason"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"FrameCaptureFailureReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"not_injectable",
|
||||
"arm_failed",
|
||||
"rearm_failed",
|
||||
"drain_failed",
|
||||
"flush_failed"
|
||||
]
|
||||
},
|
||||
"FrameCaptureInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status"
|
||||
],
|
||||
"properties": {
|
||||
"failures": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/FrameCaptureFailure"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/FrameCaptureStatus"
|
||||
}
|
||||
}
|
||||
},
|
||||
"FrameCaptureStatus": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"complete",
|
||||
"partial"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
|
||||
@@ -167,6 +167,46 @@ pub enum Step {
|
||||
},
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Frame capture metadata
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrameCaptureStatus {
|
||||
Complete,
|
||||
Partial,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrameCaptureFailureReason {
|
||||
NotInjectable,
|
||||
ArmFailed,
|
||||
RearmFailed,
|
||||
DrainFailed,
|
||||
FlushFailed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FrameCaptureFailure {
|
||||
pub reason: FrameCaptureFailureReason,
|
||||
pub frame_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub document_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FrameCaptureInfo {
|
||||
pub status: FrameCaptureStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub failures: Option<Vec<FrameCaptureFailure>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace root
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -183,6 +223,8 @@ pub struct Trace {
|
||||
pub stopped_by: StopReason,
|
||||
pub entry: TraceEntry,
|
||||
pub recorder: RecorderInfo,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frame_capture: Option<FrameCaptureInfo>,
|
||||
pub states: Vec<TraceState>,
|
||||
pub steps: Vec<Step>,
|
||||
}
|
||||
@@ -277,6 +319,7 @@ mod tests {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![
|
||||
TraceState {
|
||||
id: "s1".into(),
|
||||
@@ -479,4 +522,39 @@ mod tests {
|
||||
assert_eq!(trace.states.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_deserializes_optional_frame_capture_metadata() {
|
||||
let v = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "cli_stop",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"frame_capture": {
|
||||
"status": "partial",
|
||||
"failures": [
|
||||
{
|
||||
"reason": "not_injectable",
|
||||
"frame_id": 2,
|
||||
"url": "chrome://settings",
|
||||
"detail": "restricted frame url"
|
||||
}
|
||||
]
|
||||
},
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||
let frame_capture = trace.frame_capture.expect("frame_capture");
|
||||
assert_eq!(frame_capture.status, FrameCaptureStatus::Partial);
|
||||
let failures = frame_capture.failures.expect("failures");
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert_eq!(
|
||||
failures[0].reason,
|
||||
FrameCaptureFailureReason::NotInjectable
|
||||
);
|
||||
assert_eq!(failures[0].frame_id, 2);
|
||||
assert_eq!(failures[0].url.as_deref(), Some("chrome://settings"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user