mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(record): capture interactions in embedded frames
Track recording per document so injectable child frames are captured while partial frame failures remain visible in exported traces. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -16,6 +16,16 @@
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"frame_capture": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/FrameCaptureInfo"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -64,6 +74,77 @@
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"FrameCaptureFailure": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"frame_id",
|
||||
"reason"
|
||||
],
|
||||
"properties": {
|
||||
"detail": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"document_id": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frame_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/FrameCaptureFailureReason"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"FrameCaptureFailureReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"not_injectable",
|
||||
"arm_failed",
|
||||
"rearm_failed",
|
||||
"drain_failed",
|
||||
"flush_failed"
|
||||
]
|
||||
},
|
||||
"FrameCaptureInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status"
|
||||
],
|
||||
"properties": {
|
||||
"failures": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/FrameCaptureFailure"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/FrameCaptureStatus"
|
||||
}
|
||||
}
|
||||
},
|
||||
"FrameCaptureStatus": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"complete",
|
||||
"partial"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
|
||||
@@ -167,6 +167,46 @@ pub enum Step {
|
||||
},
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Frame capture metadata
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrameCaptureStatus {
|
||||
Complete,
|
||||
Partial,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrameCaptureFailureReason {
|
||||
NotInjectable,
|
||||
ArmFailed,
|
||||
RearmFailed,
|
||||
DrainFailed,
|
||||
FlushFailed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FrameCaptureFailure {
|
||||
pub reason: FrameCaptureFailureReason,
|
||||
pub frame_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub document_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FrameCaptureInfo {
|
||||
pub status: FrameCaptureStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub failures: Option<Vec<FrameCaptureFailure>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace root
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -183,6 +223,8 @@ pub struct Trace {
|
||||
pub stopped_by: StopReason,
|
||||
pub entry: TraceEntry,
|
||||
pub recorder: RecorderInfo,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frame_capture: Option<FrameCaptureInfo>,
|
||||
pub states: Vec<TraceState>,
|
||||
pub steps: Vec<Step>,
|
||||
}
|
||||
@@ -277,6 +319,7 @@ mod tests {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![
|
||||
TraceState {
|
||||
id: "s1".into(),
|
||||
@@ -479,4 +522,39 @@ mod tests {
|
||||
assert_eq!(trace.states.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_deserializes_optional_frame_capture_metadata() {
|
||||
let v = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "cli_stop",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"frame_capture": {
|
||||
"status": "partial",
|
||||
"failures": [
|
||||
{
|
||||
"reason": "not_injectable",
|
||||
"frame_id": 2,
|
||||
"url": "chrome://settings",
|
||||
"detail": "restricted frame url"
|
||||
}
|
||||
]
|
||||
},
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||
let frame_capture = trace.frame_capture.expect("frame_capture");
|
||||
assert_eq!(frame_capture.status, FrameCaptureStatus::Partial);
|
||||
let failures = frame_capture.failures.expect("failures");
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert_eq!(
|
||||
failures[0].reason,
|
||||
FrameCaptureFailureReason::NotInjectable
|
||||
);
|
||||
assert_eq!(failures[0].frame_id, 2);
|
||||
assert_eq!(failures[0].url.as_deref(), Some("chrome://settings"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user